diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added new file mode 100644 index 000000000..f6250801a --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added @@ -0,0 +1 @@ +Add instrumentation for the OpenAI Responses ``retrieve`` API (sync and async), reported as a ``fetch_response`` operation with no token usage. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst b/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst index adc9b01e9..820c9e309 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst @@ -62,9 +62,15 @@ Check out the `manual example `_ for more details. Instrumenting all clients ************************* -When using the instrumentor, all clients will automatically trace OpenAI operations including chat completions and embeddings. +When using the instrumentor, all clients will automatically trace OpenAI operations including chat completions, the Responses API, and embeddings. You can also optionally capture prompts and completions as log events. +Fetching a stored response with ``client.responses.retrieve(...)`` performs no +inference, so it is reported as a ``fetch_response`` operation rather than as an +inference call. Its span carries the fetched response's identifier, model, +status, and finish reasons, but no token usage — those counts belong to the +operation that originally generated the response. + Make sure to configure OpenTelemetry tracing, logging, and events to capture all telemetry emitted by the instrumentation. .. code-block:: python @@ -82,6 +88,14 @@ Make sure to configure OpenTelemetry tracing, logging, and events to capture all ], ) + # Responses API example, fetching a stored response back by its id + created = client.responses.create( + model="gpt-4o-mini", + input="Write a short poem on open telemetry.", + store=True, + ) + fetched = client.responses.retrieve(created.id) + # Embeddings example embedding_response = client.embeddings.create( model="text-embedding-3-small", diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py index 1cc26ff0b..fc70098d1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py @@ -79,8 +79,10 @@ ) from .patch_responses import ( async_responses_create, + async_responses_retrieve, async_responses_stream, responses_create, + responses_retrieve, responses_stream, ) @@ -188,6 +190,20 @@ def _instrument(self, **kwargs): async_responses_stream(handler), ) + # retrieve() fetches a stored response by id. No inference happens + # and no tokens are consumed, so it is traced as a fetch_response + # operation rather than through the create wrappers. + wrap_function_wrapper( + "openai.resources.responses.responses", + "Responses.retrieve", + responses_retrieve(handler), + ) + wrap_function_wrapper( + "openai.resources.responses.responses", + "AsyncResponses.retrieve", + async_responses_retrieve(handler), + ) + def _uninstrument(self, **kwargs): import openai # pylint: disable=import-outside-toplevel @@ -204,6 +220,8 @@ def _uninstrument(self, **kwargs): unwrap(responses_module.Responses, "stream") unwrap(responses_module.AsyncResponses, "create") unwrap(responses_module.AsyncResponses, "stream") + unwrap(responses_module.Responses, "retrieve") + unwrap(responses_module.AsyncResponses, "retrieve") def _get_responses_module(): diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py index b9e8a2388..89c5f4530 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py @@ -6,24 +6,33 @@ from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING, Any, Union, cast +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import FetchResponseInvocation from ._raw_response import wrap_stream_result from .response_extractors import ( apply_request_attributes, extract_params, + get_fetch_response_creation_kwargs, get_inference_creation_kwargs, get_response_error, + is_streamed_raw_response, + set_fetch_response_attributes, set_invocation_response_attributes, ) from .response_wrappers import ( + AsyncFetchResponseStreamWrapper, AsyncResponseStreamManagerWrapper, AsyncResponseStreamWrapper, + FetchResponseStreamWrapper, ResponseStreamManagerWrapper, ResponseStreamWrapper, responses_stream_context, ) -from .utils import is_streaming +from .utils import is_streaming, value_is_set if TYPE_CHECKING: from openai import AsyncStream as OpenAIAsyncStream @@ -91,9 +100,9 @@ def traced_method( ) set_invocation_response_attributes( - invocation, result, capture_content + invocation, result, capture_content, kwargs ) - error = get_response_error(result) + error = get_response_error(result, kwargs) if error is not None: invocation.fail(error) else: @@ -164,9 +173,9 @@ async def traced_method( ) set_invocation_response_attributes( - invocation, result, capture_content + invocation, result, capture_content, kwargs ) - error = get_response_error(result) + error = get_response_error(result, kwargs) if error is not None: invocation.fail(error) else: @@ -182,6 +191,181 @@ async def traced_method( ) +def _get_retrieve_response_id( + args: tuple[Any, ...], kwargs: dict[str, Any] +) -> str | None: + """Return the ``response_id`` a ``responses.retrieve`` call was made with.""" + response_id = args[0] if args else kwargs.get("response_id") + return response_id if isinstance(response_id, str) else None + + +def _get_stream_cursor(kwargs: dict[str, Any]) -> str | None: + """Return the ``starting_after`` cursor a streamed fetch resumes from.""" + starting_after = kwargs.get("starting_after") + if not value_is_set(starting_after): + return None + return str(starting_after) + + +def _start_fetch_response_invocation( + handler: TelemetryHandler, + instance: Responses | AsyncResponses, + response_id: str, + kwargs: dict[str, Any], +) -> FetchResponseInvocation: + invocation = handler.fetch_response( + **get_fetch_response_creation_kwargs(response_id, instance), + request_stream=( + True + if is_streaming(kwargs) or is_streamed_raw_response(kwargs) + else None + ), + ) + invocation.stream_cursor = _get_stream_cursor(kwargs) + invocation.attributes[OpenAIAttributes.OPENAI_API_TYPE] = ( + OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) + return invocation + + +def responses_retrieve( + handler: TelemetryHandler, +) -> Callable[ + ..., + Union[ + ResponseResult, + ResponseStreamResult, + FetchResponseStreamWrapper[Any], + ], +]: + """Wrap ``Responses.retrieve`` to trace fetching a stored response by id. + + Traces :meth:`openai.resources.responses.responses.Responses.retrieve`. + Fetching a stored response performs no inference and consumes no tokens, so + it is reported as a ``fetch_response`` operation rather than as an + inference call. + """ + + capture_content = handler.should_capture_content() + + def traced_method( + wrapped: Callable[..., Union[ResponseResult, ResponseStreamResult]], + instance: Responses, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Union[ + ResponseResult, + ResponseStreamResult, + FetchResponseStreamWrapper[Any], + ]: + response_id = _get_retrieve_response_id(args, kwargs) + if response_id is None: + # gen_ai.response.id is required on a fetch_response span and the + # SDK rejects the call without it, so leave it untraced. + return wrapped(*args, **kwargs) + + invocation = _start_fetch_response_invocation( + handler, instance, response_id, kwargs + ) + + try: + result = wrapped(*args, **kwargs) + if is_streaming(kwargs): + return wrap_stream_result( + FetchResponseStreamWrapper, + result, + invocation, + capture_content, + ) + + set_fetch_response_attributes( + invocation, + result, + capture_content, + kwargs, + ) + invocation.stop() + return result + except Exception as error: + invocation.fail(error) + raise + + return cast( + 'Callable[..., Union["ResponseResult", "ResponseStreamResult", FetchResponseStreamWrapper[Any]]]', + traced_method, + ) + + +def async_responses_retrieve( + handler: TelemetryHandler, +) -> Callable[ + ..., + Awaitable[ + Union[ + ResponseResult, + AsyncResponseStreamResult, + AsyncFetchResponseStreamWrapper[Any], + ] + ], +]: + """Wrap ``AsyncResponses.retrieve`` to trace fetching a stored response by id. + + Traces :meth:`openai.resources.responses.responses.AsyncResponses.retrieve`. + Fetching a stored response performs no inference and consumes no tokens, so + it is reported as a ``fetch_response`` operation rather than as an + inference call. + """ + + capture_content = handler.should_capture_content() + + async def traced_method( + wrapped: Callable[ + ..., + Awaitable[Union[ResponseResult, AsyncResponseStreamResult]], + ], + instance: AsyncResponses, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Union[ + ResponseResult, + AsyncResponseStreamResult, + AsyncFetchResponseStreamWrapper[Any], + ]: + response_id = _get_retrieve_response_id(args, kwargs) + if response_id is None: + # gen_ai.response.id is required on a fetch_response span and the + # SDK rejects the call without it, so leave it untraced. + return await wrapped(*args, **kwargs) + + invocation = _start_fetch_response_invocation( + handler, instance, response_id, kwargs + ) + + try: + result = await wrapped(*args, **kwargs) + if is_streaming(kwargs): + return wrap_stream_result( + AsyncFetchResponseStreamWrapper, + result, + invocation, + capture_content, + ) + + set_fetch_response_attributes( + invocation, result, capture_content, kwargs + ) + invocation.stop() + return result + except Exception as error: + invocation.fail(error) + raise + + return cast( + 'Callable[..., Awaitable[Union["ResponseResult", "AsyncResponseStreamResult", AsyncFetchResponseStreamWrapper[Any]]]]', + traced_method, + ) + + def responses_stream( handler: TelemetryHandler, ) -> Callable[..., ResponseStreamManagerWrapper[Any]]: diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 696530be0..7a495b4dd 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -18,6 +18,7 @@ from ._raw_response import ParsableResponse from .utils import ( _openai_response_format_to_output_type, + get_property_value, get_served_model, get_server_address_and_port, ) @@ -31,6 +32,7 @@ InputMessage, OutputMessage, Text, + ToolDefinition, ) try: @@ -61,6 +63,8 @@ try: from opentelemetry.util.genai.types import ( Error, + FunctionToolDefinition, + GenericToolDefinition, InputMessage, OutputMessage, Reasoning, @@ -71,6 +75,8 @@ ) except ImportError: Error = None + FunctionToolDefinition = None + GenericToolDefinition = None InputMessage = None OutputMessage = None Reasoning = None @@ -258,14 +264,78 @@ def _extract_reasoning_parts( return parts -def _finish_reason_from_status(status: str | None) -> str | None: +# `incomplete_details.reason` values that map onto a cross-provider finish +# reason; an unrecognized reason is reported as-is. +_INCOMPLETE_REASON_TO_FINISH_REASON = { + "max_output_tokens": "length", + "content_filter": "content_filter", +} + + +def _finish_reason_from_status( + status: str | None, + incomplete_reason: str | None = None, +) -> str | None: if status == "completed": return "stop" - if status in {"failed", "cancelled", "incomplete"}: - return status + if status in {"failed", "cancelled"}: + return "error" + if status == "incomplete": + if incomplete_reason is None: + return status + return _INCOMPLETE_REASON_TO_FINISH_REASON.get( + incomplete_reason, incomplete_reason + ) return None +def get_tool_definitions_from_response( + response: Response | None, +) -> list[ToolDefinition] | None: + """Return the tool definitions carried on a fetched response. + + Responses API tools are flat -- a function tool holds ``name``, + ``description`` and ``parameters`` directly, unlike the Chat Completions + shape that nests them under ``function``. Built-in tools (``web_search``, + ``file_search``, ...) are identified by ``type`` alone and carry no name, + so they are reported as generic definitions keyed by their type. + """ + if ( + Response is None + or not isinstance(response, Response) + or FunctionToolDefinition is None + or GenericToolDefinition is None + ): + return None + + tools = response.tools + if not tools: + return None + + definitions: list[ToolDefinition] = [] + for tool in tools: + tool_type = get_property_value(tool, "type") + if not isinstance(tool_type, str): + continue + name = get_property_value(tool, "name") + if tool_type == "function": + definitions.append( + FunctionToolDefinition( + name=name if isinstance(name, str) else "", + description=get_property_value(tool, "description"), + parameters=get_property_value(tool, "parameters"), + ) + ) + else: + definitions.append( + GenericToolDefinition( + name=name if isinstance(name, str) else tool_type, + type=tool_type, + ) + ) + return definitions or None + + def _response_types_available() -> bool: return ( Response is not None @@ -353,6 +423,16 @@ def extract_finish_reasons(response: Response | None) -> list[str]: ): return [] + incomplete_details = response.incomplete_details + response_finish_reason = _finish_reason_from_status( + response.status, + incomplete_details.reason if incomplete_details is not None else None, + ) + if response.status in {"failed", "cancelled", "incomplete"}: + return [response_finish_reason] if response_finish_reason else [] + if response.status in {"queued", "in_progress"}: + return [] + finish_reasons: list[str] = [] for item in response.output: if isinstance(item, ResponseFunctionToolCall) and item.status in { @@ -367,16 +447,24 @@ def extract_finish_reasons(response: Response | None) -> list[str]: finish_reason = _finish_reason_from_status(item.status) if finish_reason is not None: finish_reasons.append(finish_reason) - return list(dict.fromkeys(finish_reasons)) + finish_reasons = list(dict.fromkeys(finish_reasons)) + if finish_reasons: + return finish_reasons + return [response_finish_reason] if response_finish_reason else [] -def get_response_error(response: Response | None) -> Error | None: +def get_response_error( + response: object, + request_kwargs: dict[str, object] | None = None, +) -> Error | None: """Return an ``Error`` when the response failed, else ``None``. A failed response carries a ``ResponseError`` (``code`` + ``message``). Incomplete responses (``incomplete_details``) are *not* errors — they surface as a finish reason instead. """ + response = _parse_raw_response(response, request_kwargs) + if Response is None or Error is None or not isinstance(response, Response): return None error = response.error @@ -403,11 +491,32 @@ def get_inference_creation_kwargs( return creation_kwargs +def get_fetch_response_creation_kwargs( + response_id: str, + client_instance: object, +) -> dict[str, object]: + """Return ``handler.fetch_response()`` kwargs for a ``responses.retrieve`` call.""" + address, port = get_server_address_and_port(client_instance) + + creation_kwargs: dict[str, object] = { + "provider": GenAIAttributes.GenAiProviderNameValues.OPENAI.value, + "response_id": response_id, + } + if address is not None: + creation_kwargs["server_address"] = address + if port is not None: + creation_kwargs["server_port"] = port + return creation_kwargs + + def apply_request_attributes( invocation, params: ResponseRequestParams, capture_content: bool, ) -> None: + invocation.attributes[OpenAIAttributes.OPENAI_API_TYPE] = ( + OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) invocation.temperature = params.temperature invocation.top_p = params.top_p invocation.max_tokens = params.max_output_tokens @@ -456,16 +565,43 @@ def extract_usage_tokens(usage: ResponseUsage | None) -> UsageTokens: ) +_RAW_RESPONSE_HEADER = "x-stainless-raw-response" + + +def is_streamed_raw_response( + request_kwargs: dict[str, object] | None, +) -> bool: + if request_kwargs is None: + return False + extra_headers = request_kwargs.get("extra_headers") + if not isinstance(extra_headers, Mapping): + return False + return any( + key.lower() == _RAW_RESPONSE_HEADER and value == "stream" + for key, value in extra_headers.items() + ) + + +def _parse_raw_response( + response: object, + request_kwargs: dict[str, object] | None, +) -> object: + """Return the payload of a non-streaming ``with_raw_response`` result.""" + if is_streamed_raw_response(request_kwargs) or not isinstance( + response, ParsableResponse + ): + return response + return response.parse() + + def set_invocation_response_attributes( invocation, response: object, capture_content: bool, + request_kwargs: dict[str, object] | None = None, ) -> None: served_model = get_served_model(getattr(response, "headers", None)) - if isinstance(response, ParsableResponse): - # with_raw_response: safe to parse() here since this is the - # non-streaming path, so it has no side effects on the caller's stream. - response = response.parse() + response = _parse_raw_response(response, request_kwargs) if Response is None or not isinstance(response, Response): return @@ -494,3 +630,48 @@ def set_invocation_response_attributes( output_messages = get_output_messages_from_response(response) if output_messages: invocation.output_messages = output_messages + + +def set_fetch_response_attributes( + invocation, + response: object, + capture_content: bool, + request_kwargs: dict[str, object] | None = None, +) -> None: + """Record a fetched response on a ``fetch_response`` invocation. + + Token usage is deliberately not recorded: the fetch performs no inference + and the counts on the fetched response belong to the original generation. + The original input messages are not part of the fetched response either, so + only the system instructions and output messages it carries are captured. + """ + served_model = get_served_model(getattr(response, "headers", None)) + response = _parse_raw_response(response, request_kwargs) + + if Response is None or not isinstance(response, Response): + return + + invocation.response_model_name = served_model or response.model + invocation.response_status = response.status + invocation.finish_reasons = extract_finish_reasons(response) or None + + # `service_tier` is absent from the Response model on some supported SDK + # versions, so keep this attribute access guarded. + service_tier = getattr(response, "service_tier", None) + if service_tier is not None: + invocation.attributes[ + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER + ] = service_tier + + if capture_content: + invocation.system_instruction = get_system_instruction( + response.instructions + if isinstance(response.instructions, str) + else None + ) + invocation.output_messages = get_output_messages_from_response( + response + ) + invocation.tool_definitions = get_tool_definitions_from_response( + response + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py index 3add42190..37258f089 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py @@ -20,10 +20,12 @@ try: from opentelemetry.instrumentation.genai.openai.response_extractors import ( # pylint: disable=no-name-in-module get_response_error, + set_fetch_response_attributes, set_invocation_response_attributes, ) except ImportError: get_response_error = None + set_fetch_response_attributes = None set_invocation_response_attributes = None try: @@ -88,6 +90,16 @@ def _set_response_attributes( set_invocation_response_attributes(invocation, result, capture_content) +def _set_fetch_response_attributes( + invocation: GenAIInvocation, + result: ParsedResponse[TextFormatT] | Response | None, + capture_content: bool, +) -> None: + if set_fetch_response_attributes is None: + return + set_fetch_response_attributes(invocation, result, capture_content) + + def _get_stream_response(stream): try: return stream._response @@ -147,9 +159,7 @@ def _stop( ) -> None: if self._self_response_telemetry_finalized: return - _set_response_attributes( - self._self_invocation, result, self._self_capture_content - ) + self._apply_response_attributes(result) if get_served_model is not None: stream = getattr(self, "stream", None) if stream is not None: @@ -159,10 +169,25 @@ def _stop( ) if served_model: self._self_invocation.response_model_name = served_model - self._self_invocation.stop() self._self_response_telemetry_finalized = True + def _apply_response_attributes( + self, result: ParsedResponse[TextFormatT] | Response | None + ) -> None: + """Record the response on the invocation. Overridden per operation.""" + _set_response_attributes( + self._self_invocation, result, self._self_capture_content + ) + + def _on_response_failed( + self, response: ParsedResponse[TextFormatT] | Response | None + ) -> None: + """Handle a ``response.failed`` event. Overridden per operation.""" + self._apply_response_attributes(response) + error = get_response_error(response) if get_response_error else None + self._fail(error or Error(type="response.failed", message=None)) + def _fail(self, error: Error) -> None: if self._self_response_telemetry_finalized: return @@ -221,15 +246,7 @@ def process_event(self, event: ResponseStreamEvent[TextFormatT]) -> None: return if event_type == "response.failed": - _set_response_attributes( - self._self_invocation, - response, - self._self_capture_content, - ) - error = ( - get_response_error(response) if get_response_error else None - ) - self._fail(error or Error(type=event_type, message=None)) + self._on_response_failed(response) return if event.type == "error": @@ -272,6 +289,39 @@ def stream(self, stream: ResponseStream[TextFormatT]) -> None: self._self_iterator = iter(stream) +class _FetchResponseStreamMixin(Generic[TextFormatT]): + """Finalization overrides for a streamed ``responses.retrieve``. + + The stream replays a response generated by an earlier operation, so the + fetch-response attributes are recorded instead of the inference ones, and a + replayed ``response.failed`` describes that original generation rather than + a failure of this fetch. + """ + + _self_invocation: GenAIInvocation + _self_capture_content: bool + + def _apply_response_attributes( + self, result: ParsedResponse[TextFormatT] | Response | None + ) -> None: + _set_fetch_response_attributes( + self._self_invocation, result, self._self_capture_content + ) + + def _on_response_failed( + self, response: ParsedResponse[TextFormatT] | Response | None + ) -> None: + self._stop(response) + + +class FetchResponseStreamWrapper( + _FetchResponseStreamMixin[TextFormatT], + ResponseStreamWrapper[TextFormatT], + Generic[TextFormatT], +): + """Wrapper for a streamed ``Responses.retrieve`` replay.""" + + class ResponseStreamManagerWrapper(Generic[TextFormatT]): """Wrapper for OpenAI Responses API stream managers. @@ -411,6 +461,14 @@ def response(self): return _AsyncResponseProxy(response, lambda: self._stop(None)) +class AsyncFetchResponseStreamWrapper( + _FetchResponseStreamMixin[TextFormatT], + AsyncResponseStreamWrapper[TextFormatT], + Generic[TextFormatT], +): + """Wrapper for a streamed ``AsyncResponses.retrieve`` replay.""" + + class AsyncResponseStreamManagerWrapper(Generic[TextFormatT]): """Wrapper for async OpenAI Responses API stream managers.""" diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_fetch_conformance.yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_fetch_conformance.yaml new file mode 100644 index 000000000..9ef9d6b95 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_fetch_conformance.yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml new file mode 100644 index 000000000..e1ebd3075 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml @@ -0,0 +1,85 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_doesnotexist0000000000000000000000000000000000 + response: + body: + string: |- + { + "error": { + "message": "Response with id 'resp_doesnotexist0000000000000000000000000000000000' not found.", + "type": "invalid_request_error", + "param": null, + "code": null + } + } + headers: + CF-RAY: + - 9ee06c562f5c0f7d-EWR + Connection: + - keep-alive + Content-Length: + - '173' + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:08 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '107' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_48f6dc739d654972b34e069edb97d2dc + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml new file mode 100644 index 000000000..2692791ed --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml new file mode 100644 index 000000000..2692791ed --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml new file mode 100644 index 000000000..90f729695 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml @@ -0,0 +1,136 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027", + "object": "response", + "created_at": 1776481253, + "status": "failed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": { + "code": "server_error", + "message": "The model failed to generate a response." + }, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_incomplete[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_incomplete[content_mode0].yaml new file mode 100644 index 000000000..f74dd4dbd --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_incomplete[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "object": "response", + "created_at": 1776481253, + "status": "incomplete", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": { + "reason": "max_output_tokens" + }, + "instructions": "You are a helpful assistant.", + "max_output_tokens": 16, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "type": "message", + "status": "incomplete", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_raw_response[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_raw_response[content_mode0].yaml new file mode 100644 index 000000000..5d6286ca8 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_raw_response[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_streaming[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_streaming[content_mode0].yaml new file mode 100644 index 000000000..02c7d63e3 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_streaming[content_mode0].yaml @@ -0,0 +1,89 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028?starting_after=3&stream=true + response: + body: + string: |+ + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":"This is","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":4} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" a test.","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":5} + + event: response.output_text.done + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":6,"text":"This is a test."} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","object":"response","created_at":1776481253,"status":"completed","background":false,"billing":{"payer":"developer"},"completed_at":1776481257,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"This is a test."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":28},"user":null,"metadata":{}},"sequence_number":7} + + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml new file mode 100644 index 000000000..5d6286ca8 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml new file mode 100644 index 000000000..e1ebd3075 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml @@ -0,0 +1,85 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_doesnotexist0000000000000000000000000000000000 + response: + body: + string: |- + { + "error": { + "message": "Response with id 'resp_doesnotexist0000000000000000000000000000000000' not found.", + "type": "invalid_request_error", + "param": null, + "code": null + } + } + headers: + CF-RAY: + - 9ee06c562f5c0f7d-EWR + Connection: + - keep-alive + Content-Length: + - '173' + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:08 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '107' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_48f6dc739d654972b34e069edb97d2dc + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml new file mode 100644 index 000000000..2692791ed --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml new file mode 100644 index 000000000..2692791ed --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml new file mode 100644 index 000000000..9c6e04e13 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml @@ -0,0 +1,136 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027", + "object": "response", + "created_at": 1776481253, + "status": "failed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": { + "code": "server_error", + "message": "The model failed to generate a response." + }, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_incomplete[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_incomplete[content_mode0].yaml new file mode 100644 index 000000000..8f74594df --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_incomplete[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "object": "response", + "created_at": 1776481253, + "status": "incomplete", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": { + "reason": "max_output_tokens" + }, + "instructions": "You are a helpful assistant.", + "max_output_tokens": 16, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "type": "message", + "status": "incomplete", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_raw_response[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_raw_response[content_mode0].yaml new file mode 100644 index 000000000..9ef9d6b95 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_raw_response[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_streaming[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_streaming[content_mode0].yaml new file mode 100644 index 000000000..68c76bd43 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_streaming[content_mode0].yaml @@ -0,0 +1,89 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028?starting_after=3&stream=true + response: + body: + string: |+ + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":"This is","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":4} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" a test.","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":5} + + event: response.output_text.done + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":6,"text":"This is a test."} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","object":"response","created_at":1776481253,"status":"completed","background":false,"billing":{"payer":"developer"},"completed_at":1776481257,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"This is a test."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":28},"user":null,"metadata":{}},"sequence_number":7} + + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml new file mode 100644 index 000000000..9ef9d6b95 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_fetch.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_fetch.py new file mode 100644 index 000000000..9c588219d --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_fetch.py @@ -0,0 +1,48 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: OpenAI Responses API fetch by id.""" + +from __future__ import annotations + +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +RESPONSE_ID = "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" + + +class ResponsesFetchScenario(Scenario): + """Fetching a stored response emits a ``fetch_response`` span. + + The operation performs no inference, so no token usage is recorded and + ``gen_ai.client.token.usage`` is intentionally absent. + """ + + expected_spans = {"fetch_response": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("responses_fetch_conformance.yaml"): + OpenAI().responses.retrieve(RESPONSE_ID) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index a30575ec2..53baeb704 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -25,18 +25,24 @@ from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) +from opentelemetry.trace.status import StatusCode from opentelemetry.util.genai.utils import is_experimental_mode from .test_responses import assert_responses_streaming_timing_metrics from .test_utils import ( DEFAULT_MODEL, + GEN_AI_RESPONSE_STATUS, USER_ONLY_EXPECTED_INPUT_MESSAGES, USER_ONLY_PROMPT, assert_all_attributes, assert_cache_attributes, + assert_fetch_response_attributes, assert_messages_attribute, format_simple_expected_output_message, get_responses_weather_tool_definition, @@ -193,10 +199,284 @@ async def test_async_responses_create_basic( assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( "stop", ) + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes +RETRIEVE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" +) +RETRIEVE_INCOMPLETE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026" +) +RETRIEVE_FAILED_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027" +) +RETRIEVE_STREAM_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028" +) +RETRIEVE_MISSING_RESPONSE_ID = ( + "resp_doesnotexist0000000000000000000000000000000000" +) +RETRIEVE_STREAM_CURSOR = 3 + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_basic( + span_exporter, async_openai_client, instrument_no_content, vcr +): + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_basic[content_mode0].yaml" + ): + response = await async_openai_client.responses.retrieve( + RETRIEVE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, + ) + assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS not in span.attributes + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_captures_content( + span_exporter, + log_exporter, + async_openai_client, + instrument_with_content, + vcr, +): + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_captures_content[content_mode0].yaml" + ): + response = await async_openai_client.responses.retrieve( + RETRIEVE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), + ) + assert ( + json.loads(span.attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS]) + == EXPECTED_SYSTEM_INSTRUCTIONS + ) + # A fetched response does not carry the original input messages. + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + assert len(log_exporter.get_finished_logs()) == 0 + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_incomplete( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """An incomplete stored response surfaces via status and finish reasons.""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_incomplete[content_mode0].yaml" + ): + response = await async_openai_client.responses.retrieve( + RETRIEVE_INCOMPLETE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="incomplete", + finish_reasons=("length",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_failed_generation_is_not_a_fetch_error( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """A stored response whose generation failed is not a failure of the fetch.""" + _skip_if_not_latest() + + cassette = ( + "test_async_responses_retrieve_failed_generation_is_not_a_fetch_error" + "[content_mode0].yaml" + ) + with vcr.use_cassette(cassette): + response = await async_openai_client.responses.retrieve( + RETRIEVE_FAILED_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="failed", + finish_reasons=("error",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_streaming( + span_exporter, async_openai_client, instrument_with_content, vcr +): + """A streamed replay finalizes only once the caller drains the stream.""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_streaming[content_mode0].yaml" + ): + stream = await async_openai_client.responses.retrieve( + RETRIEVE_STREAM_RESPONSE_ID, + stream=True, + starting_after=RETRIEVE_STREAM_CURSOR, + ) + assert isinstance(stream, AsyncStream) + assert span_exporter.get_finished_spans() == () + + response = await _collect_completed_response(stream) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=RETRIEVE_STREAM_RESPONSE_ID, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + request_stream=True, + stream_cursor=str(RETRIEVE_STREAM_CURSOR), + response_service_tier=response.service_tier, + ) + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), + ) + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_raw_response( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """``with_raw_response`` keeps returning the raw response, still traced.""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_raw_response[content_mode0].yaml" + ): + raw_response = ( + await async_openai_client.responses.with_raw_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + ) + response = raw_response.parse() + if inspect.isawaitable(response): + response = await response + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, + ) + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_with_streaming_response_stays_lazy( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """``with_streaming_response`` must not have its body read by telemetry.""" + _skip_if_not_latest() + + cassette = ( + "test_async_responses_retrieve_with_streaming_response_stays_lazy" + "[content_mode0].yaml" + ) + with vcr.use_cassette(cassette): + async with ( + async_openai_client.responses.with_streaming_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + ) as raw_response: + # Building telemetry must not consume or close the body before the + # caller reads it. + assert not raw_response.http_response.is_stream_consumed + assert not raw_response.http_response.is_closed + response = raw_response.parse() + if inspect.isawaitable(response): + response = await response + + (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_RESPONSE_ID + ) + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_STREAM] is True + assert response.id == RETRIEVE_RESPONSE_ID + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_api_error( + span_exporter, async_openai_client, instrument_no_content, vcr +): + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_api_error[content_mode0].yaml" + ): + with pytest.raises(NotFoundError) as exc_info: + await async_openai_client.responses.retrieve( + RETRIEVE_MISSING_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_MISSING_RESPONSE_ID + ) + assert span.status.status_code is StatusCode.ERROR + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == f"openai.{type(exc_info.value).__name__}" + ) + # The fetch failed before any response existed to describe. + assert GEN_AI_RESPONSE_STATUS not in span.attributes + + @pytest.mark.asyncio() async def test_async_responses_with_raw_response_streaming( span_exporter, async_openai_client, instrument_with_content, vcr diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py index 0ed83ccd6..14c665488 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py @@ -27,6 +27,7 @@ from .conformance.inference import InferenceScenario from .conformance.inference_streaming import InferenceStreamingScenario from .conformance.responses_conversation import ResponsesConversationScenario +from .conformance.responses_fetch import ResponsesFetchScenario from .conformance.responses_stream import ResponsesStreamScenario from .conformance.responses_streaming import ResponsesStreamingScenario from .conformance.tool_calling import ToolCallingScenario @@ -40,6 +41,7 @@ EmbeddingScenario(), ToolCallingScenario(), ResponsesConversationScenario(), + ResponsesFetchScenario(), ResponsesStreamScenario(), ResponsesStreamingScenario(), ], diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index d3cfc1bb4..9faebeeda 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -11,7 +11,11 @@ from opentelemetry.semconv._incubating.attributes import ( openai_attributes as OpenAIAttributes, ) -from opentelemetry.util.genai.types import LLMInvocation +from opentelemetry.util.genai.types import ( + FunctionToolDefinition, + GenericToolDefinition, + LLMInvocation, +) try: # Responses types are not available in the oldest supported OpenAI SDK. @@ -241,6 +245,36 @@ def test_get_response_error_returns_error_for_failed_response(loaded_module): assert error.message == "boom" +def test_get_response_error_parses_raw_response(loaded_module): + response = _make_response( + status="failed", + error={"code": "server_error", "message": "boom"}, + ) + raw_response = SimpleNamespace(parse=mock.Mock(return_value=response)) + + error = loaded_module.get_response_error(raw_response) + + assert error is not None + assert error.type == "server_error" + raw_response.parse.assert_called_once_with() + + +def test_get_response_error_keeps_streaming_raw_response_lazy(loaded_module): + raw_response = SimpleNamespace(parse=mock.Mock()) + + error = loaded_module.get_response_error( + raw_response, + { + "extra_headers": { + "X-Stainless-Raw-Response": "stream", + } + }, + ) + + assert error is None + raw_response.parse.assert_not_called() + + def test_get_response_error_none_for_incomplete_response(loaded_module): # Incomplete is a finish reason, not an error. response = _make_response( @@ -463,6 +497,198 @@ def test_response_extractors_ignore_invalid_shapes_without_validation( assert not invocation.attributes +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + ({"status": "completed"}, ["stop"]), + ({"status": "failed"}, ["error"]), + ({"status": "cancelled"}, ["error"]), + ( + { + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + ["length"], + ), + ( + { + "status": "incomplete", + "incomplete_details": {"reason": "content_filter"}, + }, + ["content_filter"], + ), + ({"status": "incomplete"}, ["incomplete"]), + # Non-terminal and unknown statuses: generation is not known to have + # stopped, so there is no finish reason to report. gen_ai.response.status + # conveys the lifecycle state instead. + ({"status": "queued"}, []), + ({"status": "in_progress"}, []), + ({}, []), + ], +) +def test_extract_finish_reasons_maps_response_status( + loaded_module, overrides, expected +): + response = _make_response(**overrides) + + assert loaded_module.extract_finish_reasons(response) == expected + + +def test_set_fetch_response_attributes_tolerates_missing_service_tier(): + """`service_tier` is absent from the Response model on older SDKs. + + Deleting the field from the instance makes attribute access raise + ``AttributeError``, exactly as it does on an SDK whose ``Response`` model + never declared it (for example openai 1.70). + """ + response = _make_response(status="completed") + del response.__dict__["service_tier"] + with pytest.raises(AttributeError): + response.service_tier # pylint: disable=pointless-statement + + invocation = SimpleNamespace( + response_model_name=None, + response_status=None, + finish_reasons=None, + output_messages=[], + system_instruction=[], + attributes={}, + ) + + response_extractors.set_fetch_response_attributes( + invocation, response, capture_content=False + ) + + assert invocation.response_status == "completed" + assert invocation.finish_reasons == ["stop"] + assert ( + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER + not in invocation.attributes + ) + + +def test_get_tool_definitions_from_response_maps_flat_responses_tools( + loaded_module, +): + """Responses API tools are flat, unlike the nested Chat Completions shape.""" + response = _make_response( + tools=[ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + "strict": True, + } + ] + ) + + definitions = loaded_module.get_tool_definitions_from_response(response) + + (definition,) = definitions + assert isinstance(definition, FunctionToolDefinition) + assert definition.type == "function" + assert definition.name == "get_weather" + assert definition.description == "Get the weather" + assert definition.parameters == { + "type": "object", + "properties": {"city": {"type": "string"}}, + } + + +def test_get_tool_definitions_from_response_maps_builtin_tools_by_type( + loaded_module, +): + """Built-in tools carry no name, so their type identifies them.""" + response = _make_response( + tools=[{"type": "web_search_preview"}], + ) + + definitions = loaded_module.get_tool_definitions_from_response(response) + + (definition,) = definitions + assert isinstance(definition, GenericToolDefinition) + assert definition.type == "web_search_preview" + assert definition.name == "web_search_preview" + + +def test_get_tool_definitions_from_response_returns_none_without_tools( + loaded_module, +): + assert loaded_module.get_tool_definitions_from_response(None) is None + assert ( + loaded_module.get_tool_definitions_from_response(_make_response()) + is None + ) + + +def test_set_fetch_response_attributes_captures_tool_definitions( + loaded_module, +): + """Tool definitions are captured only when content capture is enabled.""" + response = _make_response( + status="completed", + tools=[ + { + "type": "function", + "name": "get_weather", + "description": None, + "parameters": {"type": "object"}, + "strict": True, + } + ], + ) + + def _make_invocation(): + return SimpleNamespace( + response_model_name=None, + response_status=None, + finish_reasons=None, + output_messages=[], + system_instruction=[], + tool_definitions=None, + attributes={}, + ) + + captured = _make_invocation() + loaded_module.set_fetch_response_attributes( + captured, response, capture_content=True + ) + (definition,) = captured.tool_definitions + assert isinstance(definition, FunctionToolDefinition) + assert definition.name == "get_weather" + + not_captured = _make_invocation() + loaded_module.set_fetch_response_attributes( + not_captured, response, capture_content=False + ) + assert not_captured.tool_definitions is None + + +def test_set_fetch_response_attributes_prefers_raw_served_model_header( + loaded_module, +): + invocation = SimpleNamespace( + response_model_name=None, + response_status=None, + finish_reasons=None, + output_messages=[], + system_instruction=[], + attributes={}, + ) + raw_response = _RawResponse(_make_response(model="body-gpt-4.1")) + + loaded_module.set_fetch_response_attributes( + invocation, raw_response, capture_content=False + ) + + assert raw_response.parse_count == 1 + assert invocation.response_model_name == "served-gpt-4.1" + + def test_get_served_model_returns_value_when_present(): headers = {"x-ms-served-model": "gpt-4o-2024-08-06"} assert get_served_model(headers) == "gpt-4o-2024-08-06" diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py index 4a34b411d..2d930b386 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py @@ -8,6 +8,7 @@ from opentelemetry.instrumentation.genai.openai.response_wrappers import ( AsyncResponseStreamManagerWrapper, AsyncResponseStreamWrapper, + FetchResponseStreamWrapper, ResponseStreamManagerWrapper, ResponseStreamWrapper, ) @@ -650,3 +651,91 @@ def test_process_event_error_event_records_error(): (error,) = calls["fail"] assert error.type == "rate_limit_exceeded" assert error.message == "slow down" + + +def _make_fetch_stream_wrapper(stream, invocation): + return FetchResponseStreamWrapper( + stream=stream, + invocation=invocation, + capture_content=False, + ) + + +def _capturing_fetch_invocation(): + calls = {"stop": 0, "fail": []} + invocation = SimpleNamespace( + response_model_name=None, + response_status=None, + finish_reasons=None, + stream_cursor=None, + output_messages=[], + system_instruction=[], + tool_definitions=None, + attributes={}, + ) + invocation.stop = lambda: calls.__setitem__("stop", calls["stop"] + 1) + invocation.fail = lambda error: calls["fail"].append(error) + return invocation, calls + + +@_requires_responses_types +def test_fetch_stream_completed_records_fetch_response_attributes(): + invocation, calls = _capturing_fetch_invocation() + wrapper = _make_fetch_stream_wrapper( + _FakeSyncStream(), invocation=invocation + ) + event = SimpleNamespace( + type="response.completed", + response=_make_response(status="completed"), + ) + + wrapper.process_event(event) + + assert calls["stop"] == 1 + assert calls["fail"] == [] + assert invocation.response_status == "completed" + assert invocation.finish_reasons == ["stop"] + + +@_requires_responses_types +def test_fetch_stream_replayed_failure_is_not_an_error_of_the_fetch(): + """A replayed ``response.failed`` describes the original generation.""" + invocation, calls = _capturing_fetch_invocation() + wrapper = _make_fetch_stream_wrapper( + _FakeSyncStream(), invocation=invocation + ) + event = SimpleNamespace( + type="response.failed", + response=_make_response( + status="failed", + error={"code": "server_error", "message": "boom"}, + ), + ) + + wrapper.process_event(event) + + assert calls["fail"] == [] + assert calls["stop"] == 1 + assert invocation.response_status == "failed" + assert invocation.finish_reasons == ["error"] + + +@_requires_responses_types +def test_fetch_stream_transport_error_event_still_fails_the_fetch(): + """An SSE ``error`` event is a failure of the fetch itself.""" + invocation, calls = _capturing_fetch_invocation() + wrapper = _make_fetch_stream_wrapper( + _FakeSyncStream(), invocation=invocation + ) + event = SimpleNamespace( + type="error", + code="rate_limit_exceeded", + message="slow down", + response=None, + ) + + wrapper.process_event(event) + + assert calls["stop"] == 0 + (error,) = calls["fail"] + assert error.type == "rate_limit_exceeded" diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 56fd074ee..9753d71e0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -24,18 +24,24 @@ from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.trace.status import StatusCode from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( DEFAULT_MODEL, + GEN_AI_RESPONSE_STATUS, USER_ONLY_EXPECTED_INPUT_MESSAGES, USER_ONLY_PROMPT, assert_all_attributes, assert_cache_attributes, + assert_fetch_response_attributes, assert_messages_attribute, format_simple_expected_output_message, get_responses_weather_tool_definition, @@ -148,6 +154,15 @@ def _collect_completed_response(stream): return response +def _collect_metrics(metric_reader): + metrics = {} + for rm in metric_reader.get_metrics_data().resource_metrics: + for scope in rm.scope_metrics: + for metric in scope.metrics: + metrics[metric.name] = metric + return metrics + + def assert_responses_streaming_timing_metrics(metric_reader): """Assert the streaming timing metrics are emitted through the real Responses stream wrapper path. @@ -157,11 +172,7 @@ def assert_responses_streaming_timing_metrics(metric_reader): green but silently stop emitting TTFC and per-output-chunk metrics for the Responses streaming path. """ - metrics = {} - for rm in metric_reader.get_metrics_data().resource_metrics: - for scope in rm.scope_metrics: - for metric in scope.metrics: - metrics[metric.name] = metric + metrics = _collect_metrics(metric_reader) ttfc = metrics.get( gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK @@ -252,10 +263,262 @@ def test_responses_create_basic( assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( "stop", ) + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes +RETRIEVE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" +) +RETRIEVE_INCOMPLETE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026" +) +RETRIEVE_FAILED_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027" +) +RETRIEVE_STREAM_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028" +) +RETRIEVE_MISSING_RESPONSE_ID = ( + "resp_doesnotexist0000000000000000000000000000000000" +) +RETRIEVE_STREAM_CURSOR = 3 + + +@pytest.mark.vcr() +def test_responses_retrieve_basic( + span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, + ) + assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_captures_content( + span_exporter, log_exporter, openai_client, instrument_with_content +): + _skip_if_not_latest() + + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), + ) + assert ( + json.loads(span.attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS]) + == EXPECTED_SYSTEM_INSTRUCTIONS + ) + # A fetched response does not carry the original input messages. + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + assert len(log_exporter.get_finished_logs()) == 0 + + +@pytest.mark.vcr() +def test_responses_retrieve_incomplete( + span_exporter, openai_client, instrument_no_content +): + """An incomplete stored response surfaces via status and finish reasons.""" + _skip_if_not_latest() + + response = openai_client.responses.retrieve( + RETRIEVE_INCOMPLETE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="incomplete", + finish_reasons=("length",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_failed_generation_is_not_a_fetch_error( + span_exporter, openai_client, instrument_no_content +): + """A stored response whose generation failed is not a failure of the fetch.""" + _skip_if_not_latest() + + response = openai_client.responses.retrieve(RETRIEVE_FAILED_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="failed", + finish_reasons=("error",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_streaming( + span_exporter, openai_client, instrument_with_content +): + """A streamed replay finalizes only once the caller drains the stream.""" + _skip_if_not_latest() + + stream = openai_client.responses.retrieve( + RETRIEVE_STREAM_RESPONSE_ID, + stream=True, + starting_after=RETRIEVE_STREAM_CURSOR, + ) + assert isinstance(stream, Stream) + assert span_exporter.get_finished_spans() == () + + response = _collect_completed_response(stream) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=RETRIEVE_STREAM_RESPONSE_ID, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + request_stream=True, + stream_cursor=str(RETRIEVE_STREAM_CURSOR), + response_service_tier=response.service_tier, + ) + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), + ) + + +@pytest.mark.vcr() +def test_responses_retrieve_raw_response( + span_exporter, openai_client, instrument_no_content +): + """``with_raw_response`` keeps returning the raw response, still traced.""" + _skip_if_not_latest() + + raw_response = openai_client.responses.with_raw_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + response = raw_response.parse() + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, + ) + + +@pytest.mark.vcr() +def test_responses_retrieve_with_streaming_response_stays_lazy( + span_exporter, openai_client, instrument_no_content +): + """``with_streaming_response`` must not have its body read by telemetry.""" + _skip_if_not_latest() + + with openai_client.responses.with_streaming_response.retrieve( + RETRIEVE_RESPONSE_ID + ) as raw_response: + # Building telemetry must not consume or close the body before the + # caller reads it. + assert not raw_response.http_response.is_stream_consumed + assert not raw_response.http_response.is_closed + response = raw_response.parse() + + (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_RESPONSE_ID + ) + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_STREAM] is True + assert response.id == RETRIEVE_RESPONSE_ID + + +@pytest.mark.vcr() +def test_responses_retrieve_api_error( + span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + + with pytest.raises(NotFoundError) as exc_info: + openai_client.responses.retrieve(RETRIEVE_MISSING_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_MISSING_RESPONSE_ID + ) + assert span.status.status_code is StatusCode.ERROR + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == f"openai.{type(exc_info.value).__name__}" + ) + # The fetch failed before any response existed to describe. + assert GEN_AI_RESPONSE_STATUS not in span.attributes + + +def test_responses_retrieve_does_not_record_token_usage_metric( + span_exporter, metric_reader, openai_client, instrument_no_content, vcr +): + """A fetch consumes no tokens, so only the duration metric is recorded.""" + _skip_if_not_latest() + + with vcr.use_cassette("test_responses_retrieve_basic[content_mode0].yaml"): + openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + + metrics = _collect_metrics(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE not in metrics + + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + (point,) = duration.data.data_points + assert ( + point.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + point.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + == "gpt-4o-mini-2024-07-18" + ) + # The response id is high cardinality and must stay off metrics. + assert GenAIAttributes.GEN_AI_RESPONSE_ID not in point.attributes + + @pytest.mark.vcr() def test_responses_create_captures_content( request, diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py index 6ef3cccc9..f748f95f1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py @@ -16,9 +16,16 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) +from opentelemetry.trace import SpanKind DEFAULT_MODEL = "gpt-4o-mini" DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small" +FETCH_RESPONSE_OPERATION_NAME = "fetch_response" +# TODO: use the semconv constants once these attributes are released in +# opentelemetry-semantic-conventions. Added to the GenAI semantic conventions +# in https://github.com/open-telemetry/semantic-conventions-genai/pull/353. +GEN_AI_REQUEST_STREAM_CURSOR = "gen_ai.request.stream_cursor" +GEN_AI_RESPONSE_STATUS = "gen_ai.response.status" GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = ( "gen_ai.usage.cache_creation.input_tokens" ) @@ -146,6 +153,67 @@ def assert_all_attributes( ) +def assert_fetch_response_attributes( + span: ReadableSpan, + *, + response_id: str, + response_model: str | None = None, + response_status: str | None = None, + finish_reasons: tuple | None = None, + request_stream: bool | None = None, + stream_cursor: str | None = None, + response_service_tier: str | None = None, + server_address: str = "api.openai.com", +): + """Assert a ``gen_ai.fetch_response.client`` span matches the semconv. + + Fetching a stored response performs no inference, so the span must carry + neither request-side attributes nor any token usage from the fetched + response — those counts belong to the original generation. + """ + # The response id is high cardinality, so it stays out of the span name. + assert span.name == FETCH_RESPONSE_OPERATION_NAME + assert span.kind is SpanKind.CLIENT + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == FETCH_RESPONSE_OPERATION_NAME + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] + == GenAIAttributes.GenAiProviderNameValues.OPENAI.value + ) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response_id + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) + assert span.attributes[ServerAttributes.SERVER_ADDRESS] == server_address + + assert GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS not in span.attributes + assert GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS not in span.attributes + assert GenAIAttributes.GEN_AI_REQUEST_MODEL not in span.attributes + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + + _assert_optional_attribute( + span, GenAIAttributes.GEN_AI_RESPONSE_MODEL, response_model + ) + _assert_optional_attribute(span, GEN_AI_RESPONSE_STATUS, response_status) + _assert_optional_attribute( + span, GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, finish_reasons + ) + _assert_optional_attribute( + span, GenAIAttributes.GEN_AI_REQUEST_STREAM, request_stream + ) + _assert_optional_attribute( + span, GEN_AI_REQUEST_STREAM_CURSOR, stream_cursor + ) + _assert_optional_attribute( + span, + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER, + response_service_tier, + ) + + def assert_log_parent(log, span): """Assert that the log record has the correct parent span context""" if span: diff --git a/policies/genai_span_validation.rego b/policies/genai_span_validation.rego index e53dfd538..fbd03defa 100644 --- a/policies/genai_span_validation.rego +++ b/policies/genai_span_validation.rego @@ -274,6 +274,7 @@ _known_operation_names := { "generate_content", "text_completion", "embeddings", + "fetch_response", "retrieval", "create_agent", "invoke_agent", diff --git a/util/opentelemetry-util-genai/.changelog/184.added b/util/opentelemetry-util-genai/.changelog/184.added new file mode 100644 index 000000000..69bd6cbd5 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/184.added @@ -0,0 +1 @@ +Add ``TelemetryHandler.fetch_response()`` and ``FetchResponseInvocation`` for the ``fetch_response`` operation, which fetches a previously generated response by id without performing inference. diff --git a/util/opentelemetry-util-genai/AGENTS.md b/util/opentelemetry-util-genai/AGENTS.md index a7a7c8555..05ad9aa87 100644 --- a/util/opentelemetry-util-genai/AGENTS.md +++ b/util/opentelemetry-util-genai/AGENTS.md @@ -44,6 +44,7 @@ Factory methods on `TelemetryHandler` (`handler.py`): - `inference(provider, request_model, *, server_address, server_port)` → `InferenceInvocation` - `embedding(provider, request_model, *, server_address, server_port)` → `EmbeddingInvocation` - `retrieval(*, data_source_id, provider, request_model, server_address, server_port)` → `RetrievalInvocation` +- `fetch_response(provider, *, response_id, server_address, server_port)` → `FetchResponseInvocation` - `tool(name, *, arguments, tool_call_id, tool_type, tool_description)` → `ToolInvocation` - `workflow(name)` → `WorkflowInvocation` diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py new file mode 100644 index 000000000..d3bdb75b7 --- /dev/null +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py @@ -0,0 +1,190 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Final + +from opentelemetry._logs import Logger +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import server_attributes +from opentelemetry.trace import SpanKind, Tracer +from opentelemetry.util.genai._invocation import ( + Error, + GenAIInvocation, + get_content_attributes, +) +from opentelemetry.util.genai.completion_hook import CompletionHook +from opentelemetry.util.genai.metrics import InvocationMetricsRecorder +from opentelemetry.util.genai.types import ( + ErrorTypeResolver, + MessagePart, + OutputMessage, + ToolDefinition, +) +from opentelemetry.util.types import AttributeValue + +# TODO: Migrate to gen_ai_attributes constants once available in the semconv +# package. Added to the GenAI semantic conventions in +# https://github.com/open-telemetry/semantic-conventions-genai/pull/353. +_FETCH_RESPONSE_OPERATION_NAME: Final = "fetch_response" +_GEN_AI_REQUEST_STREAM_CURSOR: Final = "gen_ai.request.stream_cursor" +_GEN_AI_RESPONSE_STATUS: Final = "gen_ai.response.status" + + +class FetchResponseInvocation(GenAIInvocation): + """Represents a single fetch of a previously generated model response. + + Use handler.fetch_response() rather than constructing this directly. + + Reference: https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-spans.md#fetch-response + + The operation performs no inference and consumes no tokens: it returns a + response produced by an earlier operation. Any token counts carried on the + fetched response describe that original generation and MUST NOT be reported + here, so this invocation deliberately exposes no token usage fields. + + Semantic convention attributes for fetch response spans: + - gen_ai.operation.name: "fetch_response" (Required) + - gen_ai.provider.name: Provider name (Required) + - gen_ai.response.id: Identifier of the response being fetched (Required) + - error.type: Error type when the fetch itself failed (Conditionally Required) + - gen_ai.request.stream_cursor: Set from ``stream_cursor`` when the fetch + resumes a streamed response from a prior position (Conditionally Required) + - gen_ai.request.stream: Set to true when the fetched response is streamed + (Conditionally Required) + - server.port: Set only when ``server_port`` is provided (Conditionally Required) + - gen_ai.response.finish_reasons: Outcome of the original generation + (Recommended) + - gen_ai.response.model: Set from ``response_model_name`` (Recommended) + - gen_ai.response.status: Lifecycle status of the fetched response + (Recommended) + - server.address: Set only when ``server_address`` is provided (Recommended) + - gen_ai.output.messages, gen_ai.system_instructions, + gen_ai.tool.definitions: content carried on the fetched response, + recorded on the span only when content capturing is enabled (Opt-In). + A fetched response does not carry the original input messages, so + gen_ai.input.messages is never set. + + A fetched response whose *original* generation failed is not a failure of + the fetch: report it through ``response_status`` and ``finish_reasons`` and + still call ``stop()``. Only call ``fail()`` when the fetch call itself + failed. + """ + + def __init__( + self, + tracer: Tracer, + metrics_recorder: InvocationMetricsRecorder, + logger: Logger, + completion_hook: CompletionHook, + provider: str, + *, + response_id: str, + request_stream: bool | None = None, + server_address: str | None = None, + server_port: int | None = None, + error_type_resolver: ErrorTypeResolver | None = None, + ) -> None: + """Use handler.fetch_response() rather than calling this directly.""" + super().__init__( + tracer, + metrics_recorder, + logger, + completion_hook, + operation_name=_FETCH_RESPONSE_OPERATION_NAME, + # The response identifier is high cardinality, so semconv keeps it + # out of the span name. + span_name=_FETCH_RESPONSE_OPERATION_NAME, + span_kind=SpanKind.CLIENT, + error_type_resolver=error_type_resolver, + ) + self._provider: str = provider + self._response_id: str = response_id + self._request_stream = request_stream + self._server_address: str | None = server_address + self._server_port: int | None = server_port + self.response_model_name: str | None = None + self.response_status: str | None = None + self.finish_reasons: list[str] | None = None + self.stream_cursor: str | None = None + self.output_messages: list[OutputMessage] = [] + self.system_instruction: list[MessagePart] = [] + self.tool_definitions: list[ToolDefinition] | None = None + self._start(self._get_start_attributes()) + + @property + def response_id(self) -> str: + """The identifier of the response being fetched.""" + return self._response_id + + def _get_start_attributes(self) -> dict[str, AttributeValue]: + """Return attributes known at span creation time.""" + optional_attrs: tuple[tuple[str, AttributeValue | None], ...] = ( + (server_attributes.SERVER_ADDRESS, self._server_address), + (server_attributes.SERVER_PORT, self._server_port), + ) + return { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + GenAI.GEN_AI_PROVIDER_NAME: self._provider, + GenAI.GEN_AI_RESPONSE_ID: self._response_id, + **( + {GenAI.GEN_AI_REQUEST_STREAM: self._request_stream} + if self._request_stream is not None + else {} + ), + **{k: v for k, v in optional_attrs if v is not None}, + } + + def _get_metric_attributes(self) -> dict[str, AttributeValue]: + # response_id intentionally excluded — high cardinality. + optional_attrs: tuple[tuple[str, AttributeValue | None], ...] = ( + (GenAI.GEN_AI_RESPONSE_MODEL, self.response_model_name), + (server_attributes.SERVER_ADDRESS, self._server_address), + (server_attributes.SERVER_PORT, self._server_port), + ) + attrs: dict[str, AttributeValue] = { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + GenAI.GEN_AI_PROVIDER_NAME: self._provider, + **{k: v for k, v in optional_attrs if v is not None}, + } + attrs.update(self.metric_attributes) + return attrs + + def _get_attributes(self) -> dict[str, AttributeValue]: + optional_attrs: tuple[tuple[str, AttributeValue | None], ...] = ( + (_GEN_AI_REQUEST_STREAM_CURSOR, self.stream_cursor), + ( + GenAI.GEN_AI_RESPONSE_FINISH_REASONS, + self.finish_reasons or None, + ), + (GenAI.GEN_AI_RESPONSE_MODEL, self.response_model_name), + (_GEN_AI_RESPONSE_STATUS, self.response_status), + ) + return {k: v for k, v in optional_attrs if v is not None} + + def _apply_finish(self, error: Error | None = None) -> None: + if error is not None: + self._apply_error_attributes(error) + attributes = self._get_attributes() + attributes.update( + get_content_attributes( + # A fetched response does not carry the original request's + # input messages. + input_messages=(), + output_messages=self.output_messages, + system_instruction=self.system_instruction, + tool_definitions=self.tool_definitions, + for_span=True, + ) + ) + attributes.update(self.attributes) + self.span.set_attributes(attributes) + self._metrics_recorder.record(self) + self._call_completion_hook( + outputs=self.output_messages, + system_instruction=self.system_instruction, + tool_definitions=self.tool_definitions, + ) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py index a3134f3c3..50e2a031c 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py @@ -57,6 +57,7 @@ ) from opentelemetry.util.genai.invocation import ( EmbeddingInvocation, + FetchResponseInvocation, InferenceInvocation, RetrievalInvocation, ToolInvocation, @@ -363,6 +364,41 @@ def embedding( server_port=server_port, ) + def fetch_response( + self, + provider: str, + *, + response_id: str, + request_stream: bool | None = None, + server_address: str | None = None, + server_port: int | None = None, + error_type_resolver: ErrorTypeResolver | None = None, + ) -> FetchResponseInvocation: + """Returns a Fetch Response invocation. Starts span when called. + + Describes fetching a previously generated model response by its + identifier. No inference is performed and no tokens are consumed, so + the fetched response's token counts must not be recorded here. + + Returned object can be used as a ContextManager which automatically calls `stop` or `fail` + to finalize the span upon exiting. If not used as a ContextManager, the caller is + responsible for calling `stop` or `fail` to finalize the span. + + Only set data attributes on the invocation object, do not modify the span or context. + """ + return FetchResponseInvocation( + self._tracer, + self._metrics_recorder, + self._logger, + self._completion_hook, + provider=provider, + response_id=response_id, + request_stream=request_stream, + server_address=server_address, + server_port=server_port, + error_type_resolver=error_type_resolver, + ) + def tool( self, name: str, diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py index 07badfef7..37b27823d 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py @@ -8,6 +8,7 @@ from opentelemetry.util.genai.invocation import ( Error, GenAIInvocation, + FetchResponseInvocation, InferenceInvocation, EmbeddingInvocation, RetrievalInvocation, @@ -18,6 +19,9 @@ from opentelemetry.util.genai._agent_invocation import AgentInvocation from opentelemetry.util.genai._embedding_invocation import EmbeddingInvocation +from opentelemetry.util.genai._fetch_response_invocation import ( + FetchResponseInvocation, +) from opentelemetry.util.genai._inference_invocation import InferenceInvocation from opentelemetry.util.genai._invocation import ( ContextToken, @@ -33,6 +37,7 @@ "ContextToken", "EmbeddingInvocation", "Error", + "FetchResponseInvocation", "GenAIInvocation", "InferenceInvocation", "RetrievalInvocation", diff --git a/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py b/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py new file mode 100644 index 000000000..e94facb66 --- /dev/null +++ b/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py @@ -0,0 +1,321 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +from unittest import TestCase +from unittest.mock import patch + +import pytest + +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import server_attributes +from opentelemetry.trace import INVALID_SPAN, SpanKind +from opentelemetry.trace.status import StatusCode +from opentelemetry.util.genai.environment_variables import ( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import FetchResponseInvocation +from opentelemetry.util.genai.types import ( + Error, + FunctionToolDefinition, + OutputMessage, + Text, +) + +# TODO: use the semconv constants once these attributes are released in +# opentelemetry-semantic-conventions. +GEN_AI_REQUEST_STREAM_CURSOR = "gen_ai.request.stream_cursor" +GEN_AI_RESPONSE_STATUS = "gen_ai.response.status" + +RESPONSE_ID = "resp_123" + + +class _FetchResponseTestBase(TestCase): + def setUp(self) -> None: + self.span_exporter = InMemorySpanExporter() + self.tracer_provider = TracerProvider() + self.tracer_provider.add_span_processor( + SimpleSpanProcessor(self.span_exporter) + ) + self.metric_reader = InMemoryMetricReader() + self.meter_provider = MeterProvider( + metric_readers=[self.metric_reader] + ) + self.handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + + def _fetch_response(self, **kwargs) -> FetchResponseInvocation: + return self.handler.fetch_response( + "openai", response_id=RESPONSE_ID, **kwargs + ) + + def _get_finished_spans(self): + return self.span_exporter.get_finished_spans() + + def _get_metrics(self): + metrics = {} + data = self.metric_reader.get_metrics_data() + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + metrics[metric.name] = metric + return metrics + + +class TelemetryHandlerFetchResponseTest(_FetchResponseTestBase): + # ------------------------------------------------------------------ + # span creation + # ------------------------------------------------------------------ + + def test_fetch_response_creates_span(self) -> None: + invocation = self._fetch_response() + self.assertIsNot(invocation.span, INVALID_SPAN) + invocation.stop() + + def test_span_name_omits_the_high_cardinality_response_id(self) -> None: + self._fetch_response().stop() + + spans = self._get_finished_spans() + self.assertEqual(len(spans), 1) + self.assertEqual(spans[0].name, "fetch_response") + + def test_span_kind_is_client(self) -> None: + self._fetch_response().stop() + + self.assertEqual(self._get_finished_spans()[0].kind, SpanKind.CLIENT) + + def test_records_monotonic_start(self) -> None: + with patch("timeit.default_timer", return_value=42.0): + invocation = self._fetch_response() + self.assertEqual(invocation._monotonic_start_s, 42.0) + invocation.stop() + + # ------------------------------------------------------------------ + # required and conditionally required attributes + # ------------------------------------------------------------------ + + def test_required_attributes_are_set_at_span_creation(self) -> None: + invocation = self._fetch_response() + + attrs = invocation.span.attributes + self.assertEqual(attrs[GenAI.GEN_AI_OPERATION_NAME], "fetch_response") + self.assertEqual(attrs[GenAI.GEN_AI_PROVIDER_NAME], "openai") + self.assertEqual(attrs[GenAI.GEN_AI_RESPONSE_ID], RESPONSE_ID) + invocation.stop() + + def test_response_id_is_exposed(self) -> None: + invocation = self._fetch_response() + self.assertEqual(invocation.response_id, RESPONSE_ID) + invocation.stop() + + def test_request_stream_is_set_at_span_creation(self) -> None: + invocation = self._fetch_response(request_stream=True) + + self.assertIs( + invocation.span.attributes[GenAI.GEN_AI_REQUEST_STREAM], True + ) + invocation.stop() + + def test_stop_sets_server_address_and_port(self) -> None: + self._fetch_response( + server_address="api.openai.com", server_port=8080 + ).stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual( + attrs[server_attributes.SERVER_ADDRESS], "api.openai.com" + ) + self.assertEqual(attrs[server_attributes.SERVER_PORT], 8080) + + def test_stop_sets_stream_cursor(self) -> None: + invocation = self._fetch_response() + invocation.stream_cursor = "42" + invocation.stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual(attrs[GEN_AI_REQUEST_STREAM_CURSOR], "42") + + # ------------------------------------------------------------------ + # recommended attributes + # ------------------------------------------------------------------ + + def test_stop_sets_response_model_status_and_finish_reasons(self) -> None: + invocation = self._fetch_response() + invocation.response_model_name = "gpt-4o-mini-2024-07-18" + invocation.response_status = "completed" + invocation.finish_reasons = ["stop"] + invocation.stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual( + attrs[GenAI.GEN_AI_RESPONSE_MODEL], "gpt-4o-mini-2024-07-18" + ) + self.assertEqual(attrs[GEN_AI_RESPONSE_STATUS], "completed") + self.assertEqual( + attrs[GenAI.GEN_AI_RESPONSE_FINISH_REASONS], ("stop",) + ) + + def test_stop_omits_unset_attributes(self) -> None: + self._fetch_response().stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertNotIn(GenAI.GEN_AI_RESPONSE_MODEL, attrs) + self.assertNotIn(GEN_AI_RESPONSE_STATUS, attrs) + self.assertNotIn(GenAI.GEN_AI_RESPONSE_FINISH_REASONS, attrs) + self.assertNotIn(GenAI.GEN_AI_REQUEST_STREAM, attrs) + self.assertNotIn(GEN_AI_REQUEST_STREAM_CURSOR, attrs) + self.assertNotIn(server_attributes.SERVER_ADDRESS, attrs) + self.assertNotIn(server_attributes.SERVER_PORT, attrs) + + def test_stop_sets_custom_attributes(self) -> None: + invocation = self._fetch_response() + invocation.attributes["openai.api.type"] = "responses" + invocation.stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual(attrs["openai.api.type"], "responses") + + # ------------------------------------------------------------------ + # metrics — a fetch performs no inference, so no token usage + # ------------------------------------------------------------------ + + def test_stop_records_duration_without_token_usage(self) -> None: + invocation = self._fetch_response() + invocation.response_model_name = "gpt-4o-mini-2024-07-18" + invocation.stop() + + metrics = self._get_metrics() + self.assertNotIn("gen_ai.client.token.usage", metrics) + + duration = metrics["gen_ai.client.operation.duration"] + (point,) = duration.data.data_points + self.assertEqual( + point.attributes[GenAI.GEN_AI_OPERATION_NAME], "fetch_response" + ) + self.assertEqual( + point.attributes[GenAI.GEN_AI_RESPONSE_MODEL], + "gpt-4o-mini-2024-07-18", + ) + # High cardinality — must stay off the metric. + self.assertNotIn(GenAI.GEN_AI_RESPONSE_ID, point.attributes) + + # ------------------------------------------------------------------ + # fail + # ------------------------------------------------------------------ + + def test_fail_sets_error_status_and_type(self) -> None: + invocation = self._fetch_response() + invocation.fail(Error(message="not found", type="NotFoundError")) + + span = self._get_finished_spans()[0] + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.status.description, "not found") + self.assertEqual(span.attributes["error.type"], "NotFoundError") + + def test_fail_with_exception_instance(self) -> None: + invocation = self._fetch_response() + invocation.fail(ValueError("oops")) + + span = self._get_finished_spans()[0] + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.attributes["error.type"], "ValueError") + + def test_fail_records_error_type_on_duration_metric(self) -> None: + self._fetch_response().fail(ValueError("oops")) + + duration = self._get_metrics()["gen_ai.client.operation.duration"] + (point,) = duration.data.data_points + self.assertEqual(point.attributes["error.type"], "ValueError") + + # ------------------------------------------------------------------ + # context manager + # ------------------------------------------------------------------ + + def test_context_manager_ends_span(self) -> None: + with self._fetch_response() as invocation: + self.assertIsInstance(invocation, FetchResponseInvocation) + + spans = self._get_finished_spans() + self.assertEqual(len(spans), 1) + self.assertEqual(spans[0].status.status_code, StatusCode.UNSET) + + def test_context_manager_reraises_exception(self) -> None: + with pytest.raises(ValueError, match="fetch failed"): + with self._fetch_response(): + raise ValueError("fetch failed") + + span = self._get_finished_spans()[0] + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.attributes["error.type"], "ValueError") + + +class TelemetryHandlerFetchResponseContentTest(_FetchResponseTestBase): + # ------------------------------------------------------------------ + # opt-in content + # ------------------------------------------------------------------ + + def _fetch_with_content(self) -> None: + invocation = self._fetch_response() + invocation.output_messages = [ + OutputMessage( + role="assistant", + parts=[Text(content="This is a test.")], + finish_reason="stop", + ) + ] + invocation.system_instruction = [ + Text(content="You are a helpful assistant.") + ] + invocation.tool_definitions = [ + FunctionToolDefinition( + name="get_weather", description=None, parameters=None + ) + ] + invocation.stop() + + def test_content_captured_on_span_when_enabled(self) -> None: + with patch.dict( + os.environ, + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_ONLY"}, + ): + self._fetch_with_content() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual( + json.loads(attrs[GenAI.GEN_AI_OUTPUT_MESSAGES]), + [ + { + "role": "assistant", + "parts": [{"content": "This is a test.", "type": "text"}], + "finish_reason": "stop", + } + ], + ) + self.assertEqual( + json.loads(attrs[GenAI.GEN_AI_SYSTEM_INSTRUCTIONS]), + [{"content": "You are a helpful assistant.", "type": "text"}], + ) + # A fetched response carries no input messages. + self.assertNotIn(GenAI.GEN_AI_INPUT_MESSAGES, attrs) + + def test_content_suppressed_on_span_when_disabled(self) -> None: + self._fetch_with_content() + + attrs = self._get_finished_spans()[0].attributes + self.assertNotIn(GenAI.GEN_AI_OUTPUT_MESSAGES, attrs) + self.assertNotIn(GenAI.GEN_AI_SYSTEM_INSTRUCTIONS, attrs)