From dee9d28de249188748c001a9950084a08fabbcd7 Mon Sep 17 00:00:00 2001 From: Jackson Weber <47067795+JacksonWeber@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:14:27 -0700 Subject: [PATCH 01/14] Instrument OpenAI Responses.retrieve and AsyncResponses.retrieve These fetch a stored response by id without going through create(), so the existing create wrappers don't cover them. Wrap both sync and async, reusing the create wrappers since the result is the same Response shape. Add sync/async tests and cassettes. Refs open-telemetry/opentelemetry-python-genai#141 Assisted-by: Claude Opus 4.8 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../+responses-retrieve-instrumentation.added | 1 + .../instrumentation/genai/openai/__init__.py | 22 +++ ...sponses_retrieve_basic[content_mode0].yaml | 149 ++++++++++++++++++ ...rieve_captures_content[content_mode0].yaml | 149 ++++++++++++++++++ ...sponses_retrieve_basic[content_mode0].yaml | 149 ++++++++++++++++++ ...rieve_captures_content[content_mode0].yaml | 149 ++++++++++++++++++ .../tests/test_async_responses.py | 62 ++++++++ .../tests/test_responses.py | 54 +++++++ 8 files changed, 735 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/+responses-retrieve-instrumentation.added create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/+responses-retrieve-instrumentation.added b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/+responses-retrieve-instrumentation.added new file mode 100644 index 000000000..7ad29db4c --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/+responses-retrieve-instrumentation.added @@ -0,0 +1 @@ +Add instrumentation for the OpenAI Responses ``retrieve`` API (sync and async). diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py index 55d74e100..95c044f66 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py @@ -177,6 +177,24 @@ def _instrument(self, **kwargs): async_responses_create(handler), ) + # retrieve() fetches a stored response by id without going through + # create(), so wrap it separately. The result is a Response, so the + # create wrappers extract its telemetry correctly. + if hasattr(responses_module.Responses, "retrieve"): + wrap_function_wrapper( + "openai.resources.responses.responses", + "Responses.retrieve", + responses_create(handler), + ) + if hasattr(responses_module, "AsyncResponses") and hasattr( + responses_module.AsyncResponses, "retrieve" + ): + wrap_function_wrapper( + "openai.resources.responses.responses", + "AsyncResponses.retrieve", + async_responses_create(handler), + ) + def _uninstrument(self, **kwargs): import openai # pylint: disable=import-outside-toplevel # noqa: PLC0415 @@ -190,8 +208,12 @@ def _uninstrument(self, **kwargs): responses_module = _get_responses_module() if responses_module is not None: unwrap(responses_module.Responses, "create") + if hasattr(responses_module.Responses, "retrieve"): + unwrap(responses_module.Responses, "retrieve") if hasattr(responses_module, "AsyncResponses"): unwrap(responses_module.AsyncResponses, "create") + if hasattr(responses_module.AsyncResponses, "retrieve"): + unwrap(responses_module.AsyncResponses, "retrieve") def _get_responses_module(): diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml new file mode 100644 index 000000000..04750000e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml @@ -0,0 +1,149 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml new file mode 100644 index 000000000..04750000e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml @@ -0,0 +1,149 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml new file mode 100644 index 000000000..04750000e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml @@ -0,0 +1,149 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml new file mode 100644 index 000000000..04750000e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml @@ -0,0 +1,149 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac362", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1575' + openai-organization: test_openai_org_id + openai-processing-ms: + - '7177' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 86d41ae10..ccb06fda8 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -184,6 +184,68 @@ async def test_async_responses_create_basic( assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes +RETRIEVE_RESPONSE_ID = "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_basic( + span_exporter, async_openai_client, instrument_no_content, vcr +): + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_basic[content_mode0].yaml" + ): + response = await async_openai_client.responses.retrieve( + RETRIEVE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + # retrieve carries no request model/input; telemetry comes from the + # fetched response only. + assert GenAIAttributes.GEN_AI_REQUEST_MODEL not in span.attributes + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + == response.model + ) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response.id + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + == response.usage.input_tokens + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + == response.usage.output_tokens + ) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( + "stop", + ) + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_captures_content( + span_exporter, async_openai_client, instrument_with_content, vcr +): + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_captures_content[content_mode0].yaml" + ): + response = await async_openai_client.responses.retrieve( + RETRIEVE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), + ) + # retrieve has no request-side input messages to capture. + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + + @pytest.mark.asyncio() async def test_async_responses_create_captures_content( span_exporter, diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 07c0e2ec6..b496bb05f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -202,6 +202,60 @@ def test_responses_create_basic( assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes +RETRIEVE_RESPONSE_ID = "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" + + +@pytest.mark.vcr() +def test_responses_retrieve_basic( + span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + # retrieve carries no request model/input; telemetry comes from the + # fetched response only. + assert GenAIAttributes.GEN_AI_REQUEST_MODEL not in span.attributes + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + == response.model + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response.id + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + == response.usage.input_tokens + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + == response.usage.output_tokens + ) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( + "stop", + ) + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_captures_content( + span_exporter, openai_client, instrument_with_content +): + _skip_if_not_latest() + + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), + ) + # retrieve has no request-side input messages to capture. + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + + @pytest.mark.vcr() def test_responses_create_captures_content( request, From adb32fca724d638fd25aa30c11176ee27f70d8cd Mon Sep 17 00:00:00 2001 From: Jackson Weber <47067795+JacksonWeber@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:22:00 -0700 Subject: [PATCH 02/14] Rename changelog fragment to PR number and apply ruff format Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...responses-retrieve-instrumentation.added => 184.added} | 0 .../tests/test_async_responses.py | 4 +++- .../tests/test_responses.py | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) rename instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/{+responses-retrieve-instrumentation.added => 184.added} (100%) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/+responses-retrieve-instrumentation.added b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added similarity index 100% rename from instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/+responses-retrieve-instrumentation.added rename to instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index ccb06fda8..dc8ace178 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -184,7 +184,9 @@ async def test_async_responses_create_basic( assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes -RETRIEVE_RESPONSE_ID = "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" +RETRIEVE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" +) @pytest.mark.asyncio() diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index b496bb05f..2e50e0371 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -202,7 +202,9 @@ def test_responses_create_basic( assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes -RETRIEVE_RESPONSE_ID = "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" +RETRIEVE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" +) @pytest.mark.vcr() @@ -221,9 +223,7 @@ def test_responses_retrieve_basic( span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] == response.model ) - assert ( - span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response.id - ) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response.id assert ( span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == response.usage.input_tokens From 13645fbf89a77e6d50250c3d7a0315e8e25e6573 Mon Sep 17 00:00:00 2001 From: Jackson Weber <47067795+JacksonWeber@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:36:56 -0700 Subject: [PATCH 03/14] Add error-scenario tests for Responses retrieve instrumentation Cover the 404 NotFoundError path for sync and async retrieve, matching the happy-path-plus-error coverage of the other Responses operations. Assisted-by: Claude Opus 4.8 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ses_retrieve_api_error[content_mode0].yaml | 84 +++++++++++++++++++ ...ses_retrieve_api_error[content_mode0].yaml | 84 +++++++++++++++++++ .../tests/test_async_responses.py | 24 ++++++ .../tests/test_responses.py | 19 +++++ 4 files changed, 211 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml new file mode 100644 index 000000000..6dedd5d58 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml @@ -0,0 +1,84 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_doesnotexist0000000000000000000000000000000000 + response: + body: + string: |- + { + "error": { + "message": "Response with id 'resp_doesnotexist0000000000000000000000000000000000' not found.", + "type": "invalid_request_error", + "param": null, + "code": null + } + } + headers: + CF-RAY: + - 9ee06c562f5c0f7d-EWR + Connection: + - keep-alive + Content-Length: + - '173' + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:08 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '107' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_48f6dc739d654972b34e069edb97d2dc + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml new file mode 100644 index 000000000..6dedd5d58 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml @@ -0,0 +1,84 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - Bearer test_openai_api_key + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_doesnotexist0000000000000000000000000000000000 + response: + body: + string: |- + { + "error": { + "message": "Response with id 'resp_doesnotexist0000000000000000000000000000000000' not found.", + "type": "invalid_request_error", + "param": null, + "code": null + } + } + headers: + CF-RAY: + - 9ee06c562f5c0f7d-EWR + Connection: + - keep-alive + Content-Length: + - '173' + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:01:08 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '107' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_48f6dc739d654972b34e069edb97d2dc + status: + code: 404 + message: Not Found +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index dc8ace178..0d0967f03 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -187,6 +187,9 @@ async def test_async_responses_create_basic( RETRIEVE_RESPONSE_ID = ( "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" ) +RETRIEVE_MISSING_RESPONSE_ID = ( + "resp_doesnotexist0000000000000000000000000000000000" +) @pytest.mark.asyncio() @@ -248,6 +251,27 @@ async def test_async_responses_retrieve_captures_content( assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes +@pytest.mark.asyncio() +async def test_async_responses_retrieve_api_error( + span_exporter, async_openai_client, instrument_no_content, vcr +): + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_api_error[content_mode0].yaml" + ): + with pytest.raises(NotFoundError) as exc_info: + await async_openai_client.responses.retrieve( + RETRIEVE_MISSING_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == type(exc_info.value).__name__ + ) + + @pytest.mark.asyncio() async def test_async_responses_create_captures_content( span_exporter, diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 2e50e0371..cf4a8c106 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -205,6 +205,9 @@ def test_responses_create_basic( RETRIEVE_RESPONSE_ID = ( "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" ) +RETRIEVE_MISSING_RESPONSE_ID = ( + "resp_doesnotexist0000000000000000000000000000000000" +) @pytest.mark.vcr() @@ -256,6 +259,22 @@ def test_responses_retrieve_captures_content( assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes +@pytest.mark.vcr() +def test_responses_retrieve_api_error( + span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + + with pytest.raises(NotFoundError) as exc_info: + openai_client.responses.retrieve(RETRIEVE_MISSING_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == type(exc_info.value).__name__ + ) + + @pytest.mark.vcr() def test_responses_create_captures_content( request, From 09bab0a4258d15a27d2706501422022076d9fafd Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Fri, 31 Jul 2026 15:26:01 -0700 Subject: [PATCH 04/14] Mark retrieve cassettes as AI-generated The six retrieve cassettes reuse the CF-RAY, x-request-id and Date headers of the existing responses create cassettes, so they were synthesized rather than recorded against the provider. AGENTS.md requires such cassettes to carry the re-record marker. Assisted-by: Claude Opus 5 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../test_async_responses_retrieve_api_error[content_mode0].yaml | 1 + .../test_async_responses_retrieve_basic[content_mode0].yaml | 1 + ...async_responses_retrieve_captures_content[content_mode0].yaml | 1 + .../test_responses_retrieve_api_error[content_mode0].yaml | 1 + .../cassettes/test_responses_retrieve_basic[content_mode0].yaml | 1 + .../test_responses_retrieve_captures_content[content_mode0].yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml index 6dedd5d58..e1ebd3075 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_api_error[content_mode0].yaml @@ -1,3 +1,4 @@ +# TODO: this is generated by AI, re-record interactions: - request: body: null diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml index 04750000e..2692791ed 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_basic[content_mode0].yaml @@ -1,3 +1,4 @@ +# TODO: this is generated by AI, re-record interactions: - request: body: null diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml index 04750000e..2692791ed 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_captures_content[content_mode0].yaml @@ -1,3 +1,4 @@ +# TODO: this is generated by AI, re-record interactions: - request: body: null diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml index 6dedd5d58..e1ebd3075 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_api_error[content_mode0].yaml @@ -1,3 +1,4 @@ +# TODO: this is generated by AI, re-record interactions: - request: body: null diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml index 04750000e..2692791ed 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_basic[content_mode0].yaml @@ -1,3 +1,4 @@ +# TODO: this is generated by AI, re-record interactions: - request: body: null diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml index 04750000e..2692791ed 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_captures_content[content_mode0].yaml @@ -1,3 +1,4 @@ +# TODO: this is generated by AI, re-record interactions: - request: body: null From 1e0556c5697d218518b554818797c9ac8b85d459 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Tue, 4 Aug 2026 11:10:52 -0700 Subject: [PATCH 05/14] Report Responses.retrieve as a fetch_response operation Adds a FetchResponseInvocation to opentelemetry-util-genai and wires the OpenAI Responses retrieve instrumentation to it, so fetching a stored response is no longer reported as an inference operation and does not record token usage. Assisted-by: Claude Opus 5 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../.changelog/184.added | 2 +- .../README.rst | 16 +- .../instrumentation/genai/openai/__init__.py | 12 +- .../genai/openai/patch_responses.py | 217 +++++++++++- .../genai/openai/response_extractors.py | 94 ++++++ .../genai/openai/response_wrappers.py | 81 ++++- .../responses_fetch_conformance.yaml | 148 +++++++++ ...n_is_not_a_fetch_error[content_mode0].yaml | 136 ++++++++ ...es_retrieve_incomplete[content_mode0].yaml | 150 +++++++++ ..._retrieve_raw_response[content_mode0].yaml | 148 +++++++++ ...ses_retrieve_streaming[content_mode0].yaml | 89 +++++ ...ng_response_stays_lazy[content_mode0].yaml | 148 +++++++++ ...n_is_not_a_fetch_error[content_mode0].yaml | 136 ++++++++ ...es_retrieve_incomplete[content_mode0].yaml | 150 +++++++++ ..._retrieve_raw_response[content_mode0].yaml | 148 +++++++++ ...ses_retrieve_streaming[content_mode0].yaml | 89 +++++ ...ng_response_stays_lazy[content_mode0].yaml | 148 +++++++++ .../tests/conformance/responses_fetch.py | 48 +++ .../tests/test_async_responses.py | 225 +++++++++++-- .../tests/test_conformance.py | 2 + .../tests/test_response_extractors.py | 70 ++++ .../tests/test_response_wrappers.py | 89 +++++ .../tests/test_responses.py | 231 +++++++++++-- .../tests/test_utils.py | 64 ++++ policies/genai_span_validation.rego | 1 + .../.changelog/184.added | 1 + util/opentelemetry-util-genai/AGENTS.md | 1 + .../util/genai/_fetch_response_invocation.py | 181 ++++++++++ .../src/opentelemetry/util/genai/handler.py | 34 ++ .../opentelemetry/util/genai/invocation.py | 5 + .../tests/test_handler_fetch_response.py | 312 ++++++++++++++++++ 31 files changed, 3110 insertions(+), 66 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_fetch_conformance.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_incomplete[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_raw_response[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_streaming[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_incomplete[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_raw_response[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_streaming[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_fetch.py create mode 100644 util/opentelemetry-util-genai/.changelog/184.added create mode 100644 util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py create mode 100644 util/opentelemetry-util-genai/tests/test_handler_fetch_response.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added index 7ad29db4c..f6250801a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/184.added @@ -1 +1 @@ -Add instrumentation for the OpenAI Responses ``retrieve`` API (sync and async). +Add instrumentation for the OpenAI Responses ``retrieve`` API (sync and async), reported as a ``fetch_response`` operation with no token usage. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst b/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst index adc9b01e9..820c9e309 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/README.rst @@ -62,9 +62,15 @@ Check out the `manual example `_ for more details. Instrumenting all clients ************************* -When using the instrumentor, all clients will automatically trace OpenAI operations including chat completions and embeddings. +When using the instrumentor, all clients will automatically trace OpenAI operations including chat completions, the Responses API, and embeddings. You can also optionally capture prompts and completions as log events. +Fetching a stored response with ``client.responses.retrieve(...)`` performs no +inference, so it is reported as a ``fetch_response`` operation rather than as an +inference call. Its span carries the fetched response's identifier, model, +status, and finish reasons, but no token usage — those counts belong to the +operation that originally generated the response. + Make sure to configure OpenTelemetry tracing, logging, and events to capture all telemetry emitted by the instrumentation. .. code-block:: python @@ -82,6 +88,14 @@ Make sure to configure OpenTelemetry tracing, logging, and events to capture all ], ) + # Responses API example, fetching a stored response back by its id + created = client.responses.create( + model="gpt-4o-mini", + input="Write a short poem on open telemetry.", + store=True, + ) + fetched = client.responses.retrieve(created.id) + # Embeddings example embedding_response = client.embeddings.create( model="text-embedding-3-small", diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py index c2ede3ce2..8d9eb62a3 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py @@ -79,8 +79,10 @@ ) from .patch_responses import ( async_responses_create, + async_responses_retrieve, async_responses_stream, responses_create, + responses_retrieve, responses_stream, ) @@ -188,20 +190,20 @@ def _instrument(self, **kwargs): async_responses_stream(handler), ) - # retrieve() fetches a stored response by id without going through - # create(), so wrap it separately. The result is a Response, so the - # create wrappers extract its telemetry correctly. + # retrieve() fetches a stored response by id. No inference happens + # and no tokens are consumed, so it is traced as a fetch_response + # operation rather than through the create wrappers. if hasattr(responses_module.Responses, "retrieve"): wrap_function_wrapper( "openai.resources.responses.responses", "Responses.retrieve", - responses_create(handler), + responses_retrieve(handler), ) if hasattr(responses_module.AsyncResponses, "retrieve"): wrap_function_wrapper( "openai.resources.responses.responses", "AsyncResponses.retrieve", - async_responses_create(handler), + async_responses_retrieve(handler), ) def _uninstrument(self, **kwargs): diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py index ea5bf0267..fb6b1b6d6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py @@ -3,26 +3,36 @@ from __future__ import annotations +import inspect +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Awaitable, Callable, Union, cast +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import FetchResponseInvocation -from ._raw_response import wrap_stream_result +from ._raw_response import ParsableResponse, wrap_stream_result from .response_extractors import ( apply_request_attributes, extract_params, + get_fetch_response_creation_kwargs, get_inference_creation_kwargs, get_response_error, + set_fetch_response_attributes, set_invocation_response_attributes, ) from .response_wrappers import ( + AsyncFetchResponseStreamWrapper, AsyncResponseStreamManagerWrapper, AsyncResponseStreamWrapper, + FetchResponseStreamWrapper, ResponseStreamManagerWrapper, ResponseStreamWrapper, responses_stream_context, ) -from .utils import is_streaming +from .utils import is_streaming, value_is_set if TYPE_CHECKING: from openai import AsyncStream as OpenAIAsyncStream @@ -45,6 +55,39 @@ "OpenAIAsyncStream[Any]", "AsyncResponseStream[Any]" ] +# Set by the SDK's ``with_streaming_response`` helpers to mark a request whose +# body the caller reads lazily. +_RAW_RESPONSE_HEADER = "x-stainless-raw-response" + + +def _is_streamed_raw_response(kwargs: dict[str, Any]) -> bool: + extra_headers = kwargs.get("extra_headers") + if not isinstance(extra_headers, Mapping): + return False + return any( + key.lower() == _RAW_RESPONSE_HEADER and value == "stream" + for key, value in extra_headers.items() + ) + + +def _parse_raw_response(result: object, kwargs: dict[str, Any]) -> object: + """Return the payload of a non-streaming ``with_raw_response`` result. + + A non-streaming raw response has already read its body and the SDK memoizes + ``parse()``, so deserializing it here does not change what the caller sees. + On the async client ``parse()`` returns an awaitable, which the caller + awaits. + + ``with_streaming_response`` hands back a raw response whose body has *not* + been read yet; parsing it would consume the body before the caller asks for + it, so those calls are left alone. + """ + if _is_streamed_raw_response(kwargs) or not isinstance( + result, ParsableResponse + ): + return result + return result.parse() + def responses_create( handler: TelemetryHandler, @@ -191,6 +234,176 @@ async def traced_method( ) +def _get_retrieve_response_id( + args: tuple[Any, ...], kwargs: dict[str, Any] +) -> str | None: + """Return the ``response_id`` a ``responses.retrieve`` call was made with.""" + response_id = args[0] if args else kwargs.get("response_id") + return response_id if isinstance(response_id, str) else None + + +def _get_stream_cursor(kwargs: dict[str, Any]) -> str | None: + """Return the ``starting_after`` cursor a streamed fetch resumes from.""" + starting_after = kwargs.get("starting_after") + if not value_is_set(starting_after): + return None + return str(starting_after) + + +def _start_fetch_response_invocation( + handler: TelemetryHandler, + instance: "Responses | AsyncResponses", + response_id: str, + kwargs: dict[str, Any], +) -> FetchResponseInvocation: + invocation = handler.fetch_response( + **get_fetch_response_creation_kwargs(response_id, instance) + ) + invocation.stream_cursor = _get_stream_cursor(kwargs) + invocation.attributes[OpenAIAttributes.OPENAI_API_TYPE] = ( + OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) + return invocation + + +def responses_retrieve( + handler: TelemetryHandler, +) -> Callable[ + ..., + Union[ + ResponseResult, + ResponseStreamResult, + FetchResponseStreamWrapper[Any], + ], +]: + """Wrap ``Responses.retrieve`` to trace fetching a stored response by id. + + Traces :meth:`openai.resources.responses.responses.Responses.retrieve`. + Fetching a stored response performs no inference and consumes no tokens, so + it is reported as a ``fetch_response`` operation rather than as an + inference call. + """ + + capture_content = handler.should_capture_content() + + def traced_method( + wrapped: Callable[..., Union[ResponseResult, ResponseStreamResult]], + instance: "Responses", + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Union[ + ResponseResult, + ResponseStreamResult, + FetchResponseStreamWrapper[Any], + ]: + response_id = _get_retrieve_response_id(args, kwargs) + if response_id is None: + # gen_ai.response.id is required on a fetch_response span and the + # SDK rejects the call without it, so leave it untraced. + return wrapped(*args, **kwargs) + + invocation = _start_fetch_response_invocation( + handler, instance, response_id, kwargs + ) + + try: + result = wrapped(*args, **kwargs) + if is_streaming(kwargs): + return wrap_stream_result( + FetchResponseStreamWrapper, + result, + invocation, + capture_content, + ) + + set_fetch_response_attributes( + invocation, + _parse_raw_response(result, kwargs), + capture_content, + ) + invocation.stop() + return result + except Exception as error: + invocation.fail(error) + raise + + return cast( + 'Callable[..., Union["ResponseResult", "ResponseStreamResult", FetchResponseStreamWrapper[Any]]]', + traced_method, + ) + + +def async_responses_retrieve( + handler: TelemetryHandler, +) -> Callable[ + ..., + Awaitable[ + Union[ + ResponseResult, + AsyncResponseStreamResult, + AsyncFetchResponseStreamWrapper[Any], + ] + ], +]: + """Wrap ``AsyncResponses.retrieve`` to trace fetching a stored response by id. + + Traces :meth:`openai.resources.responses.responses.AsyncResponses.retrieve`. + Fetching a stored response performs no inference and consumes no tokens, so + it is reported as a ``fetch_response`` operation rather than as an + inference call. + """ + + capture_content = handler.should_capture_content() + + async def traced_method( + wrapped: Callable[ + ..., + Awaitable[Union[ResponseResult, AsyncResponseStreamResult]], + ], + instance: "AsyncResponses", + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Union[ + ResponseResult, + AsyncResponseStreamResult, + AsyncFetchResponseStreamWrapper[Any], + ]: + response_id = _get_retrieve_response_id(args, kwargs) + if response_id is None: + # gen_ai.response.id is required on a fetch_response span and the + # SDK rejects the call without it, so leave it untraced. + return await wrapped(*args, **kwargs) + + invocation = _start_fetch_response_invocation( + handler, instance, response_id, kwargs + ) + + try: + result = await wrapped(*args, **kwargs) + if is_streaming(kwargs): + return wrap_stream_result( + AsyncFetchResponseStreamWrapper, + result, + invocation, + capture_content, + ) + + parsed = _parse_raw_response(result, kwargs) + if inspect.isawaitable(parsed): + parsed = await parsed + set_fetch_response_attributes(invocation, parsed, capture_content) + invocation.stop() + return result + except Exception as error: + invocation.fail(error) + raise + + return cast( + 'Callable[..., Awaitable[Union["ResponseResult", "AsyncResponseStreamResult", AsyncFetchResponseStreamWrapper[Any]]]]', + traced_method, + ) + + def responses_stream( handler: TelemetryHandler, ) -> Callable[..., ResponseStreamManagerWrapper[Any]]: diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index ffb82dbac..32438dd2e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -265,6 +265,44 @@ def _finish_reason_from_status(status: str | None) -> str | None: return None +# `incomplete_details.reason` values that map onto a cross-provider finish +# reason; an unrecognized reason is reported as-is. +_INCOMPLETE_REASON_TO_FINISH_REASON = { + "max_output_tokens": "length", + "content_filter": "content_filter", +} + + +def get_fetched_finish_reasons(response: "Response | None") -> list[str]: + """Return the finish reasons of the *original* generation of a fetched response. + + Derived from the fetched response ``status`` as the OpenAI ``fetch_response`` + semantic conventions require: a ``completed`` response maps to its stop + reason, an ``incomplete`` one to why it was cut short, and a ``failed`` or + ``cancelled`` one to ``error``. + + Statuses that are not terminal (``queued``, ``in_progress``) have no finish + reason yet, and so do a missing or unrecognized status: the generation is + not known to have stopped, so reporting one would be wrong. The lifecycle + state is conveyed by ``gen_ai.response.status`` instead. + """ + if Response is None or not isinstance(response, Response): + return [] + + status = response.status + if status == "completed": + return extract_finish_reasons(response) or ["stop"] + if status in {"failed", "cancelled"}: + return ["error"] + if status == "incomplete": + details = response.incomplete_details + reason = details.reason if details is not None else None + if reason is None: + return [status] + return [_INCOMPLETE_REASON_TO_FINISH_REASON.get(reason, reason)] + return [] + + def _response_types_available() -> bool: return ( Response is not None @@ -402,6 +440,24 @@ def get_inference_creation_kwargs( return creation_kwargs +def get_fetch_response_creation_kwargs( + response_id: str, + client_instance: object, +) -> dict[str, object]: + """Return ``handler.fetch_response()`` kwargs for a ``responses.retrieve`` call.""" + address, port = get_server_address_and_port(client_instance) + + creation_kwargs: dict[str, object] = { + "provider": GenAIAttributes.GenAiProviderNameValues.OPENAI.value, + "response_id": response_id, + } + if address is not None: + creation_kwargs["server_address"] = address + if port is not None: + creation_kwargs["server_port"] = port + return creation_kwargs + + def apply_request_attributes( invocation, params: ResponseRequestParams, @@ -490,3 +546,41 @@ def set_invocation_response_attributes( output_messages = get_output_messages_from_response(response) if output_messages: invocation.output_messages = output_messages + + +def set_fetch_response_attributes( + invocation, + response: object, + capture_content: bool, +) -> None: + """Record a fetched response on a ``fetch_response`` invocation. + + Token usage is deliberately not recorded: the fetch performs no inference + and the counts on the fetched response belong to the original generation. + The original input messages are not part of the fetched response either, so + only the system instructions and output messages it carries are captured. + """ + if Response is None or not isinstance(response, Response): + return + + invocation.response_model_name = response.model + invocation.response_status = response.status + invocation.finish_reasons = get_fetched_finish_reasons(response) or None + + # `service_tier` is absent from the Response model on some supported SDK + # versions, so keep this attribute access guarded. + service_tier = getattr(response, "service_tier", None) + if service_tier is not None: + invocation.attributes[ + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER + ] = service_tier + + if capture_content: + invocation.system_instruction = get_system_instruction( + response.instructions + if isinstance(response.instructions, str) + else None + ) + invocation.output_messages = get_output_messages_from_response( + response + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py index 2d5a1219a..8f9c57834 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py @@ -19,10 +19,12 @@ try: from opentelemetry.instrumentation.genai.openai.response_extractors import ( # pylint: disable=no-name-in-module get_response_error, + set_fetch_response_attributes, set_invocation_response_attributes, ) except ImportError: get_response_error = None + set_fetch_response_attributes = None set_invocation_response_attributes = None if TYPE_CHECKING: @@ -80,6 +82,16 @@ def _set_response_attributes( set_invocation_response_attributes(invocation, result, capture_content) +def _set_fetch_response_attributes( + invocation: "GenAIInvocation", + result: "ParsedResponse[TextFormatT] | Response | None", + capture_content: bool, +) -> None: + if set_fetch_response_attributes is None: + return + set_fetch_response_attributes(invocation, result, capture_content) + + def _get_stream_response(stream): try: return stream._response @@ -139,11 +151,25 @@ def _stop( ) -> None: if self._self_response_telemetry_finalized: return + self._apply_response_attributes(result) + self._self_invocation.stop() + self._self_response_telemetry_finalized = True + + def _apply_response_attributes( + self, result: "ParsedResponse[TextFormatT] | Response | None" + ) -> None: + """Record the response on the invocation. Overridden per operation.""" _set_response_attributes( self._self_invocation, result, self._self_capture_content ) - self._self_invocation.stop() - self._self_response_telemetry_finalized = True + + def _on_response_failed( + self, response: "ParsedResponse[TextFormatT] | Response | None" + ) -> None: + """Handle a ``response.failed`` event. Overridden per operation.""" + self._apply_response_attributes(response) + error = get_response_error(response) if get_response_error else None + self._fail(error or Error(type="response.failed", message=None)) def _fail(self, error: Error) -> None: if self._self_response_telemetry_finalized: @@ -205,15 +231,7 @@ def process_event(self, event: "ResponseStreamEvent[TextFormatT]") -> None: return if event_type == "response.failed": - _set_response_attributes( - self._self_invocation, - response, - self._self_capture_content, - ) - error = ( - get_response_error(response) if get_response_error else None - ) - self._fail(error or Error(type=event_type, message=None)) + self._on_response_failed(response) return if event.type == "error": @@ -256,6 +274,39 @@ def stream(self, stream: "ResponseStream[TextFormatT]") -> None: self._self_iterator = iter(stream) +class _FetchResponseStreamMixin(Generic[TextFormatT]): + """Finalization overrides for a streamed ``responses.retrieve``. + + The stream replays a response generated by an earlier operation, so the + fetch-response attributes are recorded instead of the inference ones, and a + replayed ``response.failed`` describes that original generation rather than + a failure of this fetch. + """ + + _self_invocation: "GenAIInvocation" + _self_capture_content: bool + + def _apply_response_attributes( + self, result: "ParsedResponse[TextFormatT] | Response | None" + ) -> None: + _set_fetch_response_attributes( + self._self_invocation, result, self._self_capture_content + ) + + def _on_response_failed( + self, response: "ParsedResponse[TextFormatT] | Response | None" + ) -> None: + self._stop(response) + + +class FetchResponseStreamWrapper( + _FetchResponseStreamMixin[TextFormatT], + ResponseStreamWrapper[TextFormatT], + Generic[TextFormatT], +): + """Wrapper for a streamed ``Responses.retrieve`` replay.""" + + class ResponseStreamManagerWrapper(Generic[TextFormatT]): """Wrapper for OpenAI Responses API stream managers. @@ -395,6 +446,14 @@ def response(self): return _AsyncResponseProxy(response, lambda: self._stop(None)) +class AsyncFetchResponseStreamWrapper( + _FetchResponseStreamMixin[TextFormatT], + AsyncResponseStreamWrapper[TextFormatT], + Generic[TextFormatT], +): + """Wrapper for a streamed ``AsyncResponses.retrieve`` replay.""" + + class AsyncResponseStreamManagerWrapper(Generic[TextFormatT]): """Wrapper for async OpenAI Responses API stream managers.""" diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_fetch_conformance.yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_fetch_conformance.yaml new file mode 100644 index 000000000..9ef9d6b95 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_fetch_conformance.yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml new file mode 100644 index 000000000..90f729695 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml @@ -0,0 +1,136 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027", + "object": "response", + "created_at": 1776481253, + "status": "failed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": { + "code": "server_error", + "message": "The model failed to generate a response." + }, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_incomplete[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_incomplete[content_mode0].yaml new file mode 100644 index 000000000..f74dd4dbd --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_incomplete[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "object": "response", + "created_at": 1776481253, + "status": "incomplete", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": { + "reason": "max_output_tokens" + }, + "instructions": "You are a helpful assistant.", + "max_output_tokens": 16, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "type": "message", + "status": "incomplete", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_raw_response[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_raw_response[content_mode0].yaml new file mode 100644 index 000000000..5d6286ca8 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_raw_response[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_streaming[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_streaming[content_mode0].yaml new file mode 100644 index 000000000..02c7d63e3 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_streaming[content_mode0].yaml @@ -0,0 +1,89 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028?starting_after=3&stream=true + response: + body: + string: |+ + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":"This is","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":4} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" a test.","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":5} + + event: response.output_text.done + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":6,"text":"This is a test."} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","object":"response","created_at":1776481253,"status":"completed","background":false,"billing":{"payer":"developer"},"completed_at":1776481257,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"This is a test."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":28},"user":null,"metadata":{}},"sequence_number":7} + + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml new file mode 100644 index 000000000..5d6286ca8 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'true' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml new file mode 100644 index 000000000..9c6e04e13 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_failed_generation_is_not_a_fetch_error[content_mode0].yaml @@ -0,0 +1,136 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027", + "object": "response", + "created_at": 1776481253, + "status": "failed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": { + "code": "server_error", + "message": "The model failed to generate a response." + }, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_incomplete[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_incomplete[content_mode0].yaml new file mode 100644 index 000000000..8f74594df --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_incomplete[content_mode0].yaml @@ -0,0 +1,150 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "object": "response", + "created_at": 1776481253, + "status": "incomplete", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": { + "reason": "max_output_tokens" + }, + "instructions": "You are a helpful assistant.", + "max_output_tokens": 16, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026", + "type": "message", + "status": "incomplete", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_raw_response[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_raw_response[content_mode0].yaml new file mode 100644 index 000000000..9ef9d6b95 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_raw_response[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_streaming[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_streaming[content_mode0].yaml new file mode 100644 index 000000000..68c76bd43 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_streaming[content_mode0].yaml @@ -0,0 +1,89 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - text/event-stream + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028?starting_after=3&stream=true + response: + body: + string: |+ + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":"This is","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":4} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" a test.","item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":5} + + event: response.output_text.done + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","logprobs":[],"output_index":0,"sequence_number":6,"text":"This is a test."} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","object":"response","created_at":1776481253,"status":"completed","background":false,"billing":{"payer":"developer"},"completed_at":1776481257,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","output":[{"id":"msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"This is a test."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":22,"input_tokens_details":{"cached_tokens":0},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":28},"user":null,"metadata":{}},"sequence_number":7} + + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml new file mode 100644 index 000000000..9ef9d6b95 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_retrieve_with_streaming_response_stays_lazy[content_mode0].yaml @@ -0,0 +1,148 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.109.1 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.12.12 + authorization: + - '******' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: GET + uri: https://api.openai.com/v1/responses/resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025 + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1776481257, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "This is a test." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + CF-RAY: + - 9ee06be8ab20d481-EWR + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 18 Apr 2026 03:00:58 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: test_openai_org_id + openai-processing-ms: + - '117' + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + x-request-id: + - req_d6009916bd4447188ce5baaea187709b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_fetch.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_fetch.py new file mode 100644 index 000000000..9c588219d --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_fetch.py @@ -0,0 +1,48 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: OpenAI Responses API fetch by id.""" + +from __future__ import annotations + +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +RESPONSE_ID = "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" + + +class ResponsesFetchScenario(Scenario): + """Fetching a stored response emits a ``fetch_response`` span. + + The operation performs no inference, so no token usage is recorded and + ``gen_ai.client.token.usage`` is intentionally absent. + """ + + expected_spans = {"fetch_response": 1} + expected_metrics = ("gen_ai.client.operation.duration",) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("responses_fetch_conformance.yaml"): + OpenAI().responses.retrieve(RESPONSE_ID) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 352e96ed9..1d30d1cec 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -28,15 +28,18 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) +from opentelemetry.trace.status import StatusCode from opentelemetry.util.genai.utils import is_experimental_mode from .test_responses import assert_responses_streaming_timing_metrics from .test_utils import ( DEFAULT_MODEL, + GEN_AI_RESPONSE_STATUS, USER_ONLY_EXPECTED_INPUT_MESSAGES, USER_ONLY_PROMPT, assert_all_attributes, assert_cache_attributes, + assert_fetch_response_attributes, assert_messages_attribute, format_simple_expected_output_message, get_responses_weather_tool_definition, @@ -200,9 +203,19 @@ async def test_async_responses_create_basic( RETRIEVE_RESPONSE_ID = ( "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" ) +RETRIEVE_INCOMPLETE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026" +) +RETRIEVE_FAILED_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027" +) +RETRIEVE_STREAM_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028" +) RETRIEVE_MISSING_RESPONSE_ID = ( "resp_doesnotexist0000000000000000000000000000000000" ) +RETRIEVE_STREAM_CURSOR = 3 @pytest.mark.asyncio() @@ -219,32 +232,25 @@ async def test_async_responses_retrieve_basic( ) (span,) = span_exporter.get_finished_spans() - # retrieve carries no request model/input; telemetry comes from the - # fetched response only. - assert GenAIAttributes.GEN_AI_REQUEST_MODEL not in span.attributes - assert ( - span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] - == response.model - ) - assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response.id - assert ( - span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] - == response.usage.input_tokens - ) - assert ( - span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] - == response.usage.output_tokens - ) - assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( - "stop", + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, ) - assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS not in span.attributes @pytest.mark.asyncio() async def test_async_responses_retrieve_captures_content( - span_exporter, async_openai_client, instrument_with_content, vcr + span_exporter, + log_exporter, + async_openai_client, + instrument_with_content, + vcr, ): _skip_if_not_latest() @@ -260,8 +266,173 @@ async def test_async_responses_retrieve_captures_content( span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], format_simple_expected_output_message(response.output_text), ) - # retrieve has no request-side input messages to capture. + assert ( + json.loads(span.attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS]) + == EXPECTED_SYSTEM_INSTRUCTIONS + ) + # A fetched response does not carry the original input messages. assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + assert len(log_exporter.get_finished_logs()) == 0 + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_incomplete( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """An incomplete stored response surfaces via status and finish reasons.""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_incomplete[content_mode0].yaml" + ): + response = await async_openai_client.responses.retrieve( + RETRIEVE_INCOMPLETE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="incomplete", + finish_reasons=("length",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_failed_generation_is_not_a_fetch_error( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """A stored response whose generation failed is not a failure of the fetch.""" + _skip_if_not_latest() + + cassette = ( + "test_async_responses_retrieve_failed_generation_is_not_a_fetch_error" + "[content_mode0].yaml" + ) + with vcr.use_cassette(cassette): + response = await async_openai_client.responses.retrieve( + RETRIEVE_FAILED_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="failed", + finish_reasons=("error",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_streaming( + span_exporter, async_openai_client, instrument_with_content, vcr +): + """A streamed replay finalizes only once the caller drains the stream.""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_streaming[content_mode0].yaml" + ): + stream = await async_openai_client.responses.retrieve( + RETRIEVE_STREAM_RESPONSE_ID, + stream=True, + starting_after=RETRIEVE_STREAM_CURSOR, + ) + assert isinstance(stream, AsyncStream) + assert span_exporter.get_finished_spans() == () + + response = await _collect_completed_response(stream) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=RETRIEVE_STREAM_RESPONSE_ID, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + stream_cursor=str(RETRIEVE_STREAM_CURSOR), + response_service_tier=response.service_tier, + ) + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), + ) + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_raw_response( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """``with_raw_response`` keeps returning the raw response, still traced.""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_retrieve_raw_response[content_mode0].yaml" + ): + raw_response = ( + await async_openai_client.responses.with_raw_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + ) + response = raw_response.parse() + if inspect.isawaitable(response): + response = await response + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, + ) + + +@pytest.mark.asyncio() +async def test_async_responses_retrieve_with_streaming_response_stays_lazy( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """``with_streaming_response`` must not have its body read by telemetry.""" + _skip_if_not_latest() + + cassette = ( + "test_async_responses_retrieve_with_streaming_response_stays_lazy" + "[content_mode0].yaml" + ) + with vcr.use_cassette(cassette): + async with ( + async_openai_client.responses.with_streaming_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + ) as raw_response: + # Building telemetry must not consume or close the body before the + # caller reads it. + assert not raw_response.http_response.is_stream_consumed + assert not raw_response.http_response.is_closed + response = raw_response.parse() + if inspect.isawaitable(response): + response = await response + + (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_RESPONSE_ID + ) + assert response.id == RETRIEVE_RESPONSE_ID @pytest.mark.asyncio() @@ -279,10 +450,22 @@ async def test_async_responses_retrieve_api_error( ) (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_MISSING_RESPONSE_ID + ) + assert span.status.status_code is StatusCode.ERROR assert ( span.attributes[ErrorAttributes.ERROR_TYPE] == f"openai.{type(exc_info.value).__name__}" ) + # The fetch failed before any response existed to describe. + assert GEN_AI_RESPONSE_STATUS not in span.attributes @pytest.mark.asyncio() diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py index a34cbc8f5..e35d73609 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py @@ -27,6 +27,7 @@ from .conformance.inference import InferenceScenario from .conformance.inference_streaming import InferenceStreamingScenario from .conformance.responses_conversation import ResponsesConversationScenario +from .conformance.responses_fetch import ResponsesFetchScenario from .conformance.responses_stream import ResponsesStreamScenario from .conformance.responses_streaming import ResponsesStreamingScenario from .conformance.tool_calling import ToolCallingScenario @@ -40,6 +41,7 @@ EmbeddingScenario(), ToolCallingScenario(), ResponsesConversationScenario(), + ResponsesFetchScenario(), ResponsesStreamScenario(), ResponsesStreamingScenario(), ], diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index a5edf21e3..20573d312 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -395,3 +395,73 @@ def test_response_extractors_ignore_invalid_shapes_without_validation( assert invocation.finish_reasons is None assert not invocation.output_messages assert not invocation.attributes + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + ({"status": "completed"}, ["stop"]), + ({"status": "failed"}, ["error"]), + ({"status": "cancelled"}, ["error"]), + ( + { + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + ["length"], + ), + ( + { + "status": "incomplete", + "incomplete_details": {"reason": "content_filter"}, + }, + ["content_filter"], + ), + ({"status": "incomplete"}, ["incomplete"]), + # Non-terminal and unknown statuses: generation is not known to have + # stopped, so there is no finish reason to report. gen_ai.response.status + # conveys the lifecycle state instead. + ({"status": "queued"}, []), + ({"status": "in_progress"}, []), + ({}, []), + ], +) +def test_get_fetched_finish_reasons_maps_status( + loaded_module, overrides, expected +): + response = _make_response(**overrides) + + assert loaded_module.get_fetched_finish_reasons(response) == expected + + +def test_set_fetch_response_attributes_tolerates_missing_service_tier(): + """`service_tier` is absent from the Response model on older SDKs. + + Deleting the field from the instance makes attribute access raise + ``AttributeError``, exactly as it does on an SDK whose ``Response`` model + never declared it (for example openai 1.70). + """ + response = _make_response(status="completed") + del response.__dict__["service_tier"] + with pytest.raises(AttributeError): + response.service_tier # pylint: disable=pointless-statement + + invocation = SimpleNamespace( + response_model_name=None, + response_status=None, + finish_reasons=None, + output_messages=[], + system_instruction=[], + attributes={}, + ) + + response_extractors.set_fetch_response_attributes( + invocation, response, capture_content=False + ) + + assert invocation.response_status == "completed" + assert invocation.finish_reasons == ["stop"] + assert ( + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER + not in invocation.attributes + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py index 9947de4b3..a61df5ceb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_wrappers.py @@ -8,6 +8,7 @@ from opentelemetry.instrumentation.genai.openai.response_wrappers import ( AsyncResponseStreamManagerWrapper, AsyncResponseStreamWrapper, + FetchResponseStreamWrapper, ResponseStreamManagerWrapper, ResponseStreamWrapper, ) @@ -621,3 +622,91 @@ def test_process_event_error_event_records_error(): (error,) = calls["fail"] assert error.type == "rate_limit_exceeded" assert error.message == "slow down" + + +def _make_fetch_stream_wrapper(stream, invocation): + return FetchResponseStreamWrapper( + stream=stream, + invocation=invocation, + capture_content=False, + ) + + +def _capturing_fetch_invocation(): + calls = {"stop": 0, "fail": []} + invocation = SimpleNamespace( + response_model_name=None, + response_status=None, + finish_reasons=None, + stream_cursor=None, + output_messages=[], + system_instruction=[], + tool_definitions=None, + attributes={}, + ) + invocation.stop = lambda: calls.__setitem__("stop", calls["stop"] + 1) + invocation.fail = lambda error: calls["fail"].append(error) + return invocation, calls + + +@_requires_responses_types +def test_fetch_stream_completed_records_fetch_response_attributes(): + invocation, calls = _capturing_fetch_invocation() + wrapper = _make_fetch_stream_wrapper( + _FakeSyncStream(), invocation=invocation + ) + event = SimpleNamespace( + type="response.completed", + response=_make_response(status="completed"), + ) + + wrapper.process_event(event) + + assert calls["stop"] == 1 + assert calls["fail"] == [] + assert invocation.response_status == "completed" + assert invocation.finish_reasons == ["stop"] + + +@_requires_responses_types +def test_fetch_stream_replayed_failure_is_not_an_error_of_the_fetch(): + """A replayed ``response.failed`` describes the original generation.""" + invocation, calls = _capturing_fetch_invocation() + wrapper = _make_fetch_stream_wrapper( + _FakeSyncStream(), invocation=invocation + ) + event = SimpleNamespace( + type="response.failed", + response=_make_response( + status="failed", + error={"code": "server_error", "message": "boom"}, + ), + ) + + wrapper.process_event(event) + + assert calls["fail"] == [] + assert calls["stop"] == 1 + assert invocation.response_status == "failed" + assert invocation.finish_reasons == ["error"] + + +@_requires_responses_types +def test_fetch_stream_transport_error_event_still_fails_the_fetch(): + """An SSE ``error`` event is a failure of the fetch itself.""" + invocation, calls = _capturing_fetch_invocation() + wrapper = _make_fetch_stream_wrapper( + _FakeSyncStream(), invocation=invocation + ) + event = SimpleNamespace( + type="error", + code="rate_limit_exceeded", + message="slow down", + response=None, + ) + + wrapper.process_event(event) + + assert calls["stop"] == 0 + (error,) = calls["fail"] + assert error.type == "rate_limit_exceeded" diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 984b73dba..4850b0066 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -28,14 +28,17 @@ server_attributes as ServerAttributes, ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.trace.status import StatusCode from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( DEFAULT_MODEL, + GEN_AI_RESPONSE_STATUS, USER_ONLY_EXPECTED_INPUT_MESSAGES, USER_ONLY_PROMPT, assert_all_attributes, assert_cache_attributes, + assert_fetch_response_attributes, assert_messages_attribute, format_simple_expected_output_message, get_responses_weather_tool_definition, @@ -148,6 +151,15 @@ def _collect_completed_response(stream): return response +def _collect_metrics(metric_reader): + metrics = {} + for rm in metric_reader.get_metrics_data().resource_metrics: + for scope in rm.scope_metrics: + for metric in scope.metrics: + metrics[metric.name] = metric + return metrics + + def assert_responses_streaming_timing_metrics(metric_reader): """Assert the streaming timing metrics are emitted through the real Responses stream wrapper path. @@ -157,11 +169,7 @@ def assert_responses_streaming_timing_metrics(metric_reader): green but silently stop emitting TTFC and per-output-chunk metrics for the Responses streaming path. """ - metrics = {} - for rm in metric_reader.get_metrics_data().resource_metrics: - for scope in rm.scope_metrics: - for metric in scope.metrics: - metrics[metric.name] = metric + metrics = _collect_metrics(metric_reader) ttfc = metrics.get( gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK @@ -259,9 +267,19 @@ def test_responses_create_basic( RETRIEVE_RESPONSE_ID = ( "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237025" ) +RETRIEVE_INCOMPLETE_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237026" +) +RETRIEVE_FAILED_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237027" +) +RETRIEVE_STREAM_RESPONSE_ID = ( + "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237028" +) RETRIEVE_MISSING_RESPONSE_ID = ( "resp_doesnotexist0000000000000000000000000000000000" ) +RETRIEVE_STREAM_CURSOR = 3 @pytest.mark.vcr() @@ -273,44 +291,169 @@ def test_responses_retrieve_basic( response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() - # retrieve carries no request model/input; telemetry comes from the - # fetched response only. - assert GenAIAttributes.GEN_AI_REQUEST_MODEL not in span.attributes - assert ( - span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] - == response.model + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, ) - assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response.id - assert ( - span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] - == response.usage.input_tokens + assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + assert GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_captures_content( + span_exporter, log_exporter, openai_client, instrument_with_content +): + _skip_if_not_latest() + + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], + format_simple_expected_output_message(response.output_text), ) assert ( - span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] - == response.usage.output_tokens - ) - assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( - "stop", + json.loads(span.attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS]) + == EXPECTED_SYSTEM_INSTRUCTIONS ) + # A fetched response does not carry the original input messages. assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes - assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes + assert len(log_exporter.get_finished_logs()) == 0 @pytest.mark.vcr() -def test_responses_retrieve_captures_content( +def test_responses_retrieve_incomplete( + span_exporter, openai_client, instrument_no_content +): + """An incomplete stored response surfaces via status and finish reasons.""" + _skip_if_not_latest() + + response = openai_client.responses.retrieve( + RETRIEVE_INCOMPLETE_RESPONSE_ID + ) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="incomplete", + finish_reasons=("length",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_failed_generation_is_not_a_fetch_error( + span_exporter, openai_client, instrument_no_content +): + """A stored response whose generation failed is not a failure of the fetch.""" + _skip_if_not_latest() + + response = openai_client.responses.retrieve(RETRIEVE_FAILED_RESPONSE_ID) + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="failed", + finish_reasons=("error",), + response_service_tier=response.service_tier, + ) + assert span.status.status_code is StatusCode.UNSET + assert ErrorAttributes.ERROR_TYPE not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_streaming( span_exporter, openai_client, instrument_with_content ): + """A streamed replay finalizes only once the caller drains the stream.""" _skip_if_not_latest() - response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + stream = openai_client.responses.retrieve( + RETRIEVE_STREAM_RESPONSE_ID, + stream=True, + starting_after=RETRIEVE_STREAM_CURSOR, + ) + assert isinstance(stream, Stream) + assert span_exporter.get_finished_spans() == () + + response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=RETRIEVE_STREAM_RESPONSE_ID, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + stream_cursor=str(RETRIEVE_STREAM_CURSOR), + response_service_tier=response.service_tier, + ) assert_messages_attribute( span.attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES], format_simple_expected_output_message(response.output_text), ) - # retrieve has no request-side input messages to capture. - assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + + +@pytest.mark.vcr() +def test_responses_retrieve_raw_response( + span_exporter, openai_client, instrument_no_content +): + """``with_raw_response`` keeps returning the raw response, still traced.""" + _skip_if_not_latest() + + raw_response = openai_client.responses.with_raw_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + response = raw_response.parse() + + (span,) = span_exporter.get_finished_spans() + assert_fetch_response_attributes( + span, + response_id=response.id, + response_model=response.model, + response_status="completed", + finish_reasons=("stop",), + response_service_tier=response.service_tier, + ) + + +@pytest.mark.vcr() +def test_responses_retrieve_with_streaming_response_stays_lazy( + span_exporter, openai_client, instrument_no_content +): + """``with_streaming_response`` must not have its body read by telemetry.""" + _skip_if_not_latest() + + with openai_client.responses.with_streaming_response.retrieve( + RETRIEVE_RESPONSE_ID + ) as raw_response: + # Building telemetry must not consume or close the body before the + # caller reads it. + assert not raw_response.http_response.is_stream_consumed + assert not raw_response.http_response.is_closed + response = raw_response.parse() + + (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_RESPONSE_ID + ) + assert response.id == RETRIEVE_RESPONSE_ID @pytest.mark.vcr() @@ -323,10 +466,48 @@ def test_responses_retrieve_api_error( openai_client.responses.retrieve(RETRIEVE_MISSING_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() + assert span.name == "fetch_response" + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + == RETRIEVE_MISSING_RESPONSE_ID + ) + assert span.status.status_code is StatusCode.ERROR assert ( span.attributes[ErrorAttributes.ERROR_TYPE] == f"openai.{type(exc_info.value).__name__}" ) + # The fetch failed before any response existed to describe. + assert GEN_AI_RESPONSE_STATUS not in span.attributes + + +def test_responses_retrieve_does_not_record_token_usage_metric( + span_exporter, metric_reader, openai_client, instrument_no_content, vcr +): + """A fetch consumes no tokens, so only the duration metric is recorded.""" + _skip_if_not_latest() + + with vcr.use_cassette("test_responses_retrieve_basic[content_mode0].yaml"): + openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + + metrics = _collect_metrics(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE not in metrics + + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + (point,) = duration.data.data_points + assert ( + point.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" + ) + assert ( + point.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + == "gpt-4o-mini-2024-07-18" + ) + # The response id is high cardinality and must stay off metrics. + assert GenAIAttributes.GEN_AI_RESPONSE_ID not in point.attributes @pytest.mark.vcr() diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py index f8113bc9c..ec33562ed 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py @@ -16,9 +16,16 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) +from opentelemetry.trace import SpanKind DEFAULT_MODEL = "gpt-4o-mini" DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small" +FETCH_RESPONSE_OPERATION_NAME = "fetch_response" +# TODO: use the semconv constants once these attributes are released in +# opentelemetry-semantic-conventions. Added to the GenAI semantic conventions +# in https://github.com/open-telemetry/semantic-conventions-genai/pull/353. +GEN_AI_REQUEST_STREAM_CURSOR = "gen_ai.request.stream_cursor" +GEN_AI_RESPONSE_STATUS = "gen_ai.response.status" GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = ( "gen_ai.usage.cache_creation.input_tokens" ) @@ -146,6 +153,63 @@ def assert_all_attributes( ) +def assert_fetch_response_attributes( + span: ReadableSpan, + *, + response_id: str, + response_model: Optional[str] = None, + response_status: Optional[str] = None, + finish_reasons: Optional[tuple] = None, + stream_cursor: Optional[str] = None, + response_service_tier: Optional[str] = None, + server_address: str = "api.openai.com", +): + """Assert a ``gen_ai.fetch_response.client`` span matches the semconv. + + Fetching a stored response performs no inference, so the span must carry + neither request-side attributes nor any token usage from the fetched + response — those counts belong to the original generation. + """ + # The response id is high cardinality, so it stays out of the span name. + assert span.name == FETCH_RESPONSE_OPERATION_NAME + assert span.kind is SpanKind.CLIENT + assert ( + span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == FETCH_RESPONSE_OPERATION_NAME + ) + assert ( + span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] + == GenAIAttributes.GenAiProviderNameValues.OPENAI.value + ) + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == response_id + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) + assert span.attributes[ServerAttributes.SERVER_ADDRESS] == server_address + + assert GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS not in span.attributes + assert GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS not in span.attributes + assert GenAIAttributes.GEN_AI_REQUEST_MODEL not in span.attributes + assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes + + _assert_optional_attribute( + span, GenAIAttributes.GEN_AI_RESPONSE_MODEL, response_model + ) + _assert_optional_attribute(span, GEN_AI_RESPONSE_STATUS, response_status) + _assert_optional_attribute( + span, GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, finish_reasons + ) + _assert_optional_attribute( + span, GEN_AI_REQUEST_STREAM_CURSOR, stream_cursor + ) + _assert_optional_attribute( + span, + OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER, + response_service_tier, + ) + + def assert_log_parent(log, span): """Assert that the log record has the correct parent span context""" if span: diff --git a/policies/genai_span_validation.rego b/policies/genai_span_validation.rego index e53dfd538..fbd03defa 100644 --- a/policies/genai_span_validation.rego +++ b/policies/genai_span_validation.rego @@ -274,6 +274,7 @@ _known_operation_names := { "generate_content", "text_completion", "embeddings", + "fetch_response", "retrieval", "create_agent", "invoke_agent", diff --git a/util/opentelemetry-util-genai/.changelog/184.added b/util/opentelemetry-util-genai/.changelog/184.added new file mode 100644 index 000000000..69bd6cbd5 --- /dev/null +++ b/util/opentelemetry-util-genai/.changelog/184.added @@ -0,0 +1 @@ +Add ``TelemetryHandler.fetch_response()`` and ``FetchResponseInvocation`` for the ``fetch_response`` operation, which fetches a previously generated response by id without performing inference. diff --git a/util/opentelemetry-util-genai/AGENTS.md b/util/opentelemetry-util-genai/AGENTS.md index ada792cc0..b97de50cd 100644 --- a/util/opentelemetry-util-genai/AGENTS.md +++ b/util/opentelemetry-util-genai/AGENTS.md @@ -42,6 +42,7 @@ Factory methods on `TelemetryHandler` (`handler.py`): - `inference(provider, request_model, *, server_address, server_port)` → `InferenceInvocation` - `embedding(provider, request_model, *, server_address, server_port)` → `EmbeddingInvocation` - `retrieval(*, data_source_id, provider, request_model, server_address, server_port)` → `RetrievalInvocation` +- `fetch_response(provider, *, response_id, server_address, server_port)` → `FetchResponseInvocation` - `tool(name, *, arguments, tool_call_id, tool_type, tool_description)` → `ToolInvocation` - `workflow(name)` → `WorkflowInvocation` diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py new file mode 100644 index 000000000..53f9aaaef --- /dev/null +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py @@ -0,0 +1,181 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Final + +from opentelemetry._logs import Logger +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import server_attributes +from opentelemetry.trace import SpanKind, Tracer +from opentelemetry.util.genai._invocation import ( + Error, + GenAIInvocation, + get_content_attributes, +) +from opentelemetry.util.genai.completion_hook import CompletionHook +from opentelemetry.util.genai.metrics import InvocationMetricsRecorder +from opentelemetry.util.genai.types import ( + ErrorTypeResolver, + MessagePart, + OutputMessage, + ToolDefinition, +) +from opentelemetry.util.types import AttributeValue + +# TODO: Migrate to gen_ai_attributes constants once available in the semconv +# package. Added to the GenAI semantic conventions in +# https://github.com/open-telemetry/semantic-conventions-genai/pull/353. +_FETCH_RESPONSE_OPERATION_NAME: Final = "fetch_response" +_GEN_AI_REQUEST_STREAM_CURSOR: Final = "gen_ai.request.stream_cursor" +_GEN_AI_RESPONSE_STATUS: Final = "gen_ai.response.status" + + +class FetchResponseInvocation(GenAIInvocation): + """Represents a single fetch of a previously generated model response. + + Use handler.fetch_response() rather than constructing this directly. + + Reference: https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-spans.md#fetch-response + + The operation performs no inference and consumes no tokens: it returns a + response produced by an earlier operation. Any token counts carried on the + fetched response describe that original generation and MUST NOT be reported + here, so this invocation deliberately exposes no token usage fields. + + Semantic convention attributes for fetch response spans: + - gen_ai.operation.name: "fetch_response" (Required) + - gen_ai.provider.name: Provider name (Required) + - gen_ai.response.id: Identifier of the response being fetched (Required) + - error.type: Error type when the fetch itself failed (Conditionally Required) + - gen_ai.request.stream_cursor: Set from ``stream_cursor`` when the fetch + resumes a streamed response from a prior position (Conditionally Required) + - server.port: Set only when ``server_port`` is provided (Conditionally Required) + - gen_ai.response.finish_reasons: Outcome of the original generation + (Recommended) + - gen_ai.response.model: Set from ``response_model_name`` (Recommended) + - gen_ai.response.status: Lifecycle status of the fetched response + (Recommended) + - server.address: Set only when ``server_address`` is provided (Recommended) + - gen_ai.output.messages, gen_ai.system_instructions, + gen_ai.tool.definitions: content carried on the fetched response, + recorded on the span only when content capturing is enabled (Opt-In). + A fetched response does not carry the original input messages, so + gen_ai.input.messages is never set. + + A fetched response whose *original* generation failed is not a failure of + the fetch: report it through ``response_status`` and ``finish_reasons`` and + still call ``stop()``. Only call ``fail()`` when the fetch call itself + failed. + """ + + def __init__( + self, + tracer: Tracer, + metrics_recorder: InvocationMetricsRecorder, + logger: Logger, + completion_hook: CompletionHook, + provider: str, + *, + response_id: str, + server_address: str | None = None, + server_port: int | None = None, + error_type_resolver: ErrorTypeResolver | None = None, + ) -> None: + """Use handler.fetch_response() rather than calling this directly.""" + super().__init__( + tracer, + metrics_recorder, + logger, + completion_hook, + operation_name=_FETCH_RESPONSE_OPERATION_NAME, + # The response identifier is high cardinality, so semconv keeps it + # out of the span name. + span_name=_FETCH_RESPONSE_OPERATION_NAME, + span_kind=SpanKind.CLIENT, + error_type_resolver=error_type_resolver, + ) + self._provider: str = provider + self._response_id: str = response_id + self._server_address: str | None = server_address + self._server_port: int | None = server_port + self.response_model_name: str | None = None + self.response_status: str | None = None + self.finish_reasons: list[str] | None = None + self.stream_cursor: str | None = None + self.output_messages: list[OutputMessage] = [] + self.system_instruction: list[MessagePart] = [] + self.tool_definitions: list[ToolDefinition] | None = None + self._start(self._get_start_attributes()) + + @property + def response_id(self) -> str: + """The identifier of the response being fetched.""" + return self._response_id + + def _get_start_attributes(self) -> dict[str, AttributeValue]: + """Return attributes known at span creation time.""" + optional_attrs: tuple[tuple[str, AttributeValue | None], ...] = ( + (server_attributes.SERVER_ADDRESS, self._server_address), + (server_attributes.SERVER_PORT, self._server_port), + ) + return { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + GenAI.GEN_AI_PROVIDER_NAME: self._provider, + GenAI.GEN_AI_RESPONSE_ID: self._response_id, + **{k: v for k, v in optional_attrs if v is not None}, + } + + def _get_metric_attributes(self) -> dict[str, AttributeValue]: + # response_id intentionally excluded — high cardinality. + optional_attrs: tuple[tuple[str, AttributeValue | None], ...] = ( + (GenAI.GEN_AI_RESPONSE_MODEL, self.response_model_name), + (server_attributes.SERVER_ADDRESS, self._server_address), + (server_attributes.SERVER_PORT, self._server_port), + ) + attrs: dict[str, AttributeValue] = { + GenAI.GEN_AI_OPERATION_NAME: self._operation_name, + GenAI.GEN_AI_PROVIDER_NAME: self._provider, + **{k: v for k, v in optional_attrs if v is not None}, + } + attrs.update(self.metric_attributes) + return attrs + + def _get_attributes(self) -> dict[str, AttributeValue]: + optional_attrs: tuple[tuple[str, AttributeValue | None], ...] = ( + (_GEN_AI_REQUEST_STREAM_CURSOR, self.stream_cursor), + ( + GenAI.GEN_AI_RESPONSE_FINISH_REASONS, + self.finish_reasons or None, + ), + (GenAI.GEN_AI_RESPONSE_MODEL, self.response_model_name), + (_GEN_AI_RESPONSE_STATUS, self.response_status), + ) + return {k: v for k, v in optional_attrs if v is not None} + + def _apply_finish(self, error: Error | None = None) -> None: + if error is not None: + self._apply_error_attributes(error) + attributes = self._get_attributes() + attributes.update( + get_content_attributes( + # A fetched response does not carry the original request's + # input messages. + input_messages=(), + output_messages=self.output_messages, + system_instruction=self.system_instruction, + tool_definitions=self.tool_definitions, + for_span=True, + ) + ) + attributes.update(self.attributes) + self.span.set_attributes(attributes) + self._metrics_recorder.record(self) + self._call_completion_hook( + outputs=self.output_messages, + system_instruction=self.system_instruction, + tool_definitions=self.tool_definitions, + ) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py index a3134f3c3..2f72271a0 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py @@ -57,6 +57,7 @@ ) from opentelemetry.util.genai.invocation import ( EmbeddingInvocation, + FetchResponseInvocation, InferenceInvocation, RetrievalInvocation, ToolInvocation, @@ -363,6 +364,39 @@ def embedding( server_port=server_port, ) + def fetch_response( + self, + provider: str, + *, + response_id: str, + server_address: str | None = None, + server_port: int | None = None, + error_type_resolver: ErrorTypeResolver | None = None, + ) -> FetchResponseInvocation: + """Returns a Fetch Response invocation. Starts span when called. + + Describes fetching a previously generated model response by its + identifier. No inference is performed and no tokens are consumed, so + the fetched response's token counts must not be recorded here. + + Returned object can be used as a ContextManager which automatically calls `stop` or `fail` + to finalize the span upon exiting. If not used as a ContextManager, the caller is + responsible for calling `stop` or `fail` to finalize the span. + + Only set data attributes on the invocation object, do not modify the span or context. + """ + return FetchResponseInvocation( + self._tracer, + self._metrics_recorder, + self._logger, + self._completion_hook, + provider=provider, + response_id=response_id, + server_address=server_address, + server_port=server_port, + error_type_resolver=error_type_resolver, + ) + def tool( self, name: str, diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py index 2d779910e..a0491b098 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py @@ -8,6 +8,7 @@ from opentelemetry.util.genai.invocation import ( Error, GenAIInvocation, + FetchResponseInvocation, InferenceInvocation, EmbeddingInvocation, RetrievalInvocation, @@ -18,6 +19,9 @@ from opentelemetry.util.genai._agent_invocation import AgentInvocation from opentelemetry.util.genai._embedding_invocation import EmbeddingInvocation +from opentelemetry.util.genai._fetch_response_invocation import ( + FetchResponseInvocation, +) from opentelemetry.util.genai._inference_invocation import InferenceInvocation from opentelemetry.util.genai._invocation import ( ContextToken, @@ -32,6 +36,7 @@ "AgentInvocation", "ContextToken", "Error", + "FetchResponseInvocation", "GenAIInvocation", "InferenceInvocation", "EmbeddingInvocation", diff --git a/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py b/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py new file mode 100644 index 000000000..0e45cd9d1 --- /dev/null +++ b/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py @@ -0,0 +1,312 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +from unittest import TestCase +from unittest.mock import patch + +import pytest + +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import server_attributes +from opentelemetry.trace import INVALID_SPAN, SpanKind +from opentelemetry.trace.status import StatusCode +from opentelemetry.util.genai.environment_variables import ( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, +) +from opentelemetry.util.genai.handler import TelemetryHandler +from opentelemetry.util.genai.invocation import FetchResponseInvocation +from opentelemetry.util.genai.types import ( + Error, + FunctionToolDefinition, + OutputMessage, + Text, +) + +# TODO: use the semconv constants once these attributes are released in +# opentelemetry-semantic-conventions. +GEN_AI_REQUEST_STREAM_CURSOR = "gen_ai.request.stream_cursor" +GEN_AI_RESPONSE_STATUS = "gen_ai.response.status" + +RESPONSE_ID = "resp_123" + + +class _FetchResponseTestBase(TestCase): + def setUp(self) -> None: + self.span_exporter = InMemorySpanExporter() + self.tracer_provider = TracerProvider() + self.tracer_provider.add_span_processor( + SimpleSpanProcessor(self.span_exporter) + ) + self.metric_reader = InMemoryMetricReader() + self.meter_provider = MeterProvider( + metric_readers=[self.metric_reader] + ) + self.handler = TelemetryHandler( + tracer_provider=self.tracer_provider, + meter_provider=self.meter_provider, + ) + + def _fetch_response(self, **kwargs) -> FetchResponseInvocation: + return self.handler.fetch_response( + "openai", response_id=RESPONSE_ID, **kwargs + ) + + def _get_finished_spans(self): + return self.span_exporter.get_finished_spans() + + def _get_metrics(self): + metrics = {} + data = self.metric_reader.get_metrics_data() + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + metrics[metric.name] = metric + return metrics + + +class TelemetryHandlerFetchResponseTest(_FetchResponseTestBase): + # ------------------------------------------------------------------ + # span creation + # ------------------------------------------------------------------ + + def test_fetch_response_creates_span(self) -> None: + invocation = self._fetch_response() + self.assertIsNot(invocation.span, INVALID_SPAN) + invocation.stop() + + def test_span_name_omits_the_high_cardinality_response_id(self) -> None: + self._fetch_response().stop() + + spans = self._get_finished_spans() + self.assertEqual(len(spans), 1) + self.assertEqual(spans[0].name, "fetch_response") + + def test_span_kind_is_client(self) -> None: + self._fetch_response().stop() + + self.assertEqual(self._get_finished_spans()[0].kind, SpanKind.CLIENT) + + def test_records_monotonic_start(self) -> None: + with patch("timeit.default_timer", return_value=42.0): + invocation = self._fetch_response() + self.assertEqual(invocation._monotonic_start_s, 42.0) + invocation.stop() + + # ------------------------------------------------------------------ + # required and conditionally required attributes + # ------------------------------------------------------------------ + + def test_required_attributes_are_set_at_span_creation(self) -> None: + invocation = self._fetch_response() + + attrs = invocation.span.attributes + self.assertEqual(attrs[GenAI.GEN_AI_OPERATION_NAME], "fetch_response") + self.assertEqual(attrs[GenAI.GEN_AI_PROVIDER_NAME], "openai") + self.assertEqual(attrs[GenAI.GEN_AI_RESPONSE_ID], RESPONSE_ID) + invocation.stop() + + def test_response_id_is_exposed(self) -> None: + invocation = self._fetch_response() + self.assertEqual(invocation.response_id, RESPONSE_ID) + invocation.stop() + + def test_stop_sets_server_address_and_port(self) -> None: + self._fetch_response( + server_address="api.openai.com", server_port=8080 + ).stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual( + attrs[server_attributes.SERVER_ADDRESS], "api.openai.com" + ) + self.assertEqual(attrs[server_attributes.SERVER_PORT], 8080) + + def test_stop_sets_stream_cursor(self) -> None: + invocation = self._fetch_response() + invocation.stream_cursor = "42" + invocation.stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual(attrs[GEN_AI_REQUEST_STREAM_CURSOR], "42") + + # ------------------------------------------------------------------ + # recommended attributes + # ------------------------------------------------------------------ + + def test_stop_sets_response_model_status_and_finish_reasons(self) -> None: + invocation = self._fetch_response() + invocation.response_model_name = "gpt-4o-mini-2024-07-18" + invocation.response_status = "completed" + invocation.finish_reasons = ["stop"] + invocation.stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual( + attrs[GenAI.GEN_AI_RESPONSE_MODEL], "gpt-4o-mini-2024-07-18" + ) + self.assertEqual(attrs[GEN_AI_RESPONSE_STATUS], "completed") + self.assertEqual( + attrs[GenAI.GEN_AI_RESPONSE_FINISH_REASONS], ("stop",) + ) + + def test_stop_omits_unset_attributes(self) -> None: + self._fetch_response().stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertNotIn(GenAI.GEN_AI_RESPONSE_MODEL, attrs) + self.assertNotIn(GEN_AI_RESPONSE_STATUS, attrs) + self.assertNotIn(GenAI.GEN_AI_RESPONSE_FINISH_REASONS, attrs) + self.assertNotIn(GEN_AI_REQUEST_STREAM_CURSOR, attrs) + self.assertNotIn(server_attributes.SERVER_ADDRESS, attrs) + self.assertNotIn(server_attributes.SERVER_PORT, attrs) + + def test_stop_sets_custom_attributes(self) -> None: + invocation = self._fetch_response() + invocation.attributes["openai.api.type"] = "responses" + invocation.stop() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual(attrs["openai.api.type"], "responses") + + # ------------------------------------------------------------------ + # metrics — a fetch performs no inference, so no token usage + # ------------------------------------------------------------------ + + def test_stop_records_duration_without_token_usage(self) -> None: + invocation = self._fetch_response() + invocation.response_model_name = "gpt-4o-mini-2024-07-18" + invocation.stop() + + metrics = self._get_metrics() + self.assertNotIn("gen_ai.client.token.usage", metrics) + + duration = metrics["gen_ai.client.operation.duration"] + (point,) = duration.data.data_points + self.assertEqual( + point.attributes[GenAI.GEN_AI_OPERATION_NAME], "fetch_response" + ) + self.assertEqual( + point.attributes[GenAI.GEN_AI_RESPONSE_MODEL], + "gpt-4o-mini-2024-07-18", + ) + # High cardinality — must stay off the metric. + self.assertNotIn(GenAI.GEN_AI_RESPONSE_ID, point.attributes) + + # ------------------------------------------------------------------ + # fail + # ------------------------------------------------------------------ + + def test_fail_sets_error_status_and_type(self) -> None: + invocation = self._fetch_response() + invocation.fail(Error(message="not found", type="NotFoundError")) + + span = self._get_finished_spans()[0] + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.status.description, "not found") + self.assertEqual(span.attributes["error.type"], "NotFoundError") + + def test_fail_with_exception_instance(self) -> None: + invocation = self._fetch_response() + invocation.fail(ValueError("oops")) + + span = self._get_finished_spans()[0] + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.attributes["error.type"], "ValueError") + + def test_fail_records_error_type_on_duration_metric(self) -> None: + self._fetch_response().fail(ValueError("oops")) + + duration = self._get_metrics()["gen_ai.client.operation.duration"] + (point,) = duration.data.data_points + self.assertEqual(point.attributes["error.type"], "ValueError") + + # ------------------------------------------------------------------ + # context manager + # ------------------------------------------------------------------ + + def test_context_manager_ends_span(self) -> None: + with self._fetch_response() as invocation: + self.assertIsInstance(invocation, FetchResponseInvocation) + + spans = self._get_finished_spans() + self.assertEqual(len(spans), 1) + self.assertEqual(spans[0].status.status_code, StatusCode.UNSET) + + def test_context_manager_reraises_exception(self) -> None: + with pytest.raises(ValueError, match="fetch failed"): + with self._fetch_response(): + raise ValueError("fetch failed") + + span = self._get_finished_spans()[0] + self.assertEqual(span.status.status_code, StatusCode.ERROR) + self.assertEqual(span.attributes["error.type"], "ValueError") + + +class TelemetryHandlerFetchResponseContentTest(_FetchResponseTestBase): + # ------------------------------------------------------------------ + # opt-in content + # ------------------------------------------------------------------ + + def _fetch_with_content(self) -> None: + invocation = self._fetch_response() + invocation.output_messages = [ + OutputMessage( + role="assistant", + parts=[Text(content="This is a test.")], + finish_reason="stop", + ) + ] + invocation.system_instruction = [ + Text(content="You are a helpful assistant.") + ] + invocation.tool_definitions = [ + FunctionToolDefinition( + name="get_weather", description=None, parameters=None + ) + ] + invocation.stop() + + def test_content_captured_on_span_when_enabled(self) -> None: + with patch.dict( + os.environ, + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_ONLY"}, + ): + self._fetch_with_content() + + attrs = self._get_finished_spans()[0].attributes + self.assertEqual( + json.loads(attrs[GenAI.GEN_AI_OUTPUT_MESSAGES]), + [ + { + "role": "assistant", + "parts": [{"content": "This is a test.", "type": "text"}], + "finish_reason": "stop", + } + ], + ) + self.assertEqual( + json.loads(attrs[GenAI.GEN_AI_SYSTEM_INSTRUCTIONS]), + [{"content": "You are a helpful assistant.", "type": "text"}], + ) + # A fetched response carries no input messages. + self.assertNotIn(GenAI.GEN_AI_INPUT_MESSAGES, attrs) + + def test_content_suppressed_on_span_when_disabled(self) -> None: + self._fetch_with_content() + + attrs = self._get_finished_spans()[0].attributes + self.assertNotIn(GenAI.GEN_AI_OUTPUT_MESSAGES, attrs) + self.assertNotIn(GenAI.GEN_AI_SYSTEM_INSTRUCTIONS, attrs) From a47f40441341d91a3796898543b3708e1955c2f2 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Tue, 4 Aug 2026 13:10:54 -0700 Subject: [PATCH 06/14] Capture tool definitions on fetch_response spans A fetched response carries the tool definitions the original generation was made with, and semconv lists gen_ai.tool.definitions as opt-in content on the fetch_response span, but set_fetch_response_attributes never recorded them. Responses API tools are flat, so the Chat Completions helper _prepare_tool_definitions() does not apply: it reads tool.function.name and would return nothing here. Add a Responses-specific extractor that maps function tools onto FunctionToolDefinition and built-in tools, which carry no name, onto GenericToolDefinition keyed by their type. Assisted-by: Claude Opus 5 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../genai/openai/response_extractors.py | 56 +++++++++ .../tests/test_response_extractors.py | 107 +++++++++++++++++- 2 files changed, 162 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 32438dd2e..5c52d06bf 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -18,6 +18,7 @@ from ._raw_response import ParsableResponse from .utils import ( _openai_response_format_to_output_type, + get_property_value, get_server_address_and_port, ) @@ -30,6 +31,7 @@ InputMessage, OutputMessage, Text, + ToolDefinition, ) try: @@ -60,6 +62,8 @@ try: from opentelemetry.util.genai.types import ( Error, + FunctionToolDefinition, + GenericToolDefinition, InputMessage, OutputMessage, Reasoning, @@ -70,6 +74,8 @@ ) except ImportError: Error = None + FunctionToolDefinition = None + GenericToolDefinition = None InputMessage = None OutputMessage = None Reasoning = None @@ -303,6 +309,53 @@ def get_fetched_finish_reasons(response: "Response | None") -> list[str]: return [] +def get_tool_definitions_from_response( + response: "Response | None", +) -> list["ToolDefinition"] | None: + """Return the tool definitions carried on a fetched response. + + Responses API tools are flat -- a function tool holds ``name``, + ``description`` and ``parameters`` directly, unlike the Chat Completions + shape that nests them under ``function``. Built-in tools (``web_search``, + ``file_search``, ...) are identified by ``type`` alone and carry no name, + so they are reported as generic definitions keyed by their type. + """ + if ( + Response is None + or not isinstance(response, Response) + or FunctionToolDefinition is None + or GenericToolDefinition is None + ): + return None + + tools = response.tools + if not tools: + return None + + definitions: list["ToolDefinition"] = [] + for tool in tools: + tool_type = get_property_value(tool, "type") + if not isinstance(tool_type, str): + continue + name = get_property_value(tool, "name") + if tool_type == "function": + definitions.append( + FunctionToolDefinition( + name=name if isinstance(name, str) else "", + description=get_property_value(tool, "description"), + parameters=get_property_value(tool, "parameters"), + ) + ) + else: + definitions.append( + GenericToolDefinition( + name=name if isinstance(name, str) else tool_type, + type=tool_type, + ) + ) + return definitions or None + + def _response_types_available() -> bool: return ( Response is not None @@ -584,3 +637,6 @@ def set_fetch_response_attributes( invocation.output_messages = get_output_messages_from_response( response ) + invocation.tool_definitions = get_tool_definitions_from_response( + response + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index 20573d312..13559f306 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -10,7 +10,11 @@ from opentelemetry.semconv._incubating.attributes import ( openai_attributes as OpenAIAttributes, ) -from opentelemetry.util.genai.types import LLMInvocation +from opentelemetry.util.genai.types import ( + FunctionToolDefinition, + GenericToolDefinition, + LLMInvocation, +) try: # Responses types are not available in the oldest supported OpenAI SDK. @@ -465,3 +469,104 @@ def test_set_fetch_response_attributes_tolerates_missing_service_tier(): OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER not in invocation.attributes ) + + +def test_get_tool_definitions_from_response_maps_flat_responses_tools( + loaded_module, +): + """Responses API tools are flat, unlike the nested Chat Completions shape.""" + response = _make_response( + tools=[ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + "strict": True, + } + ] + ) + + definitions = loaded_module.get_tool_definitions_from_response(response) + + (definition,) = definitions + assert isinstance(definition, FunctionToolDefinition) + assert definition.type == "function" + assert definition.name == "get_weather" + assert definition.description == "Get the weather" + assert definition.parameters == { + "type": "object", + "properties": {"city": {"type": "string"}}, + } + + +def test_get_tool_definitions_from_response_maps_builtin_tools_by_type( + loaded_module, +): + """Built-in tools carry no name, so their type identifies them.""" + response = _make_response( + tools=[{"type": "web_search_preview"}], + ) + + definitions = loaded_module.get_tool_definitions_from_response(response) + + (definition,) = definitions + assert isinstance(definition, GenericToolDefinition) + assert definition.type == "web_search_preview" + assert definition.name == "web_search_preview" + + +def test_get_tool_definitions_from_response_returns_none_without_tools( + loaded_module, +): + assert loaded_module.get_tool_definitions_from_response(None) is None + assert ( + loaded_module.get_tool_definitions_from_response(_make_response()) + is None + ) + + +def test_set_fetch_response_attributes_captures_tool_definitions( + loaded_module, +): + """Tool definitions are captured only when content capture is enabled.""" + response = _make_response( + status="completed", + tools=[ + { + "type": "function", + "name": "get_weather", + "description": None, + "parameters": {"type": "object"}, + "strict": True, + } + ], + ) + + def _make_invocation(): + return SimpleNamespace( + response_model_name=None, + response_status=None, + finish_reasons=None, + output_messages=[], + system_instruction=[], + tool_definitions=None, + attributes={}, + ) + + captured = _make_invocation() + loaded_module.set_fetch_response_attributes( + captured, response, capture_content=True + ) + (definition,) = captured.tool_definitions + assert isinstance(definition, FunctionToolDefinition) + assert definition.name == "get_weather" + + not_captured = _make_invocation() + loaded_module.set_fetch_response_attributes( + not_captured, response, capture_content=False + ) + assert not_captured.tool_definitions is None From d26768a02c7edf4c505d9f72ad2202a48b1e0025 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Wed, 5 Aug 2026 14:31:53 -0700 Subject: [PATCH 07/14] Reuse response raw parsing for retrieval Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../genai/openai/patch_responses.py | 54 ++++--------------- .../genai/openai/response_extractors.py | 38 +++++++++++-- 2 files changed, 44 insertions(+), 48 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py index b50b16f7c..5abb9d790 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py @@ -3,9 +3,8 @@ from __future__ import annotations -import inspect -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Union, cast +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Any, Union, cast from opentelemetry.semconv._incubating.attributes import ( openai_attributes as OpenAIAttributes, @@ -13,7 +12,7 @@ from opentelemetry.util.genai.handler import TelemetryHandler from opentelemetry.util.genai.invocation import FetchResponseInvocation -from ._raw_response import ParsableResponse, wrap_stream_result +from ._raw_response import wrap_stream_result from .response_extractors import ( apply_request_attributes, extract_params, @@ -55,39 +54,6 @@ "OpenAIAsyncStream[Any]", "AsyncResponseStream[Any]" ] -# Set by the SDK's ``with_streaming_response`` helpers to mark a request whose -# body the caller reads lazily. -_RAW_RESPONSE_HEADER = "x-stainless-raw-response" - - -def _is_streamed_raw_response(kwargs: dict[str, Any]) -> bool: - extra_headers = kwargs.get("extra_headers") - if not isinstance(extra_headers, Mapping): - return False - return any( - key.lower() == _RAW_RESPONSE_HEADER and value == "stream" - for key, value in extra_headers.items() - ) - - -def _parse_raw_response(result: object, kwargs: dict[str, Any]) -> object: - """Return the payload of a non-streaming ``with_raw_response`` result. - - A non-streaming raw response has already read its body and the SDK memoizes - ``parse()``, so deserializing it here does not change what the caller sees. - On the async client ``parse()`` returns an awaitable, which the caller - awaits. - - ``with_streaming_response`` hands back a raw response whose body has *not* - been read yet; parsing it would consume the body before the caller asks for - it, so those calls are left alone. - """ - if _is_streamed_raw_response(kwargs) or not isinstance( - result, ParsableResponse - ): - return result - return result.parse() - def responses_create( handler: TelemetryHandler, @@ -133,7 +99,7 @@ def traced_method( ) set_invocation_response_attributes( - invocation, result, capture_content + invocation, result, capture_content, kwargs ) error = get_response_error(result) if error is not None: @@ -206,7 +172,7 @@ async def traced_method( ) set_invocation_response_attributes( - invocation, result, capture_content + invocation, result, capture_content, kwargs ) error = get_response_error(result) if error is not None: @@ -308,8 +274,9 @@ def traced_method( set_fetch_response_attributes( invocation, - _parse_raw_response(result, kwargs), + result, capture_content, + kwargs, ) invocation.stop() return result @@ -378,10 +345,9 @@ async def traced_method( capture_content, ) - parsed = _parse_raw_response(result, kwargs) - if inspect.isawaitable(parsed): - parsed = await parsed - set_fetch_response_attributes(invocation, parsed, capture_content) + set_fetch_response_attributes( + invocation, result, capture_content, kwargs + ) invocation.stop() return result except Exception as error: diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 19ac89d60..e341aa090 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -564,15 +564,42 @@ def extract_usage_tokens(usage: ResponseUsage | None) -> UsageTokens: ) +_RAW_RESPONSE_HEADER = "x-stainless-raw-response" + + +def _is_streamed_raw_response( + request_kwargs: dict[str, object] | None, +) -> bool: + if request_kwargs is None: + return False + extra_headers = request_kwargs.get("extra_headers") + if not isinstance(extra_headers, Mapping): + return False + return any( + key.lower() == _RAW_RESPONSE_HEADER and value == "stream" + for key, value in extra_headers.items() + ) + + +def _parse_raw_response( + response: object, + request_kwargs: dict[str, object] | None, +) -> object: + """Return the payload of a non-streaming ``with_raw_response`` result.""" + if _is_streamed_raw_response(request_kwargs) or not isinstance( + response, ParsableResponse + ): + return response + return response.parse() + + def set_invocation_response_attributes( invocation, response: object, capture_content: bool, + request_kwargs: dict[str, object] | None = None, ) -> None: - if isinstance(response, ParsableResponse): - # with_raw_response: safe to parse() here since this is the - # non-streaming path, so it has no side effects on the caller's stream. - response = response.parse() + response = _parse_raw_response(response, request_kwargs) if Response is None or not isinstance(response, Response): return @@ -605,6 +632,7 @@ def set_fetch_response_attributes( invocation, response: object, capture_content: bool, + request_kwargs: dict[str, object] | None = None, ) -> None: """Record a fetched response on a ``fetch_response`` invocation. @@ -613,6 +641,8 @@ def set_fetch_response_attributes( The original input messages are not part of the fetched response either, so only the system instructions and output messages it carries are captured. """ + response = _parse_raw_response(response, request_kwargs) + if Response is None or not isinstance(response, Response): return From f15387a81cf63bd900cfec5bcf9c6450688c9380 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Wed, 5 Aug 2026 14:38:33 -0700 Subject: [PATCH 08/14] Handle errors from raw Responses results Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../genai/openai/patch_responses.py | 4 +-- .../genai/openai/response_extractors.py | 7 ++++- .../tests/test_response_extractors.py | 30 +++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py index 5abb9d790..748fa939a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py @@ -101,7 +101,7 @@ def traced_method( set_invocation_response_attributes( invocation, result, capture_content, kwargs ) - error = get_response_error(result) + error = get_response_error(result, kwargs) if error is not None: invocation.fail(error) else: @@ -174,7 +174,7 @@ async def traced_method( set_invocation_response_attributes( invocation, result, capture_content, kwargs ) - error = get_response_error(result) + error = get_response_error(result, kwargs) if error is not None: invocation.fail(error) else: diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index e341aa090..6966444ae 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -460,13 +460,18 @@ def extract_finish_reasons(response: Response | None) -> list[str]: return list(dict.fromkeys(finish_reasons)) -def get_response_error(response: Response | None) -> Error | None: +def get_response_error( + response: object, + request_kwargs: dict[str, object] | None = None, +) -> Error | None: """Return an ``Error`` when the response failed, else ``None``. A failed response carries a ``ResponseError`` (``code`` + ``message``). Incomplete responses (``incomplete_details``) are *not* errors — they surface as a finish reason instead. """ + response = _parse_raw_response(response, request_kwargs) + if Response is None or Error is None or not isinstance(response, Response): return None error = response.error diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index e9b9d7601..dcd248a47 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -227,6 +227,36 @@ def test_get_response_error_returns_error_for_failed_response(loaded_module): assert error.message == "boom" +def test_get_response_error_parses_raw_response(loaded_module): + response = _make_response( + status="failed", + error={"code": "server_error", "message": "boom"}, + ) + raw_response = SimpleNamespace(parse=mock.Mock(return_value=response)) + + error = loaded_module.get_response_error(raw_response) + + assert error is not None + assert error.type == "server_error" + raw_response.parse.assert_called_once_with() + + +def test_get_response_error_keeps_streaming_raw_response_lazy(loaded_module): + raw_response = SimpleNamespace(parse=mock.Mock()) + + error = loaded_module.get_response_error( + raw_response, + { + "extra_headers": { + "X-Stainless-Raw-Response": "stream", + } + }, + ) + + assert error is None + raw_response.parse.assert_not_called() + + def test_get_response_error_none_for_incomplete_response(loaded_module): # Incomplete is a finish reason, not an error. response = _make_response( From 5a51616c68f8f89267d27646d4e61bdff67717bc Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Wed, 5 Aug 2026 15:02:42 -0700 Subject: [PATCH 09/14] Align Responses finish reason extraction Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../genai/openai/response_extractors.py | 62 ++++++++----------- .../tests/test_response_extractors.py | 4 +- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 6966444ae..b31e196dd 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -263,14 +263,6 @@ def _extract_reasoning_parts( return parts -def _finish_reason_from_status(status: str | None) -> str | None: - if status == "completed": - return "stop" - if status in {"failed", "cancelled", "incomplete"}: - return status - return None - - # `incomplete_details.reason` values that map onto a cross-provider finish # reason; an unrecognized reason is reported as-is. _INCOMPLETE_REASON_TO_FINISH_REASON = { @@ -279,34 +271,21 @@ def _finish_reason_from_status(status: str | None) -> str | None: } -def get_fetched_finish_reasons(response: "Response | None") -> list[str]: - """Return the finish reasons of the *original* generation of a fetched response. - - Derived from the fetched response ``status`` as the OpenAI ``fetch_response`` - semantic conventions require: a ``completed`` response maps to its stop - reason, an ``incomplete`` one to why it was cut short, and a ``failed`` or - ``cancelled`` one to ``error``. - - Statuses that are not terminal (``queued``, ``in_progress``) have no finish - reason yet, and so do a missing or unrecognized status: the generation is - not known to have stopped, so reporting one would be wrong. The lifecycle - state is conveyed by ``gen_ai.response.status`` instead. - """ - if Response is None or not isinstance(response, Response): - return [] - - status = response.status +def _finish_reason_from_status( + status: str | None, + incomplete_reason: str | None = None, +) -> str | None: if status == "completed": - return extract_finish_reasons(response) or ["stop"] + return "stop" if status in {"failed", "cancelled"}: - return ["error"] + return "error" if status == "incomplete": - details = response.incomplete_details - reason = details.reason if details is not None else None - if reason is None: - return [status] - return [_INCOMPLETE_REASON_TO_FINISH_REASON.get(reason, reason)] - return [] + if incomplete_reason is None: + return status + return _INCOMPLETE_REASON_TO_FINISH_REASON.get( + incomplete_reason, incomplete_reason + ) + return None def get_tool_definitions_from_response( @@ -443,6 +422,16 @@ def extract_finish_reasons(response: Response | None) -> list[str]: ): return [] + incomplete_details = response.incomplete_details + response_finish_reason = _finish_reason_from_status( + response.status, + incomplete_details.reason if incomplete_details is not None else None, + ) + if response.status in {"failed", "cancelled", "incomplete"}: + return [response_finish_reason] if response_finish_reason else [] + if response.status in {"queued", "in_progress"}: + return [] + finish_reasons: list[str] = [] for item in response.output: if isinstance(item, ResponseFunctionToolCall) and item.status in { @@ -457,7 +446,10 @@ def extract_finish_reasons(response: Response | None) -> list[str]: finish_reason = _finish_reason_from_status(item.status) if finish_reason is not None: finish_reasons.append(finish_reason) - return list(dict.fromkeys(finish_reasons)) + finish_reasons = list(dict.fromkeys(finish_reasons)) + if finish_reasons: + return finish_reasons + return [response_finish_reason] if response_finish_reason else [] def get_response_error( @@ -653,7 +645,7 @@ def set_fetch_response_attributes( invocation.response_model_name = response.model invocation.response_status = response.status - invocation.finish_reasons = get_fetched_finish_reasons(response) or None + invocation.finish_reasons = extract_finish_reasons(response) or None # `service_tier` is absent from the Response model on some supported SDK # versions, so keep this attribute access guarded. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index dcd248a47..43c7f01b9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -460,12 +460,12 @@ def test_response_extractors_ignore_invalid_shapes_without_validation( ({}, []), ], ) -def test_get_fetched_finish_reasons_maps_status( +def test_extract_finish_reasons_maps_response_status( loaded_module, overrides, expected ): response = _make_response(**overrides) - assert loaded_module.get_fetched_finish_reasons(response) == expected + assert loaded_module.extract_finish_reasons(response) == expected def test_set_fetch_response_attributes_tolerates_missing_service_tier(): From d790b69c0c4f89ace33e96713f2ac4fb750d0bde Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Wed, 5 Aug 2026 15:15:19 -0700 Subject: [PATCH 10/14] Set API type for Responses create Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../instrumentation/genai/openai/response_extractors.py | 3 +++ .../tests/test_async_responses.py | 7 +++++++ .../tests/test_responses.py | 7 +++++++ 3 files changed, 17 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index b31e196dd..b35062840 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -513,6 +513,9 @@ def apply_request_attributes( params: ResponseRequestParams, capture_content: bool, ) -> None: + invocation.attributes[OpenAIAttributes.OPENAI_API_TYPE] = ( + OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) invocation.temperature = params.temperature invocation.top_p = params.top_p invocation.max_tokens = params.max_output_tokens diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 1d30d1cec..5750e401b 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -25,6 +25,9 @@ from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) @@ -196,6 +199,10 @@ async def test_async_responses_create_basic( assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( "stop", ) + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 4850b0066..c1a061e28 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -24,6 +24,9 @@ from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) +from opentelemetry.semconv._incubating.attributes import ( + openai_attributes as OpenAIAttributes, +) from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) @@ -260,6 +263,10 @@ def test_responses_create_basic( assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( "stop", ) + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes assert GenAIAttributes.GEN_AI_OUTPUT_MESSAGES not in span.attributes From 6b5d4baf6357ab1d46776790e1a5d5a6090311be Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Wed, 5 Aug 2026 15:23:45 -0700 Subject: [PATCH 11/14] Record streaming fetch requests Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../instrumentation/genai/openai/patch_responses.py | 8 +++++++- .../instrumentation/genai/openai/response_extractors.py | 4 ++-- .../tests/test_async_responses.py | 2 ++ .../tests/test_responses.py | 2 ++ .../tests/test_utils.py | 4 ++++ .../util/genai/_fetch_response_invocation.py | 9 +++++++++ .../src/opentelemetry/util/genai/handler.py | 2 ++ .../tests/test_handler_fetch_response.py | 9 +++++++++ 8 files changed, 37 insertions(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py index 748fa939a..5cfac2495 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py @@ -19,6 +19,7 @@ get_fetch_response_creation_kwargs, get_inference_creation_kwargs, get_response_error, + is_streamed_raw_response, set_fetch_response_attributes, set_invocation_response_attributes, ) @@ -213,7 +214,12 @@ def _start_fetch_response_invocation( kwargs: dict[str, Any], ) -> FetchResponseInvocation: invocation = handler.fetch_response( - **get_fetch_response_creation_kwargs(response_id, instance) + **get_fetch_response_creation_kwargs(response_id, instance), + request_stream=( + True + if is_streaming(kwargs) or is_streamed_raw_response(kwargs) + else None + ), ) invocation.stream_cursor = _get_stream_cursor(kwargs) invocation.attributes[OpenAIAttributes.OPENAI_API_TYPE] = ( diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index b35062840..5f6b17ece 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -567,7 +567,7 @@ def extract_usage_tokens(usage: ResponseUsage | None) -> UsageTokens: _RAW_RESPONSE_HEADER = "x-stainless-raw-response" -def _is_streamed_raw_response( +def is_streamed_raw_response( request_kwargs: dict[str, object] | None, ) -> bool: if request_kwargs is None: @@ -586,7 +586,7 @@ def _parse_raw_response( request_kwargs: dict[str, object] | None, ) -> object: """Return the payload of a non-streaming ``with_raw_response`` result.""" - if _is_streamed_raw_response(request_kwargs) or not isinstance( + if is_streamed_raw_response(request_kwargs) or not isinstance( response, ParsableResponse ): return response diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 5750e401b..53baeb704 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -365,6 +365,7 @@ async def test_async_responses_retrieve_streaming( response_model=response.model, response_status="completed", finish_reasons=("stop",), + request_stream=True, stream_cursor=str(RETRIEVE_STREAM_CURSOR), response_service_tier=response.service_tier, ) @@ -439,6 +440,7 @@ async def test_async_responses_retrieve_with_streaming_response_stays_lazy( span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == RETRIEVE_RESPONSE_ID ) + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_STREAM] is True assert response.id == RETRIEVE_RESPONSE_ID diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index c1a061e28..9753d71e0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -402,6 +402,7 @@ def test_responses_retrieve_streaming( response_model=response.model, response_status="completed", finish_reasons=("stop",), + request_stream=True, stream_cursor=str(RETRIEVE_STREAM_CURSOR), response_service_tier=response.service_tier, ) @@ -460,6 +461,7 @@ def test_responses_retrieve_with_streaming_response_stays_lazy( span.attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] == RETRIEVE_RESPONSE_ID ) + assert span.attributes[GenAIAttributes.GEN_AI_REQUEST_STREAM] is True assert response.id == RETRIEVE_RESPONSE_ID diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py index f5ba2edb1..f748f95f1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_utils.py @@ -160,6 +160,7 @@ def assert_fetch_response_attributes( response_model: str | None = None, response_status: str | None = None, finish_reasons: tuple | None = None, + request_stream: bool | None = None, stream_cursor: str | None = None, response_service_tier: str | None = None, server_address: str = "api.openai.com", @@ -200,6 +201,9 @@ def assert_fetch_response_attributes( _assert_optional_attribute( span, GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, finish_reasons ) + _assert_optional_attribute( + span, GenAIAttributes.GEN_AI_REQUEST_STREAM, request_stream + ) _assert_optional_attribute( span, GEN_AI_REQUEST_STREAM_CURSOR, stream_cursor ) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py index 53f9aaaef..d3bdb75b7 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_fetch_response_invocation.py @@ -53,6 +53,8 @@ class FetchResponseInvocation(GenAIInvocation): - error.type: Error type when the fetch itself failed (Conditionally Required) - gen_ai.request.stream_cursor: Set from ``stream_cursor`` when the fetch resumes a streamed response from a prior position (Conditionally Required) + - gen_ai.request.stream: Set to true when the fetched response is streamed + (Conditionally Required) - server.port: Set only when ``server_port`` is provided (Conditionally Required) - gen_ai.response.finish_reasons: Outcome of the original generation (Recommended) @@ -81,6 +83,7 @@ def __init__( provider: str, *, response_id: str, + request_stream: bool | None = None, server_address: str | None = None, server_port: int | None = None, error_type_resolver: ErrorTypeResolver | None = None, @@ -100,6 +103,7 @@ def __init__( ) self._provider: str = provider self._response_id: str = response_id + self._request_stream = request_stream self._server_address: str | None = server_address self._server_port: int | None = server_port self.response_model_name: str | None = None @@ -126,6 +130,11 @@ def _get_start_attributes(self) -> dict[str, AttributeValue]: GenAI.GEN_AI_OPERATION_NAME: self._operation_name, GenAI.GEN_AI_PROVIDER_NAME: self._provider, GenAI.GEN_AI_RESPONSE_ID: self._response_id, + **( + {GenAI.GEN_AI_REQUEST_STREAM: self._request_stream} + if self._request_stream is not None + else {} + ), **{k: v for k, v in optional_attrs if v is not None}, } diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py index 2f72271a0..50e2a031c 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py @@ -369,6 +369,7 @@ def fetch_response( provider: str, *, response_id: str, + request_stream: bool | None = None, server_address: str | None = None, server_port: int | None = None, error_type_resolver: ErrorTypeResolver | None = None, @@ -392,6 +393,7 @@ def fetch_response( self._completion_hook, provider=provider, response_id=response_id, + request_stream=request_stream, server_address=server_address, server_port=server_port, error_type_resolver=error_type_resolver, diff --git a/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py b/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py index 0e45cd9d1..e94facb66 100644 --- a/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py +++ b/util/opentelemetry-util-genai/tests/test_handler_fetch_response.py @@ -123,6 +123,14 @@ def test_response_id_is_exposed(self) -> None: self.assertEqual(invocation.response_id, RESPONSE_ID) invocation.stop() + def test_request_stream_is_set_at_span_creation(self) -> None: + invocation = self._fetch_response(request_stream=True) + + self.assertIs( + invocation.span.attributes[GenAI.GEN_AI_REQUEST_STREAM], True + ) + invocation.stop() + def test_stop_sets_server_address_and_port(self) -> None: self._fetch_response( server_address="api.openai.com", server_port=8080 @@ -169,6 +177,7 @@ def test_stop_omits_unset_attributes(self) -> None: self.assertNotIn(GenAI.GEN_AI_RESPONSE_MODEL, attrs) self.assertNotIn(GEN_AI_RESPONSE_STATUS, attrs) self.assertNotIn(GenAI.GEN_AI_RESPONSE_FINISH_REASONS, attrs) + self.assertNotIn(GenAI.GEN_AI_REQUEST_STREAM, attrs) self.assertNotIn(GEN_AI_REQUEST_STREAM_CURSOR, attrs) self.assertNotIn(server_attributes.SERVER_ADDRESS, attrs) self.assertNotIn(server_attributes.SERVER_PORT, attrs) From 4c74d37a228ca7da78e0c9d8f85ebf8ca1621d40 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Wed, 5 Aug 2026 15:29:21 -0700 Subject: [PATCH 12/14] Remove redundant retrieve guards Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../instrumentation/genai/openai/__init__.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py index 39e9e6347..fc70098d1 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py @@ -193,18 +193,16 @@ def _instrument(self, **kwargs): # retrieve() fetches a stored response by id. No inference happens # and no tokens are consumed, so it is traced as a fetch_response # operation rather than through the create wrappers. - if hasattr(responses_module.Responses, "retrieve"): - wrap_function_wrapper( - "openai.resources.responses.responses", - "Responses.retrieve", - responses_retrieve(handler), - ) - if hasattr(responses_module.AsyncResponses, "retrieve"): - wrap_function_wrapper( - "openai.resources.responses.responses", - "AsyncResponses.retrieve", - async_responses_retrieve(handler), - ) + wrap_function_wrapper( + "openai.resources.responses.responses", + "Responses.retrieve", + responses_retrieve(handler), + ) + wrap_function_wrapper( + "openai.resources.responses.responses", + "AsyncResponses.retrieve", + async_responses_retrieve(handler), + ) def _uninstrument(self, **kwargs): import openai # pylint: disable=import-outside-toplevel @@ -222,10 +220,8 @@ def _uninstrument(self, **kwargs): unwrap(responses_module.Responses, "stream") unwrap(responses_module.AsyncResponses, "create") unwrap(responses_module.AsyncResponses, "stream") - if hasattr(responses_module.Responses, "retrieve"): - unwrap(responses_module.Responses, "retrieve") - if hasattr(responses_module.AsyncResponses, "retrieve"): - unwrap(responses_module.AsyncResponses, "retrieve") + unwrap(responses_module.Responses, "retrieve") + unwrap(responses_module.AsyncResponses, "retrieve") def _get_responses_module(): From b45cd5a20927f5803eceeb2c70efd5b6792028b1 Mon Sep 17 00:00:00 2001 From: Jackson Weber Date: Wed, 5 Aug 2026 15:34:17 -0700 Subject: [PATCH 13/14] Fix OpenAI Ruff violations Assisted-by: GPT-5.6 Sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../genai/openai/patch_responses.py | 6 +++--- .../genai/openai/response_extractors.py | 6 +++--- .../genai/openai/response_wrappers.py | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py index 5cfac2495..89c5f4530 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/patch_responses.py @@ -209,7 +209,7 @@ def _get_stream_cursor(kwargs: dict[str, Any]) -> str | None: def _start_fetch_response_invocation( handler: TelemetryHandler, - instance: "Responses | AsyncResponses", + instance: Responses | AsyncResponses, response_id: str, kwargs: dict[str, Any], ) -> FetchResponseInvocation: @@ -250,7 +250,7 @@ def responses_retrieve( def traced_method( wrapped: Callable[..., Union[ResponseResult, ResponseStreamResult]], - instance: "Responses", + instance: Responses, args: tuple[Any, ...], kwargs: dict[str, Any], ) -> Union[ @@ -323,7 +323,7 @@ async def traced_method( ..., Awaitable[Union[ResponseResult, AsyncResponseStreamResult]], ], - instance: "AsyncResponses", + instance: AsyncResponses, args: tuple[Any, ...], kwargs: dict[str, Any], ) -> Union[ diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 5f6b17ece..203da2cc0 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -289,8 +289,8 @@ def _finish_reason_from_status( def get_tool_definitions_from_response( - response: "Response | None", -) -> list["ToolDefinition"] | None: + response: Response | None, +) -> list[ToolDefinition] | None: """Return the tool definitions carried on a fetched response. Responses API tools are flat -- a function tool holds ``name``, @@ -311,7 +311,7 @@ def get_tool_definitions_from_response( if not tools: return None - definitions: list["ToolDefinition"] = [] + definitions: list[ToolDefinition] = [] for tool in tools: tool_type = get_property_value(tool, "type") if not isinstance(tool_type, str): diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py index 3c3127e1a..bb5c009db 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_wrappers.py @@ -84,8 +84,8 @@ def _set_response_attributes( def _set_fetch_response_attributes( - invocation: "GenAIInvocation", - result: "ParsedResponse[TextFormatT] | Response | None", + invocation: GenAIInvocation, + result: ParsedResponse[TextFormatT] | Response | None, capture_content: bool, ) -> None: if set_fetch_response_attributes is None: @@ -157,7 +157,7 @@ def _stop( self._self_response_telemetry_finalized = True def _apply_response_attributes( - self, result: "ParsedResponse[TextFormatT] | Response | None" + self, result: ParsedResponse[TextFormatT] | Response | None ) -> None: """Record the response on the invocation. Overridden per operation.""" _set_response_attributes( @@ -165,7 +165,7 @@ def _apply_response_attributes( ) def _on_response_failed( - self, response: "ParsedResponse[TextFormatT] | Response | None" + self, response: ParsedResponse[TextFormatT] | Response | None ) -> None: """Handle a ``response.failed`` event. Overridden per operation.""" self._apply_response_attributes(response) @@ -282,18 +282,18 @@ class _FetchResponseStreamMixin(Generic[TextFormatT]): a failure of this fetch. """ - _self_invocation: "GenAIInvocation" + _self_invocation: GenAIInvocation _self_capture_content: bool def _apply_response_attributes( - self, result: "ParsedResponse[TextFormatT] | Response | None" + self, result: ParsedResponse[TextFormatT] | Response | None ) -> None: _set_fetch_response_attributes( self._self_invocation, result, self._self_capture_content ) def _on_response_failed( - self, response: "ParsedResponse[TextFormatT] | Response | None" + self, response: ParsedResponse[TextFormatT] | Response | None ) -> None: self._stop(response) From 37949a9caa1639ee137292eac3bff15ca9b5af2f Mon Sep 17 00:00:00 2001 From: Jackson Weber <47067795+JacksonWeber@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:40:28 -0700 Subject: [PATCH 14/14] chore: retrigger CI Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>