From 4de1a844460392252f0e49a67ada5a559d0a237d Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 3 Jul 2026 15:01:49 +0200 Subject: [PATCH 01/11] Add AI Guard<->OpenAI integration test under INTEGRATION_FRAMEWORKS (APPSEC-68977) Exercise the auto-instrumented AI Guard/OpenAI integration (ai_guard_listen) the way the LLM Obs suite does: call the OpenAI SDK directly through the existing /chat/completions weblog endpoint with DD_AI_GUARD_ENABLED=true, and assert an ai_guard span is emitted for before-model, after-model (streamed) and tool-call evaluation. The evaluation outcome (ALLOW/DENY/ABORT) stays covered by the dedicated AI_GUARD scenario and is not re-asserted here. The AI Guard endpoint is not yet wired into the INTEGRATION_FRAMEWORKS scenario (no DD_AI_GUARD_ENDPOINT / no aiguard VCR cassettes), so the integration path 500s for every language. Marked missing_feature for python, nodejs and java until the scenario wiring + cassettes land. Investigation task: APPSEC-68977 Co-Authored-By: Claude Opus 4.8 (1M context) --- manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/python.yml | 1 + .../llm/openai/test_openai_ai_guard.py | 109 ++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 tests/integration_frameworks/llm/openai/test_openai_ai_guard.py diff --git a/manifests/java.yml b/manifests/java.yml index 3b9d80209a8..69d770a77be 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3273,6 +3273,7 @@ manifest: "*": irrelevant spring-boot: v1.63.0 tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py: missing_feature (APPSEC-68977, AI Guard/OpenAI integration not yet wired into INTEGRATION_FRAMEWORKS) tests/integration_frameworks/llm/openai/test_openai_apm.py: v1.61.0 tests/integration_frameworks/llm/openai/test_openai_llmobs.py: v1.61.0 tests/integrations/crossed_integrations/test_kafka.py::Test_Kafka: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index bdb9e546293..a8cbeee93ba 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1748,6 +1748,7 @@ manifest: : bug (MLOB-5070) ? tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools::test_generate_content_with_tools : missing_feature (Node.js LLM Observability Google GenAI integration does not submit tool definitions) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py::TestOpenAiAiGuard: missing_feature (APPSEC-68977, AI Guard/OpenAI integration not yet wired into INTEGRATION_FRAMEWORKS) tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmChatCompletions: *ref_5_76_0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmCompletions: *ref_5_76_0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmEmbeddings: *ref_5_76_0 diff --git a/manifests/python.yml b/manifests/python.yml index 5b41c3820b0..a9822f64dba 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1362,6 +1362,7 @@ manifest: tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools: v3.13.0 ? tests/integration_frameworks/llm/google_genai/test_google_genai_llmobs.py::TestGoogleGenAiGenerateContentWithTools::test_generate_content_with_tools : bug (MLOB-5071) + tests/integration_frameworks/llm/openai/test_openai_ai_guard.py::TestOpenAiAiGuard: v4.10.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmChatCompletions: v3.13.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmCompletions: v3.13.0 tests/integration_frameworks/llm/openai/test_openai_apm.py::TestOpenAiApmEmbeddings: v3.13.0 diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py new file mode 100644 index 00000000000..a7d113989d5 --- /dev/null +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -0,0 +1,109 @@ +"""AI Guard <-> OpenAI integration tests, run under the INTEGRATION_FRAMEWORKS scenario. + +Investigation task: https://datadoghq.atlassian.net/browse/APPSEC-68977 + +Unlike ``tests/ai_guard/test_ai_guard_sdk.py`` (which drives the AI Guard SDK directly via +``/ai_guard/evaluate``), this suite exercises the *integration* between AI Guard and the +OpenAI client, the same way the LLM Observability suite does: it calls the OpenAI SDK +directly through the existing weblog endpoints (``/chat/completions``). When +``DD_AI_GUARD_ENABLED=true``, ``ai_guard_listen()`` auto-wires into the OpenAI SDK, so AI +Guard evaluates the call at three points with no manual ``evaluate()`` call: + +- **before-model**: the request/prompt is evaluated before the model is called; +- **after-model**: the model response is evaluated (streamed responses require + ``DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED=true``, which buffers and reconstructs + the response before running the evaluation); +- **tool-call**: tool calls produced by the model are evaluated. + +We assert only that the integration wires each evaluation point and emits an ``ai_guard`` +span attached to the OpenAI trace. The evaluation *outcome* (ALLOW / DENY / ABORT) is +already covered by the ``AI_GUARD`` scenario and is intentionally not re-asserted here. +""" + +import pytest + +from utils import features, scenarios +from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI + +from .utils import TOOLS, BaseOpenaiTest + + +@pytest.fixture +def library_env() -> dict[str, str]: + return { + "DD_AI_GUARD_ENABLED": "true", + # after-model evaluation of streamed responses is opt-in + "DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED": "true", + # the AI Guard client requires both an API key and an app key + "DD_API_KEY": "mock_api_key", + "DD_APP_KEY": "mock_app_key", + } + + +def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: + return [span for trace in traces for span in trace if span.get("resource") == "ai_guard"] + + +@features.ai_guard +@scenarios.integration_frameworks +class TestOpenAiAiGuard(BaseOpenaiTest): + """AI Guard evaluation triggered through the auto-instrumented OpenAI integration.""" + + def test_before_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """The prompt is evaluated by AI Guard before the OpenAI model is called.""" + with test_agent.vcr_context(): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is the weather like today?"}], + parameters=dict(max_tokens=35), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the before-model evaluation" + assert any(span["meta"].get("ai_guard.target") == "prompt" for span in guard_spans), ( + "expected a before-model ai_guard span with target 'prompt'" + ) + + def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """The streamed model response is evaluated by AI Guard after the model returns.""" + with test_agent.vcr_context(stream=True): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "Tell me a short story about a robot."}], + parameters=dict(max_tokens=35, stream=True), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" + + def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): + """Tool calls produced by the model are evaluated by AI Guard.""" + with test_agent.vcr_context(): + test_client.request( + "POST", + "/chat/completions", + dict( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": "Bob is a student at Stanford University. He is studying computer science.", + } + ], + parameters=dict(tool_choice="auto", tools=TOOLS), + ), + ) + + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + assert guard_spans, "expected an ai_guard span from the tool-call evaluation" + assert any(span["meta"].get("ai_guard.target") == "tool" for span in guard_spans), ( + "expected a tool-call ai_guard span with target 'tool'" + ) From 885e13f9346b568459488e58ce0f3bbff2811162 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 3 Jul 2026 16:00:29 +0200 Subject: [PATCH 02/11] Use real AI Guard keys when recording cassettes (APPSEC-68977) library_env now injects real DD_API_KEY/DD_APP_KEY in --generate-cassettes mode (the AI Guard client calls the real API to record the aiguard cassette) and keeps mock keys on replay, where the VCR proxy matches on the request rather than auth. Investigation task: APPSEC-68977 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../llm/openai/test_openai_ai_guard.py | 19 ++++++++++++++----- utils/docker_fixtures/_test_agent.py | 4 ++++ .../_test_client_framework_integrations.py | 3 +++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index a7d113989d5..fc99a904e87 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -20,6 +20,8 @@ already covered by the ``AI_GUARD`` scenario and is intentionally not re-asserted here. """ +import os + import pytest from utils import features, scenarios @@ -29,15 +31,22 @@ @pytest.fixture -def library_env() -> dict[str, str]: - return { +def library_env(request: pytest.FixtureRequest) -> dict[str, str]: + env = { "DD_AI_GUARD_ENABLED": "true", # after-model evaluation of streamed responses is opt-in "DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED": "true", - # the AI Guard client requires both an API key and an app key - "DD_API_KEY": "mock_api_key", - "DD_APP_KEY": "mock_app_key", } + # The AI Guard client needs an API key + app key. Real keys are required when recording + # cassettes (the client calls the real AI Guard API); mock keys are fine on replay since + # the VCR proxy matches on the request, not on auth. + if request.config.option.generate_cassettes: + env["DD_API_KEY"] = os.environ["DD_API_KEY"] + env["DD_APP_KEY"] = os.environ["DD_APP_KEY"] + else: + env["DD_API_KEY"] = "mock_api_key" + env["DD_APP_KEY"] = "mock_app_key" + return env def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: diff --git a/utils/docker_fixtures/_test_agent.py b/utils/docker_fixtures/_test_agent.py index aa1f441bef2..4b70035bdc6 100644 --- a/utils/docker_fixtures/_test_agent.py +++ b/utils/docker_fixtures/_test_agent.py @@ -89,6 +89,10 @@ def get_test_agent_api( "OTLP_HTTP_PORT": str(container_otlp_http_port), "OTLP_GRPC_PORT": str(container_otlp_grpc_port), "VCR_CASSETTES_DIRECTORY": "/vcr-cassettes", # TODO comment + # Register the AI Guard REST API as a VCR upstream so the AI Guard/OpenAI integration + # can be replayed. Additive: extends the image's built-in providers (openai, anthropic, + # ...) without overriding them (see dd-apm-test-agent vcr_proxy.PROVIDER_BASE_URLS). + "VCR_PROVIDER_MAP": "aiguard=https://app.datadoghq.com/api/v2/ai-guard", } if os.getenv("DEV_MODE") is not None: env["SNAPSHOT_CI"] = "0" diff --git a/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py b/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py index 39bfa5692a6..16c7f0b4fb6 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py +++ b/utils/docker_fixtures/_test_clients/_test_client_framework_integrations.py @@ -57,6 +57,9 @@ def get_client( environment["DD_AGENT_HOST"] = test_agent.container_name environment["DD_TRACE_AGENT_PORT"] = str(test_agent.container_port) environment["FRAMEWORK_TEST_CLIENT_SERVER_PORT"] = str(container_port) + # AI Guard evaluations (when DD_AI_GUARD_ENABLED=true) are replayed by the test-agent VCR proxy. + # Harmless when AI Guard is disabled, which is the default for framework tests. + environment["DD_AI_GUARD_ENDPOINT"] = f"{environment['DD_TRACE_AGENT_URL']}/vcr/aiguard" # overwrite env with the one provided by the test environment |= library_env From e19802ed0f0ed29018a8c2f8f71a9f44742768d3 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 3 Jul 2026 20:39:14 +0200 Subject: [PATCH 03/11] Record openai + aiguard cassettes for TestOpenAiAiGuard (APPSEC-68977) --- ...m_True_aiguard_evaluate_post_5dec5d6a.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_ca44130a.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_d13cc790.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_9e556a11.json | 38 +++++++++++ ...dation_aiguard_evaluate_post_a304603c.json | 38 +++++++++++ ...openai_chat_completions_post_5c14b241.json | 62 ++++++++++++++++++ ...openai_chat_completions_post_eb8df755.json | 63 +++++++++++++++++++ ...openai_chat_completions_post_f2dffea2.json | 63 +++++++++++++++++++ 8 files changed, 378 insertions(+) create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json create mode 100644 tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json new file mode 100644 index 00000000000..13bf51778c5 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "153", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115544" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Tell me a short story about a robot.\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "222", + "date": "Fri, 03 Jul 2026 14:50:04 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4999", + "x-ratelimit-reset": "56", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"ee239b30-9d6e-4ac1-b8b5-ebbc5d96c91d\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00159454345703125,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json new file mode 100644 index 00000000000..c555d5cebbf --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_ca44130a.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "148", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115200" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"What is the weather like today?\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "216", + "date": "Fri, 03 Jul 2026 14:49:53 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4999", + "x-ratelimit-reset": "7", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"f0863f0c-9c8c-4041-b178-d932f449d0f2\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00244140625,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json new file mode 100644 index 00000000000..81871716f6c --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_before_model_validation_aiguard_evaluate_post_d13cc790.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "394", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115200" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"What is the weather like today?\"}, {\"role\": \"assistant\", \"content\": \"I'm unable to provide real-time information, including current weather updates. I recommend checking a reliable weather website or app for the most accurate and up-to-date information about today's weather in\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "221", + "date": "Fri, 03 Jul 2026 14:49:58 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4998", + "x-ratelimit-reset": "2", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"6f8dceb7-054d-4d1f-8e35-7c4487977669\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.0013275146484375,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json new file mode 100644 index 00000000000..c0d782f31d9 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_9e556a11.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "417", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115716" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Bob is a student at Stanford University. He is studying computer science.\"}, {\"role\": \"assistant\", \"tool_calls\": [{\"id\": \"call_C5dXIRBkTQjYoQrBr3WrkQYY\", \"function\": {\"name\": \"extract_student_info\", \"arguments\": \"{\\\"name\\\":\\\"Bob\\\",\\\"major\\\":\\\"computer science\\\",\\\"school\\\":\\\"Stanford University\\\"}\"}}]}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "219", + "date": "Fri, 03 Jul 2026 14:50:13 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4997", + "x-ratelimit-reset": "47", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"4aefdbd6-761e-4bb5-ba69-f543586e8b4e\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00628662109375,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json new file mode 100644 index 00000000000..87774a918b2 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_tool_call_validation_aiguard_evaluate_post_a304603c.json @@ -0,0 +1,38 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Accept-Encoding": "identity", + "Content-Length": "190", + "Content-Type": "application/json", + "DD-AI-GUARD-VERSION": "4.10.6", + "DD-AI-GUARD-SOURCE": "SDK", + "DD-AI-GUARD-LANGUAGE": "python", + "Datadog-Entity-ID": "in-115716" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Bob is a student at Stanford University. He is studying computer science.\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", + "content-type": "application/vnd.api+json", + "vary": "Accept-Encoding", + "x-frame-options": "SAMEORIGIN", + "content-length": "219", + "date": "Fri, 03 Jul 2026 14:50:11 GMT", + "x-content-type-options": "nosniff", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit": "5000", + "x-ratelimit-period": "60", + "x-ratelimit-remaining": "4998", + "x-ratelimit-reset": "49", + "x-ratelimit-name": "ai_guard_evaluate_per_org" + }, + "body": "{\"data\":{\"id\":\"c3372620-7e64-47c1-bbd6-63a2c664334c\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00885009765625,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json new file mode 100644 index 00000000000..7bd1c1f13db --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json @@ -0,0 +1,62 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "171", + "x-datadog-trace-id": "12575373332985498166", + "x-datadog-parent-id": "4288392994079136126", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc1c00000000", + "traceparent": "00-6a47cc1c00000000ae84b36b4e854a36-3b836eb50d5c597e-01", + "tracestate": "dd=p:3b836eb50d5c597e;s:2;t.dm:-4;t.tid:6a47cc1c00000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"Tell me a short story about a robot.\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":35,\"stream\":true,\"stream_options\":{\"include_usage\":true}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:50:05 GMT", + "Content-Type": "text/event-stream; charset=utf-8", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3553e710284-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "243", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_30bd6e2837694e1abe7c71e7bfff16e1", + "set-cookie": "__cf_bm=Nfpb5Ib0QbV67YYDeOlS6T8D2.SIupabQ_RGl9iS82Y-1783090204.998873-1.0.1.1-LBFP99yI1awBQD1I3zpk3KOKCcfo3Dq2MOgJQs8Jm7Lwu1UivkU.0vZvTqsRHHt77mcU_Lw9eLo50vYY6aoRbjeuwDPRikB46h7zZ.nP1DErdbO.1PafaU0aBZelqjO9; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:20:05 GMT", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "data: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xyfxvNW9y\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Once\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"972kFcG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"n7UHCOY0\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"s2tZKJhmu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" bustling\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"un\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" city\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"THufE2\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" of\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"VOR4gL7e\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"YxFDNUv\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" future\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lQTD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"JYCvZIfp28\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" there\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"4HPFe\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" lived\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"f3Umg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"C8KAXId0X\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" small\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eVoWO\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robot\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"SysZD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" named\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rS8Me\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"yEhuhD0uo\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"HZ4wms\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"1VOK5kDedg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j1aEuFkvN\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xlfZsW\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6OpZfn5\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" not\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Ny8Opqu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" like\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"II8CLL\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UKcB2W4\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" towering\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"dd\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"40DlCSkZSu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sophisticated\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"RjOSw8b4qC9Rk\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robots\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j4hq\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" that\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"fQpCvG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" handled\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"sxE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" daily\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rcBMZ\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" tasks\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ExR3b\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\";\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UXX8Vxhpn3\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" he\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"jOQYzkBE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"bTtFJWp\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"length\"}],\"usage\":null,\"obfuscation\":\"h8n\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[],\"usage\":{\"prompt_tokens\":16,\"completion_tokens\":35,\"total_tokens\":51,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"J8fQjeaPos\"}\n\ndata: [DONE]\n\n" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json new file mode 100644 index 00000000000..3db865a00a4 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_before_model_validation_openai_chat_completions_post_eb8df755.json @@ -0,0 +1,63 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "112", + "x-datadog-trace-id": "11307714882692163746", + "x-datadog-parent-id": "2798732504086163091", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc1100000000", + "traceparent": "00-6a47cc11000000009ced12c66443a8a2-26d7186257652e93-01", + "tracestate": "dd=p:26d7186257652e93;s:2;t.dm:-4;t.tid:6a47cc1100000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"What is the weather like today?\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":35}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:49:58 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3111ac3ae85-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1215", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9263056278434d2696c121eb01e0a54f", + "set-cookie": "__cf_bm=Lg_OtVyDALy0TiXdy_TD_RwWwe69PHvFgttVvQIfG3M-1783090194.0993586-1.0.1.1-KoSSM0CzYR9uaV2VpejfgbplM9KWTnym3Z2bra1WakC0HAaN3YJpAZQT8DX__hhXuwI3ce1Va7ppxNf4OEeBgpJLoHRzP_4oHta5ndbd7L.IAwnCIz2Ba0jRF2cCLS.5; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:19:58 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-DxZVFdYaeYYcW0e9vLEFWDoRazh3p\",\n \"object\": \"chat.completion\",\n \"created\": 1783090197,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"I'm unable to provide real-time information, including current weather updates. I recommend checking a reliable weather website or app for the most accurate and up-to-date information about today's weather in\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"length\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 14,\n \"completion_tokens\": 35,\n \"total_tokens\": 49,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_b4b9deb02a\"\n}\n" + } +} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json new file mode 100644 index 00000000000..8427c9e7cf7 --- /dev/null +++ b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_tool_call_validation_openai_chat_completions_post_f2dffea2.json @@ -0,0 +1,63 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "OpenAI/Python 2.0.0", + "X-Stainless-Lang": "python", + "X-Stainless-Package-Version": "2.0.0", + "X-Stainless-OS": "Linux", + "X-Stainless-Arch": "x64", + "X-Stainless-Runtime": "CPython", + "X-Stainless-Runtime-Version": "3.11.15", + "X-Stainless-Async": "false", + "x-stainless-retry-count": "0", + "x-stainless-read-timeout": "600", + "Content-Length": "535", + "x-datadog-trace-id": "11679812058723169118", + "x-datadog-parent-id": "17144347262350322105", + "x-datadog-sampling-priority": "2", + "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc2300000000", + "traceparent": "00-6a47cc2300000000a217072f6384b35e-edecf5001e54e9b9-01", + "tracestate": "dd=p:edecf5001e54e9b9;s:2;t.dm:-4;t.tid:6a47cc2300000000" + }, + "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"Bob is a student at Stanford University. He is studying computer science.\"}],\"model\":\"gpt-4o-mini\",\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"extract_student_info\",\"description\":\"Get the student information from the body of the input text\",\"parameters\":{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\",\"description\":\"Name of the person\"},\"major\":{\"type\":\"string\",\"description\":\"Major subject.\"},\"school\":{\"type\":\"string\",\"description\":\"The university name.\"}}}}}]}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Fri, 03 Jul 2026 14:50:12 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a156b3815dbe0142-MAD", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "690", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999980", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9b5983560f2c46209300bbc7dc7d1398", + "set-cookie": "__cf_bm=lFOi_XfqoatTdp8UUQviTl2ntLZoPNYZNNUPOmYoIS4-1783090212.0518818-1.0.1.1-8LZKca4JkZ_Z6AmofdkXp3ls.q2mUD2uefWtfGYruqEynmongOkfGw5EZnfsFEASpchoGpnXbCKtqo0tXp49uTvHNVExyvAYfGy1SUoqiGwyC7GwLjkxauXRvOdNrwjZ; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:20:12 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-DxZVUSEzvfuZVXYVyOwL7CSxKngPg\",\n \"object\": \"chat.completion\",\n \"created\": 1783090212,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_C5dXIRBkTQjYoQrBr3WrkQYY\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"extract_student_info\",\n \"arguments\": \"{\\\"name\\\":\\\"Bob\\\",\\\"major\\\":\\\"computer science\\\",\\\"school\\\":\\\"Stanford University\\\"}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 85,\n \"completion_tokens\": 26,\n \"total_tokens\": 111,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_8d97543466\"\n}\n" + } +} \ No newline at end of file From 824409e2ae605e451a0f200b1f5dfc1358a01b9d Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Mon, 13 Jul 2026 11:21:42 +0200 Subject: [PATCH 04/11] Apply suggestion from @avara1986 --- tests/integration_frameworks/llm/openai/test_openai_ai_guard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index fc99a904e87..51824ba7037 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -90,7 +90,7 @@ def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: Fra ), ) - guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=2) assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): From 8ddfd08507cb5f583d124bd99731ece637da281e Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 15 Jul 2026 13:31:34 +0200 Subject: [PATCH 05/11] fix codestyle --- tests/integration_frameworks/llm/openai/test_openai_ai_guard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 51824ba7037..0f4762813a9 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -90,7 +90,7 @@ def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: Fra ), ) - guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=2) + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=2)) assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): From 155c452451e91f607d120822c308ddaffef888cb Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 15 Jul 2026 15:09:21 +0200 Subject: [PATCH 06/11] fix test --- tests/integration_frameworks/llm/openai/test_openai_ai_guard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 0f4762813a9..fc99a904e87 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -90,7 +90,7 @@ def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: Fra ), ) - guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=2)) + guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): From d95adff1f04234d1ea2f2cb22f6df8c5f45d44b8 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 15 Jul 2026 15:44:43 +0200 Subject: [PATCH 07/11] fix test --- .../llm/openai/test_openai_ai_guard.py | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index fc99a904e87..70ae39d5881 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -21,8 +21,10 @@ """ import os +import time import pytest +import requests from utils import features, scenarios from utils.docker_fixtures import FrameworkTestClientApi, TestAgentAPI @@ -53,6 +55,35 @@ def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: return [span for trace in traces for span in trace if span.get("resource") == "ai_guard"] +def _wait_for_ai_guard_spans( + test_agent: TestAgentAPI, *, target: str | None = None, wait_loops: int = 30 +) -> list[dict]: + """Poll the test agent until at least one matching ``ai_guard`` span is received. + + We assert on the presence of the ``ai_guard`` span rather than on a fixed number of + traces: the tracer does not deterministically group the ``ai_guard`` span with the + OpenAI span. In particular the streamed after-model evaluation may emit the + ``ai_guard`` span either nested in the OpenAI trace (1 trace) or as its own trace + (2 traces), so ``wait_for_num_traces`` with a hard-coded count is inherently racy. + When ``target`` is given, only spans whose ``ai_guard.target`` matches are considered + (so we keep polling until the specific evaluation point we care about has arrived). + """ + spans: list[dict] = [] + for _ in range(wait_loops): + try: + traces = test_agent.traces(clear=False) + except requests.exceptions.RequestException: + pass + else: + spans = _ai_guard_spans(traces) + if target is not None: + spans = [span for span in spans if span["meta"].get("ai_guard.target") == target] + if spans: + return spans + time.sleep(0.1) + return spans + + @features.ai_guard @scenarios.integration_frameworks class TestOpenAiAiGuard(BaseOpenaiTest): @@ -71,11 +102,8 @@ def test_before_model_validation(self, test_agent: TestAgentAPI, test_client: Fr ), ) - guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) - assert guard_spans, "expected an ai_guard span from the before-model evaluation" - assert any(span["meta"].get("ai_guard.target") == "prompt" for span in guard_spans), ( - "expected a before-model ai_guard span with target 'prompt'" - ) + guard_spans = _wait_for_ai_guard_spans(test_agent, target="prompt") + assert guard_spans, "expected a before-model ai_guard span with target 'prompt'" def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): """The streamed model response is evaluated by AI Guard after the model returns.""" @@ -90,7 +118,7 @@ def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: Fra ), ) - guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) + guard_spans = _wait_for_ai_guard_spans(test_agent) assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): @@ -111,8 +139,5 @@ def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: Frame ), ) - guard_spans = _ai_guard_spans(test_agent.wait_for_num_traces(num=1)) - assert guard_spans, "expected an ai_guard span from the tool-call evaluation" - assert any(span["meta"].get("ai_guard.target") == "tool" for span in guard_spans), ( - "expected a tool-call ai_guard span with target 'tool'" - ) + guard_spans = _wait_for_ai_guard_spans(test_agent, target="tool") + assert guard_spans, "expected a tool-call ai_guard span with target 'tool'" From 0656125dd8d9d903c0a42485ecbefd698205061a Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Thu, 16 Jul 2026 20:42:56 +0200 Subject: [PATCH 08/11] update tests --- tests/ai_guard/test_ai_guard_sdk.py | 5 ++ .../llm/openai/test_openai_ai_guard.py | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/tests/ai_guard/test_ai_guard_sdk.py b/tests/ai_guard/test_ai_guard_sdk.py index 51ae84d93d9..f19cfbf9843 100644 --- a/tests/ai_guard/test_ai_guard_sdk.py +++ b/tests/ai_guard/test_ai_guard_sdk.py @@ -652,6 +652,11 @@ def test_standalone_keeps_ai_guard_trace(self): assert root_span.get("meta", {}).get("_dd.p.dm") == "-" + str(SamplingMechanism.AI_GUARD), ( "Decision maker (_dd.p.dm) must match AI_GUARD sampling mechanism in standalone mode" ) + # ai_guard.event:true is the tracer-emitted root-span marker that AI Guard ran on this + # trace. (The _dd.ai_guard.enabled facet seen in the UI is not present in raw payloads.) + assert root_span.get("meta", {}).get("ai_guard.event", False) in (True, "true"), ( + "Root span must be tagged ai_guard.event:true for AI Guard traces in standalone mode" + ) @rfc("https://datadoghq.atlassian.net/wiki/x/54JqiQE") diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 70ae39d5881..35fc1b7f409 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -55,6 +55,25 @@ def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: return [span for trace in traces for span in trace if span.get("resource") == "ai_guard"] +def _ai_guard_event_root_spans(traces: list[list[dict]]) -> list[dict]: + """Local root (service-entry) spans tagged ``ai_guard.event:true``. + + When AI Guard evaluates a call it tags the trace's local root span with + ``ai_guard.event:true`` (dd-trace-py ``appsec/ai_guard/_api_client.py``). This is a + tracer-emitted marker that AI Guard ran on the trace, and is what we assert on here. + + Note: the ``_dd.ai_guard.enabled:1`` facet that is searchable in the Datadog UI is NOT + present in the raw payloads captured by the test agent (it is not emitted by the tracer; + it is produced somewhere in intake), so it cannot be asserted on directly. + """ + return [ + span + for trace in traces + for span in trace + if span.get("parent_id") in (0, None) and span.get("meta", {}).get("ai_guard.event", False) in (True, "true") + ] + + def _wait_for_ai_guard_spans( test_agent: TestAgentAPI, *, target: str | None = None, wait_loops: int = 30 ) -> list[dict]: @@ -84,6 +103,26 @@ def _wait_for_ai_guard_spans( return spans +def _wait_for_ai_guard_event_root_spans(test_agent: TestAgentAPI, *, wait_loops: int = 30) -> list[dict]: + """Poll the test agent until at least one root span tagged ``ai_guard.event:true`` arrives. + + Like the ``ai_guard`` span itself, the tagged local root span may land in a later trace + chunk than the evaluation span, so we poll rather than reading a single snapshot. + """ + spans: list[dict] = [] + for _ in range(wait_loops): + try: + traces = test_agent.traces(clear=False) + except requests.exceptions.RequestException: + pass + else: + spans = _ai_guard_event_root_spans(traces) + if spans: + return spans + time.sleep(0.1) + return spans + + @features.ai_guard @scenarios.integration_frameworks class TestOpenAiAiGuard(BaseOpenaiTest): @@ -105,6 +144,9 @@ def test_before_model_validation(self, test_agent: TestAgentAPI, test_client: Fr guard_spans = _wait_for_ai_guard_spans(test_agent, target="prompt") assert guard_spans, "expected a before-model ai_guard span with target 'prompt'" + event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) + assert event_root_spans, "expected a local root span tagged ai_guard.event:true" + def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): """The streamed model response is evaluated by AI Guard after the model returns.""" with test_agent.vcr_context(stream=True): @@ -121,6 +163,9 @@ def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: Fra guard_spans = _wait_for_ai_guard_spans(test_agent) assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" + event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) + assert event_root_spans, "expected a local root span tagged ai_guard.event:true" + def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): """Tool calls produced by the model are evaluated by AI Guard.""" with test_agent.vcr_context(): @@ -141,3 +186,6 @@ def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: Frame guard_spans = _wait_for_ai_guard_spans(test_agent, target="tool") assert guard_spans, "expected a tool-call ai_guard span with target 'tool'" + + event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) + assert event_root_spans, "expected a local root span tagged ai_guard.event:true" From fbb137301ed3659a6cf3f1d5ebb1351cbd9033f4 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 17 Jul 2026 13:56:53 +0200 Subject: [PATCH 09/11] fix(ai_guard): address PoC review findings on the INTEGRATION_FRAMEWORKS AI Guard test - Keep DD_API_KEY/DD_APP_KEY out of library_env (report metadata) by injecting them via the scenario environment, and fail fast when they are missing during cassette generation instead of a silent xfail. - Assert the tool-call payload (assistant tool_calls) actually reached AI Guard. - Require an assistant message for the after-model eval and skip it until the after-model cassette is regenerated (APPSEC-68977). - Align the module docstring with what is actually asserted (no trace linkage). - Drop the redundant, out-of-scope ai_guard.event assertion from the standalone sampling test. APPSEC-68977 Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/ai_guard/test_ai_guard_sdk.py | 5 -- .../llm/openai/test_openai_ai_guard.py | 55 +++++++++++++------ .../_scenarios/integration_frameworks.py | 7 ++- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/tests/ai_guard/test_ai_guard_sdk.py b/tests/ai_guard/test_ai_guard_sdk.py index f19cfbf9843..51ae84d93d9 100644 --- a/tests/ai_guard/test_ai_guard_sdk.py +++ b/tests/ai_guard/test_ai_guard_sdk.py @@ -652,11 +652,6 @@ def test_standalone_keeps_ai_guard_trace(self): assert root_span.get("meta", {}).get("_dd.p.dm") == "-" + str(SamplingMechanism.AI_GUARD), ( "Decision maker (_dd.p.dm) must match AI_GUARD sampling mechanism in standalone mode" ) - # ai_guard.event:true is the tracer-emitted root-span marker that AI Guard ran on this - # trace. (The _dd.ai_guard.enabled facet seen in the UI is not present in raw payloads.) - assert root_span.get("meta", {}).get("ai_guard.event", False) in (True, "true"), ( - "Root span must be tagged ai_guard.event:true for AI Guard traces in standalone mode" - ) @rfc("https://datadoghq.atlassian.net/wiki/x/54JqiQE") diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 35fc1b7f409..5580f627d09 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -15,12 +15,16 @@ the response before running the evaluation); - **tool-call**: tool calls produced by the model are evaluated. -We assert only that the integration wires each evaluation point and emits an ``ai_guard`` -span attached to the OpenAI trace. The evaluation *outcome* (ALLOW / DENY / ABORT) is -already covered by the ``AI_GUARD`` scenario and is intentionally not re-asserted here. +We assert that the integration wires each evaluation point: that it emits an ``ai_guard`` +span for the specific evaluation being exercised (identified by ``ai_guard.target`` and by +the messages captured in ``meta_struct.ai_guard``) and tags the local root span with +``ai_guard.event:true``. We do not assert trace *linkage* to the ``openai.request`` span: +the tracer does not deterministically nest the ``ai_guard`` span in the OpenAI trace (it +may be emitted as its own trace), so a shared ``trace_id`` is not guaranteed. The +evaluation *outcome* (ALLOW / DENY / ABORT) is already covered by the ``AI_GUARD`` scenario +and is intentionally not re-asserted here. """ -import os import time import pytest @@ -33,28 +37,27 @@ @pytest.fixture -def library_env(request: pytest.FixtureRequest) -> dict[str, str]: - env = { +def library_env() -> dict[str, str]: + # The AI Guard client also needs DD_API_KEY / DD_APP_KEY, but those are injected via the + # scenario environment (see IntegrationFrameworksScenario._required_cassette_generation_api_keys) + # rather than here: library_env is copied into the JSON report metadata, so keeping secrets + # out of it prevents real keys from leaking into logs/artifacts during cassette generation. + return { "DD_AI_GUARD_ENABLED": "true", # after-model evaluation of streamed responses is opt-in "DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED": "true", } - # The AI Guard client needs an API key + app key. Real keys are required when recording - # cassettes (the client calls the real AI Guard API); mock keys are fine on replay since - # the VCR proxy matches on the request, not on auth. - if request.config.option.generate_cassettes: - env["DD_API_KEY"] = os.environ["DD_API_KEY"] - env["DD_APP_KEY"] = os.environ["DD_APP_KEY"] - else: - env["DD_API_KEY"] = "mock_api_key" - env["DD_APP_KEY"] = "mock_app_key" - return env def _ai_guard_spans(traces: list[list[dict]]) -> list[dict]: return [span for trace in traces for span in trace if span.get("resource") == "ai_guard"] +def _guard_messages(span: dict) -> list[dict]: + """The messages AI Guard evaluated, as captured in ``meta_struct.ai_guard.messages``.""" + return span.get("meta_struct", {}).get("ai_guard", {}).get("messages", []) + + def _ai_guard_event_root_spans(traces: list[list[dict]]) -> list[dict]: """Local root (service-entry) spans tagged ``ai_guard.event:true``. @@ -147,6 +150,11 @@ def test_before_model_validation(self, test_agent: TestAgentAPI, test_client: Fr event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) assert event_root_spans, "expected a local root span tagged ai_guard.event:true" + @pytest.mark.skip( + reason="After-model streamed cassette records only the prompt eval - the reconstructed " + "assistant response was never recorded, so the after-model path cannot be replayed yet. " + "Regenerate the aiguard cassette with an assistant message before enabling (APPSEC-68977)." + ) def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): """The streamed model response is evaluated by AI Guard after the model returns.""" with test_agent.vcr_context(stream=True): @@ -160,8 +168,13 @@ def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: Fra ), ) + # An unfiltered wait would be satisfied by the before-model ``prompt`` span that every + # AI-Guard-enabled call emits, masking a regression in the after-model hook. Require a + # span whose evaluated messages include the assistant response to prove it actually ran. guard_spans = _wait_for_ai_guard_spans(test_agent) - assert guard_spans, "expected an ai_guard span from the after-model (streamed) evaluation" + assert any( + msg.get("role") == "assistant" for span in guard_spans for msg in _guard_messages(span) + ), "expected an after-model ai_guard span whose messages include the assistant response" event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) assert event_root_spans, "expected a local root span tagged ai_guard.event:true" @@ -186,6 +199,14 @@ def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: Frame guard_spans = _wait_for_ai_guard_spans(test_agent, target="tool") assert guard_spans, "expected a tool-call ai_guard span with target 'tool'" + # ``target == "tool"`` alone can also come from an ordinary after-model eval of an + # assistant response, so require the assistant tool_calls entry to actually be in the + # payload sent to AI Guard - that is what proves the tool-call path was forwarded. + assert any( + msg.get("role") == "assistant" and msg.get("tool_calls") + for span in guard_spans + for msg in _guard_messages(span) + ), "expected the assistant tool_calls entry in the ai_guard evaluation payload" event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) assert event_root_spans, "expected a local root span tagged ai_guard.event:true" diff --git a/utils/_context/_scenarios/integration_frameworks.py b/utils/_context/_scenarios/integration_frameworks.py index 6783a6b672e..1247456a05b 100644 --- a/utils/_context/_scenarios/integration_frameworks.py +++ b/utils/_context/_scenarios/integration_frameworks.py @@ -21,7 +21,12 @@ class IntegrationFrameworksScenario(DockerFixturesScenario): _test_client_factory: FrameworkTestClientFactory _required_cassette_generation_api_keys: dict[str, list[str]] = { - "openai": ["OPENAI_API_KEY"], + # DD_API_KEY / DD_APP_KEY are needed by the AI Guard client, which calls the real + # AI Guard API while recording. Declaring them here means they are injected into the + # container via the scenario environment (never through ``library_env``, which is + # serialized into the JSON report) and that a missing key fails fast at configure() + # time instead of surfacing as a silent xfail during cassette generation. + "openai": ["OPENAI_API_KEY", "DD_API_KEY", "DD_APP_KEY"], "anthropic": ["ANTHROPIC_API_KEY"], "google_genai": ["GEMINI_API_KEY"], } From 3e817dfb0c947ab4a029abe7c3904c1c228553fe Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 17 Jul 2026 14:09:47 +0200 Subject: [PATCH 10/11] style: apply ruff format to test_openai_ai_guard.py Co-Authored-By: Claude Opus 4.8 (1M context) --- .../llm/openai/test_openai_ai_guard.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 5580f627d09..834f38cc781 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -172,9 +172,9 @@ def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: Fra # AI-Guard-enabled call emits, masking a regression in the after-model hook. Require a # span whose evaluated messages include the assistant response to prove it actually ran. guard_spans = _wait_for_ai_guard_spans(test_agent) - assert any( - msg.get("role") == "assistant" for span in guard_spans for msg in _guard_messages(span) - ), "expected an after-model ai_guard span whose messages include the assistant response" + assert any(msg.get("role") == "assistant" for span in guard_spans for msg in _guard_messages(span)), ( + "expected an after-model ai_guard span whose messages include the assistant response" + ) event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) assert event_root_spans, "expected a local root span tagged ai_guard.event:true" From bc915c933cb8a93c6bfeb985a0a09571ddccd4c8 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Fri, 17 Jul 2026 14:22:44 +0200 Subject: [PATCH 11/11] test(ai_guard): drop after-model streaming test to fix Finding 1 Streamed after-model evaluation (DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED) is not implemented across the other tracer libraries yet, so the after-model test could never replay at cross-language parity and was a false positive (it passed on the before-model prompt span). Remove the test, the stream env var, and the two orphaned streaming cassettes; keep before-model and tool-call coverage. APPSEC-68977 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../llm/openai/test_openai_ai_guard.py | 49 +++------------ ...m_True_aiguard_evaluate_post_5dec5d6a.json | 38 ------------ ...openai_chat_completions_post_5c14b241.json | 62 ------------------- 3 files changed, 10 insertions(+), 139 deletions(-) delete mode 100644 tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json delete mode 100644 tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json diff --git a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py index 834f38cc781..4c8aba64122 100644 --- a/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py +++ b/tests/integration_frameworks/llm/openai/test_openai_ai_guard.py @@ -10,11 +10,13 @@ Guard evaluates the call at three points with no manual ``evaluate()`` call: - **before-model**: the request/prompt is evaluated before the model is called; -- **after-model**: the model response is evaluated (streamed responses require - ``DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED=true``, which buffers and reconstructs - the response before running the evaluation); - **tool-call**: tool calls produced by the model are evaluated. +The after-model evaluation is intentionally not covered here: exercising it end-to-end needs +the streamed-response path (``DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED``), which is not +implemented across the other tracer libraries yet, so we keep this suite at cross-language +parity. + We assert that the integration wires each evaluation point: that it emits an ``ai_guard`` span for the specific evaluation being exercised (identified by ``ai_guard.target`` and by the messages captured in ``meta_struct.ai_guard``) and tags the local root span with @@ -44,8 +46,6 @@ def library_env() -> dict[str, str]: # out of it prevents real keys from leaking into logs/artifacts during cassette generation. return { "DD_AI_GUARD_ENABLED": "true", - # after-model evaluation of streamed responses is opt-in - "DD_AI_GUARD_ANALYZE_STREAM_RESPONSES_ENABLED": "true", } @@ -84,11 +84,11 @@ def _wait_for_ai_guard_spans( We assert on the presence of the ``ai_guard`` span rather than on a fixed number of traces: the tracer does not deterministically group the ``ai_guard`` span with the - OpenAI span. In particular the streamed after-model evaluation may emit the - ``ai_guard`` span either nested in the OpenAI trace (1 trace) or as its own trace - (2 traces), so ``wait_for_num_traces`` with a hard-coded count is inherently racy. - When ``target`` is given, only spans whose ``ai_guard.target`` matches are considered - (so we keep polling until the specific evaluation point we care about has arrived). + OpenAI span. The ``ai_guard`` span may be emitted either nested in the OpenAI trace + (1 trace) or as its own trace (2 traces), so ``wait_for_num_traces`` with a hard-coded + count is inherently racy. When ``target`` is given, only spans whose ``ai_guard.target`` + matches are considered (so we keep polling until the specific evaluation point we care + about has arrived). """ spans: list[dict] = [] for _ in range(wait_loops): @@ -150,35 +150,6 @@ def test_before_model_validation(self, test_agent: TestAgentAPI, test_client: Fr event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) assert event_root_spans, "expected a local root span tagged ai_guard.event:true" - @pytest.mark.skip( - reason="After-model streamed cassette records only the prompt eval - the reconstructed " - "assistant response was never recorded, so the after-model path cannot be replayed yet. " - "Regenerate the aiguard cassette with an assistant message before enabling (APPSEC-68977)." - ) - def test_after_model_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): - """The streamed model response is evaluated by AI Guard after the model returns.""" - with test_agent.vcr_context(stream=True): - test_client.request( - "POST", - "/chat/completions", - dict( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "Tell me a short story about a robot."}], - parameters=dict(max_tokens=35, stream=True), - ), - ) - - # An unfiltered wait would be satisfied by the before-model ``prompt`` span that every - # AI-Guard-enabled call emits, masking a regression in the after-model hook. Require a - # span whose evaluated messages include the assistant response to prove it actually ran. - guard_spans = _wait_for_ai_guard_spans(test_agent) - assert any(msg.get("role") == "assistant" for span in guard_spans for msg in _guard_messages(span)), ( - "expected an after-model ai_guard span whose messages include the assistant response" - ) - - event_root_spans = _wait_for_ai_guard_event_root_spans(test_agent) - assert event_root_spans, "expected a local root span tagged ai_guard.event:true" - def test_tool_call_validation(self, test_agent: TestAgentAPI, test_client: FrameworkTestClientApi): """Tool calls produced by the model are evaluated by AI Guard.""" with test_agent.vcr_context(): diff --git a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json b/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json deleted file mode 100644 index 13bf51778c5..00000000000 --- a/tests/integration_frameworks/utils/vcr-cassettes/aiguard/test_after_model_validation_stream_True_aiguard_evaluate_post_5dec5d6a.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "request": { - "method": "POST", - "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", - "headers": { - "Accept-Encoding": "identity", - "Content-Length": "153", - "Content-Type": "application/json", - "DD-AI-GUARD-VERSION": "4.10.6", - "DD-AI-GUARD-SOURCE": "SDK", - "DD-AI-GUARD-LANGUAGE": "python", - "Datadog-Entity-ID": "in-115544" - }, - "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Tell me a short story about a robot.\"}], \"meta\": {\"service\": \"openai\", \"env\": null}}}}" - }, - "response": { - "status": { - "code": 200, - "message": "OK" - }, - "headers": { - "content-security-policy": "frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com", - "content-type": "application/vnd.api+json", - "vary": "Accept-Encoding", - "x-frame-options": "SAMEORIGIN", - "content-length": "222", - "date": "Fri, 03 Jul 2026 14:50:04 GMT", - "x-content-type-options": "nosniff", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "x-ratelimit-limit": "5000", - "x-ratelimit-period": "60", - "x-ratelimit-remaining": "4999", - "x-ratelimit-reset": "56", - "x-ratelimit-name": "ai_guard_evaluate_per_org" - }, - "body": "{\"data\":{\"id\":\"ee239b30-9d6e-4ac1-b8b5-ebbc5d96c91d\",\"type\":\"evaluations\",\"attributes\":{\"action\":\"ALLOW\",\"global_prob\":0.00159454345703125,\"is_blocking_enabled\":false,\"reason\":\"No rule match.\",\"tag_probs\":null,\"tags\":[]}}}" - } -} \ No newline at end of file diff --git a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json b/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json deleted file mode 100644 index 7bd1c1f13db..00000000000 --- a/tests/integration_frameworks/utils/vcr-cassettes/openai/test_after_model_validation_stream_True_openai_chat_completions_post_5c14b241.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "request": { - "method": "POST", - "url": "https://api.openai.com/v1/chat/completions", - "headers": { - "Accept-Encoding": "gzip, deflate", - "Connection": "keep-alive", - "Accept": "application/json", - "Content-Type": "application/json", - "User-Agent": "OpenAI/Python 2.0.0", - "X-Stainless-Lang": "python", - "X-Stainless-Package-Version": "2.0.0", - "X-Stainless-OS": "Linux", - "X-Stainless-Arch": "x64", - "X-Stainless-Runtime": "CPython", - "X-Stainless-Runtime-Version": "3.11.15", - "X-Stainless-Async": "false", - "x-stainless-retry-count": "0", - "x-stainless-read-timeout": "600", - "Content-Length": "171", - "x-datadog-trace-id": "12575373332985498166", - "x-datadog-parent-id": "4288392994079136126", - "x-datadog-sampling-priority": "2", - "x-datadog-tags": "_dd.p.dm=-4,_dd.p.tid=6a47cc1c00000000", - "traceparent": "00-6a47cc1c00000000ae84b36b4e854a36-3b836eb50d5c597e-01", - "tracestate": "dd=p:3b836eb50d5c597e;s:2;t.dm:-4;t.tid:6a47cc1c00000000" - }, - "body": "{\"messages\":[{\"role\":\"user\",\"content\":\"Tell me a short story about a robot.\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":35,\"stream\":true,\"stream_options\":{\"include_usage\":true}}" - }, - "response": { - "status": { - "code": 200, - "message": "OK" - }, - "headers": { - "Date": "Fri, 03 Jul 2026 14:50:05 GMT", - "Content-Type": "text/event-stream; charset=utf-8", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "CF-Ray": "a156b3553e710284-MAD", - "CF-Cache-Status": "DYNAMIC", - "Server": "cloudflare", - "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", - "X-Content-Type-Options": "nosniff", - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "openai-processing-ms": "243", - "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", - "openai-version": "2020-10-01", - "x-openai-proxy-wasm": "v0.1", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "150000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999990", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_30bd6e2837694e1abe7c71e7bfff16e1", - "set-cookie": "__cf_bm=Nfpb5Ib0QbV67YYDeOlS6T8D2.SIupabQ_RGl9iS82Y-1783090204.998873-1.0.1.1-LBFP99yI1awBQD1I3zpk3KOKCcfo3Dq2MOgJQs8Jm7Lwu1UivkU.0vZvTqsRHHt77mcU_Lw9eLo50vYY6aoRbjeuwDPRikB46h7zZ.nP1DErdbO.1PafaU0aBZelqjO9; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Jul 2026 15:20:05 GMT", - "alt-svc": "h3=\":443\"; ma=86400" - }, - "body": "data: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xyfxvNW9y\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Once\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"972kFcG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"n7UHCOY0\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"s2tZKJhmu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" bustling\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"un\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" city\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"THufE2\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" of\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"VOR4gL7e\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"YxFDNUv\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" future\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"lQTD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"JYCvZIfp28\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" there\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"4HPFe\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" lived\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"f3Umg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"C8KAXId0X\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" small\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"eVoWO\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robot\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"SysZD\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" named\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rS8Me\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"yEhuhD0uo\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"HZ4wms\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"1VOK5kDedg\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" T\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j1aEuFkvN\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"inker\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"xlfZsW\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"6OpZfn5\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" not\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"Ny8Opqu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" like\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"II8CLL\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" the\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UKcB2W4\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" towering\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"dd\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"40DlCSkZSu\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sophisticated\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"RjOSw8b4qC9Rk\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" robots\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"j4hq\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" that\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"fQpCvG\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" handled\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"sxE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" daily\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"rcBMZ\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" tasks\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"ExR3b\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\";\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"UXX8Vxhpn3\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" he\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"jOQYzkBE\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" was\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null,\"obfuscation\":\"bTtFJWp\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"length\"}],\"usage\":null,\"obfuscation\":\"h8n\"}\n\ndata: {\"id\":\"chatcmpl-DxZVNdNI9ESNgAmCG4vLAbyBTeLzI\",\"object\":\"chat.completion.chunk\",\"created\":1783090205,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_b4b9deb02a\",\"choices\":[],\"usage\":{\"prompt_tokens\":16,\"completion_tokens\":35,\"total_tokens\":51,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},\"obfuscation\":\"J8fQjeaPos\"}\n\ndata: [DONE]\n\n" - } -} \ No newline at end of file