From a238901cee1168ded1acac2cf4c360d31f3584c6 Mon Sep 17 00:00:00 2001 From: ahmadalguydi Date: Tue, 28 Jul 2026 18:30:33 +0300 Subject: [PATCH 1/4] test(openai): enforce experimental embedding metric conformance Run embedding conformance with the latest semantic conventions and assert both emitted metrics carry gen_ai.provider.name. Fixes #38 --- .../.changelog/38.fixed | 1 + .../tests/conformance/embedding.py | 3 ++ .../tests/test_embedding_invocation_unit.py | 34 +++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/38.fixed diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/38.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/38.fixed new file mode 100644 index 000000000..e13137cab --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/38.fixed @@ -0,0 +1 @@ +Ensure OpenAI embedding conformance runs in experimental semantic-convention mode and guards `gen_ai.provider.name` on both emitted metrics. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py index c899cfd4d..78f5e1b93 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py @@ -38,6 +38,9 @@ def run( logger_provider=logger_provider, meter_provider=meter_provider, content_capture="SPAN_ONLY", + extra_env={ + "OTEL_SEMCONV_STABILITY_OPT_IN": "gen_ai_latest_experimental" + }, ): with vcr.use_cassette("embedding_conformance.yaml"): OpenAI().embeddings.create( diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py index 6afba67f6..c6656cc7e 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py @@ -140,6 +140,40 @@ def test_dimensions_omitted_when_not_provided(handler): invocation.stop() +def test_embedding_metrics_use_provider_name_attribute(handler, metric_reader): + """Experimental metrics must use the current provider attribute.""" + invocation = create_embedding_invocation( + handler, {"model": "text-embedding-3-small"}, _make_client() + ) + invocation.input_tokens = 7 + invocation.stop() + + metrics = metric_reader.get_metrics_data() + metric_names = { + metric.name + for resource_metric in metrics.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + } + assert { + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + } <= metric_names + + for resource_metric in metrics.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + if metric.name in { + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + }: + for point in metric.data.data_points: + assert ( + point.attributes["gen_ai.provider.name"] + == "openai" + ) + + def test_encoding_format_mapped_to_invocation(handler): invocation = create_embedding_invocation( handler, From c220d9ddd4b3736cd3b822c0f0e0eaee0d3cbe68 Mon Sep 17 00:00:00 2001 From: ahmadalguydi Date: Wed, 29 Jul 2026 07:06:35 +0300 Subject: [PATCH 2/4] fix(test): use repository semantic convention defaults Remove the scenario-local stability opt-in so the conformance test follows the repository-wide experimental semantic convention default. --- .../tests/conformance/embedding.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py index 78f5e1b93..c899cfd4d 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/embedding.py @@ -38,9 +38,6 @@ def run( logger_provider=logger_provider, meter_provider=meter_provider, content_capture="SPAN_ONLY", - extra_env={ - "OTEL_SEMCONV_STABILITY_OPT_IN": "gen_ai_latest_experimental" - }, ): with vcr.use_cassette("embedding_conformance.yaml"): OpenAI().embeddings.create( From 8895d2f262802a787d14b5d45c443480d8b591a8 Mon Sep 17 00:00:00 2001 From: ahmadalguydi Date: Wed, 29 Jul 2026 07:23:26 +0300 Subject: [PATCH 3/4] test(openai): strengthen embedding metric assertions Move the provider-name regression into the existing embedding integration test. Check both expected metrics explicitly, require data points, and reject the deprecated provider attribute. --- .../tests/test_embedding_invocation_unit.py | 34 ------------------- .../tests/test_embeddings.py | 9 +++++ 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py index c6656cc7e..6afba67f6 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embedding_invocation_unit.py @@ -140,40 +140,6 @@ def test_dimensions_omitted_when_not_provided(handler): invocation.stop() -def test_embedding_metrics_use_provider_name_attribute(handler, metric_reader): - """Experimental metrics must use the current provider attribute.""" - invocation = create_embedding_invocation( - handler, {"model": "text-embedding-3-small"}, _make_client() - ) - invocation.input_tokens = 7 - invocation.stop() - - metrics = metric_reader.get_metrics_data() - metric_names = { - metric.name - for resource_metric in metrics.resource_metrics - for scope_metric in resource_metric.scope_metrics - for metric in scope_metric.metrics - } - assert { - "gen_ai.client.operation.duration", - "gen_ai.client.token.usage", - } <= metric_names - - for resource_metric in metrics.resource_metrics: - for scope_metric in resource_metric.scope_metrics: - for metric in scope_metric.metrics: - if metric.name in { - "gen_ai.client.operation.duration", - "gen_ai.client.token.usage", - }: - for point in metric.data.data_points: - assert ( - point.attributes["gen_ai.provider.name"] - == "openai" - ) - - def test_encoding_format_mapped_to_invocation(handler): invocation = create_embedding_invocation( handler, diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py index 9bd87679c..44afa6a18 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py @@ -377,6 +377,15 @@ def test_embeddings_token_metrics( ) assert token_metric is not None + for metric in (duration_metric, token_metric): + assert metric.data.data_points, f"{metric.name} metric has no data points" + for point in metric.data.data_points: + assert ( + point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] + == GenAIAttributes.GenAiProviderNameValues.OPENAI.value + ) + assert GenAIAttributes.GEN_AI_SYSTEM not in point.attributes + # Find the input token data point input_token_point = None for point in token_metric.data.data_points: From 488c73510d4db6e10a973e5954e78d65c5439f7e Mon Sep 17 00:00:00 2001 From: ahmadalguydi Date: Thu, 30 Jul 2026 12:39:37 +0300 Subject: [PATCH 4/4] test(openai): remove redundant embedding metric assertions --- .../tests/test_embeddings.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py index 44afa6a18..9bd87679c 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_embeddings.py @@ -377,15 +377,6 @@ def test_embeddings_token_metrics( ) assert token_metric is not None - for metric in (duration_metric, token_metric): - assert metric.data.data_points, f"{metric.name} metric has no data points" - for point in metric.data.data_points: - assert ( - point.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] - == GenAIAttributes.GenAiProviderNameValues.OPENAI.value - ) - assert GenAIAttributes.GEN_AI_SYSTEM not in point.attributes - # Find the input token data point input_token_point = None for point in token_metric.data.data_points: