diff --git a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java index 7cef3e02924..16be2e84b98 100644 --- a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java +++ b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java @@ -185,10 +185,11 @@ private void doDiscovery(State newState) { if (log.isDebugEnabled()) { log.debug( - "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, logEndpoint={}, snapshotEndpoint={}, diagnosticsEndpoint={}, evpProxyEndpoint={}, telemetryProxyEndpoint={}", + "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsClientSideStats={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, logEndpoint={}, snapshotEndpoint={}, diagnosticsEndpoint={}, evpProxyEndpoint={}, telemetryProxyEndpoint={}", newState.traceEndpoint, newState.metricsEndpoint, newState.supportsDropping, + newState.supportsClientSideStats, newState.supportsLongRunning, newState.dataStreamsEndpoint, newState.configEndpoint, diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java index 7b9aae5d76a..90fd9fd046f 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java @@ -31,7 +31,7 @@ public final class OtlpConfig { public static final String OTLP_METRICS_TEMPORALITY_PREFERENCE = "otlp.metrics.temporality.preference"; - public static final String TRACES_SPAN_METRICS_ENABLED = "traces.span.metrics.enabled"; + public static final String OTEL_TRACES_SPAN_METRICS_ENABLED = "otel.traces.span.metrics.enabled"; public static final String TRACE_OTEL_ENABLED = "trace.otel.enabled"; public static final String TRACE_OTEL_EXPORTER = "trace.otel.exporter"; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ClientStatsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ClientStatsAggregator.java index 3a3ded31ff8..805f5c89ee6 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ClientStatsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ClientStatsAggregator.java @@ -4,6 +4,7 @@ import static datadog.trace.api.DDSpanTypes.RPC; import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_ENDPOINT; import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_METHOD; +import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_ROUTE; import static datadog.trace.common.metrics.AggregateEntry.ERROR_TAG; import static datadog.trace.common.metrics.AggregateEntry.TOP_LEVEL_TAG; import static datadog.trace.common.metrics.SignalItem.ClearSignal.CLEAR; @@ -12,6 +13,7 @@ import static datadog.trace.util.AgentThreadFactory.AgentThread.METRICS_AGGREGATOR; import static datadog.trace.util.AgentThreadFactory.THREAD_JOIN_TIMOUT_MS; import static datadog.trace.util.AgentThreadFactory.newAgentThread; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import datadog.common.queue.Queues; @@ -26,6 +28,7 @@ import datadog.trace.core.DDTraceCoreInfo; import datadog.trace.core.SpanKindFilter; import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.otlp.metrics.OtlpStatsMetricWriter; import datadog.trace.util.AgentTaskScheduler; import java.util.Collections; import java.util.List; @@ -62,6 +65,15 @@ public final class ClientStatsAggregator implements MetricsAggregator, EventList private static final SpanKindFilter INTERNAL_KIND = SpanKindFilter.builder().includeInternal().build(); + // gRPC status-code source tags, probed in priority order on the OTLP export path + private static final String[] GRPC_STATUS_CODE_KEYS = { + InstrumentationTags.GRPC_STATUS_CODE, // "rpc.grpc.status_code" + "grpc.code", + "rpc.grpc.status.code", + "grpc.status.code", + "rpc.response.status_code", + }; + private final Set ignoredResources; private final Thread thread; private final MessagePassingQueue inbox; @@ -72,6 +84,7 @@ public final class ClientStatsAggregator implements MetricsAggregator, EventList private final DDAgentFeaturesDiscovery features; private final HealthMetrics healthMetrics; private final boolean includeEndpointInMetrics; + private final boolean otlpStatsExportEnabled; /** * Cached peer-aggregation schema read by producer threads. @@ -114,6 +127,30 @@ public ClientStatsAggregator( config.isTraceResourceRenamingEnabled()); } + /** + * OTLP span-metrics export variant. Reuses the same span selection + DDSketch aggregation as the + * native path, but emits via the injected {@link OtlpStatsMetricWriter} instead of msgpack. No + * agent {@link Sink} is needed, so a {@link NoOpSink} satisfies the register()/backpressure + * contract, and the reporting interval comes from {@code trace.stats.interval} (milliseconds). + */ + public ClientStatsAggregator( + Config config, + SharedCommunicationObjects sharedCommunicationObjects, + HealthMetrics healthMetrics, + OtlpStatsMetricWriter metricWriter) { + this( + config.getMetricsIgnoredResources(), + sharedCommunicationObjects.featuresDiscovery(config), + healthMetrics, + NoOpSink.INSTANCE, + metricWriter, + config.getTracerMetricsMaxAggregates(), + config.getTracerMetricsMaxPending(), + config.getTraceStatsInterval(), + MILLISECONDS, + true); + } + ClientStatsAggregator( WellKnownTags wellKnownTags, Set ignoredResources, @@ -173,6 +210,7 @@ public ClientStatsAggregator( boolean includeEndpointInMetrics) { this.ignoredResources = ignoredResources; this.includeEndpointInMetrics = includeEndpointInMetrics; + this.otlpStatsExportEnabled = metricWriter instanceof OtlpStatsMetricWriter; this.inbox = Queues.mpscArrayQueue(queueSize); this.features = features; this.healthMetrics = healthMetric; @@ -191,6 +229,22 @@ public ClientStatsAggregator( this.reportingIntervalTimeUnit = timeUnit; } + // ── visible for testing ───────────────────────────────────────────────────── + // Expose the writer-selection outcome and reporting cadence so tests can assert + // the native-vs-OTLP XOR choice without reflecting into private fields. + + boolean isOtlpStatsExportEnabled() { + return otlpStatsExportEnabled; + } + + long reportingInterval() { + return reportingInterval; + } + + TimeUnit reportingIntervalTimeUnit() { + return reportingIntervalTimeUnit; + } + @Override public void start() { sink.register(this); @@ -206,11 +260,16 @@ public void start() { log.debug("started metrics aggregator"); } + private boolean statsExportEnabled() { + return otlpStatsExportEnabled || features.supportsMetrics(); + } + private boolean isMetricsEnabled() { - if (features.getMetricsEndpoint() == null) { + // The discovery refresh only helps the native path. + if (!otlpStatsExportEnabled && features.getMetricsEndpoint() == null) { features.discoverIfOutdated(); } - return features.supportsMetrics(); + return statsExportEnabled(); } @Override @@ -266,7 +325,7 @@ public Future forceReport() { public boolean publish(List> trace) { boolean forceKeep = false; int counted = 0; - if (features.supportsMetrics()) { + if (statsExportEnabled()) { // Producer-side fast path: one volatile read and use whatever schema is currently cached. // The aggregator thread keeps this schema in sync with feature discovery in // resetCardinalityHandlers(). The only producer-side rebuild is the one-time bootstrap on @@ -315,12 +374,21 @@ private boolean publish(CoreSpan span, boolean isTopLevel, PeerTagSchema peer Object httpMethodObj = span.unsafeGetTag(HTTP_METHOD); httpMethod = httpMethodObj != null ? httpMethodObj.toString() : null; Object httpEndpointObj = span.unsafeGetTag(HTTP_ENDPOINT); + // OTLP path falls back to http.route (mirrors libdatadog). The native v0.6 path keeps its + // http.endpoint-only lookup so this doesn't change its aggregation key / wire output. + if (otlpStatsExportEnabled && httpEndpointObj == null) { + httpEndpointObj = span.unsafeGetTag(HTTP_ROUTE); + } httpEndpoint = httpEndpointObj != null ? httpEndpointObj.toString() : null; } CharSequence spanType = span.getType(); String grpcStatusCode = null; - if (spanType != null && RPC.contentEquals(spanType)) { + if (otlpStatsExportEnabled) { + // OTLP path: probe every known gRPC status-code convention, no span-type gate, so a span + // typed "grpc" (or carrying an OTel-style key) still surfaces rpc.response.status_code. + grpcStatusCode = firstTag(span, GRPC_STATUS_CODE_KEYS); + } else if (spanType != null && RPC.contentEquals(spanType)) { Object grpcStatusObj = span.unsafeGetTag(InstrumentationTags.GRPC_STATUS_CODE); grpcStatusCode = grpcStatusObj != null ? grpcStatusObj.toString() : null; } @@ -367,6 +435,17 @@ private boolean publish(CoreSpan span, boolean isTopLevel, PeerTagSchema peer return error; } + /** Returns the first non-null span tag among {@code keys}, in order, or {@code null} if none. */ + private static String firstTag(CoreSpan span, String[] keys) { + for (String key : keys) { + Object value = span.unsafeGetTag(key); + if (value != null) { + return value.toString(); + } + } + return null; + } + /** * One-time producer-side bootstrap of {@link #cachedPeerTagSchema}. Synchronized double-check * guards against two producers racing on the very first publish; after this returns, {@code diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java index b9530871763..2b91906afe3 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricsAggregatorFactory.java @@ -1,8 +1,12 @@ package datadog.trace.common.metrics; +import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_EXPORTER; +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; + import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.otlp.metrics.OtlpStatsMetricWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,6 +17,33 @@ public static MetricsAggregator createMetricsAggregator( Config config, SharedCommunicationObjects sharedCommunicationObjects, HealthMetrics healthMetrics) { + // OTLP span-metrics export and native msgpack stats are mutually exclusive (XOR): both hang off + // the same ClientStatsAggregator span selection + DDSketch aggregation, differing only in + // the injected MetricWriter. + if (config.isOtelTracesSpanMetricsEnabled()) { + if (config.isTracerMetricsEnabled()) { + log.warn( + "Both OTLP trace span metrics and native tracer metrics are enabled; " + + "using OTLP export and ignoring native tracer metrics (the two are mutually " + + "exclusive)."); + } + if (!config.isMetricsOtlpExporterEnabled()) { + log.warn( + "OTLP trace span metrics are enabled but the OTLP metrics exporter is not " + + "(" + + METRICS_OTEL_EXPORTER + + " is not 'otlp'); span metrics will still be exported over " + + "OTLP using the otlp.metrics.* transport settings. Set " + + METRICS_OTEL_EXPORTER + + "=otlp " + + "to make this explicit, or disable " + + OTEL_TRACES_SPAN_METRICS_ENABLED + + " to suppress them."); + } + log.debug("OTLP trace span metrics enabled"); + return new ClientStatsAggregator( + config, sharedCommunicationObjects, healthMetrics, new OtlpStatsMetricWriter(config)); + } if (config.isTracerMetricsEnabled()) { log.debug("tracer metrics enabled"); return new ClientStatsAggregator(config, sharedCommunicationObjects, healthMetrics); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/NoOpSink.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/NoOpSink.java new file mode 100644 index 00000000000..75236791324 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/NoOpSink.java @@ -0,0 +1,17 @@ +package datadog.trace.common.metrics; + +import java.nio.ByteBuffer; + +/** A {@link Sink} that discards everything. */ +public final class NoOpSink implements Sink { + + public static final NoOpSink INSTANCE = new NoOpSink(); + + private NoOpSink() {} + + @Override + public void accept(int messageCount, ByteBuffer buffer) {} + + @Override + public void register(EventListener listener) {} +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java index 12e6d74b719..d0fbc11cda8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/DDAgentWriter.java @@ -7,6 +7,7 @@ import static datadog.trace.common.writer.ddagent.Prioritization.FAST_LANE; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.DroppingPolicy; import datadog.metrics.api.Monitoring; import datadog.trace.api.Config; import datadog.trace.api.ProtocolVersion; @@ -39,7 +40,7 @@ public static class DDAgentWriterBuilder { int flushIntervalMilliseconds = 1000; Monitoring monitoring = Monitoring.DISABLED; ProtocolVersion protocolVersion = Config.get().getProtocolVersion(); - boolean metricsReportingEnabled = Config.get().isTracerMetricsEnabled(); + boolean nativeMetricsReportingEnabled = Config.get().isTracerMetricsEnabled(); boolean metricsIgnoreAgentVersion = Config.get().isTracerMetricsIgnoreAgentVersion(); private int flushTimeout = 1; private TimeUnit flushTimeoutUnit = TimeUnit.SECONDS; @@ -48,6 +49,7 @@ public static class DDAgentWriterBuilder { private DDAgentApi agentApi; private Prioritization prioritization; private DDAgentFeaturesDiscovery featureDiscovery; + private DroppingPolicy droppingPolicy; private SingleSpanSampler singleSpanSampler; public DDAgentWriterBuilder agentApi(DDAgentApi agentApi) { @@ -110,8 +112,9 @@ public DDAgentWriterBuilder traceAgentProtocolVersion(ProtocolVersion protocolVe return this; } - public DDAgentWriterBuilder metricsReportingEnabled(boolean metricsReportingEnabled) { - this.metricsReportingEnabled = metricsReportingEnabled; + public DDAgentWriterBuilder nativeMetricsReportingEnabled( + boolean nativeMetricsReportingEnabled) { + this.nativeMetricsReportingEnabled = nativeMetricsReportingEnabled; return this; } @@ -125,6 +128,11 @@ public DDAgentWriterBuilder featureDiscovery(DDAgentFeaturesDiscovery featureDis return this; } + public DDAgentWriterBuilder droppingPolicy(DroppingPolicy droppingPolicy) { + this.droppingPolicy = droppingPolicy; + return this; + } + public DDAgentWriterBuilder flushTimeout(int flushTimeout, TimeUnit flushTimeoutUnit) { this.flushTimeout = flushTimeout; this.flushTimeoutUnit = flushTimeoutUnit; @@ -154,12 +162,13 @@ public DDAgentWriter build() { monitoring, agentUrl, protocolVersion, - metricsReportingEnabled, + nativeMetricsReportingEnabled, metricsIgnoreAgentVersion); } if (null == agentApi) { agentApi = - new DDAgentApi(client, agentUrl, featureDiscovery, monitoring, metricsReportingEnabled); + new DDAgentApi( + client, agentUrl, featureDiscovery, monitoring, nativeMetricsReportingEnabled); } final DDAgentMapperDiscovery mapperDiscovery = new DDAgentMapperDiscovery(featureDiscovery); @@ -170,7 +179,8 @@ public DDAgentWriter build() { traceBufferSize, healthMetrics, dispatcher, - featureDiscovery, + // allow custom dropping policy for OTLP; otherwise fall back to feature discovery + droppingPolicy != null ? droppingPolicy : featureDiscovery, null == prioritization ? FAST_LANE : prioritization, flushIntervalMilliseconds, TimeUnit.MILLISECONDS, diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java index 3f38d45881f..23b15c39544 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/WriterFactory.java @@ -193,10 +193,17 @@ public static Writer createWriter( ddAgentApi.addResponseListener((RemoteResponseListener) sampler); } + // Drop p0 (sampled-out) traces when client-side stats are being computed -- either via the + // native agent-stats path (featuresDiscovery) or the OTLP trace metrics path + final boolean otlpSpanMetricsEnabled = config.isOtelTracesSpanMetricsEnabled(); + final DroppingPolicy droppingPolicy = + () -> otlpSpanMetricsEnabled || featuresDiscovery.active(); + DDAgentWriter.DDAgentWriterBuilder builder = DDAgentWriter.builder() .agentApi(ddAgentApi) .featureDiscovery(featuresDiscovery) + .droppingPolicy(droppingPolicy) .prioritization(prioritization) .healthMetrics(healthMetrics) .monitoring(commObjects.monitoring) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java index d65b2f77b80..8907e634cfc 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/DDAgentApi.java @@ -43,7 +43,7 @@ public class DDAgentApi extends RemoteApi { private static final String DATADOG_AGENT_STATE = "Datadog-Agent-State"; private final List responseListeners = new ArrayList<>(); - private final boolean metricsEnabled; + private final boolean nativeMetricsEnabled; private final Recording sendPayloadTimer; private final Counter agentErrorCounter; @@ -67,14 +67,14 @@ public DDAgentApi( HttpUrl agentUrl, DDAgentFeaturesDiscovery featuresDiscovery, Monitoring monitoring, - boolean metricsEnabled) { + boolean nativeMetricsEnabled) { super(false); this.featuresDiscovery = featuresDiscovery; this.agentUrl = agentUrl; this.httpClient = client; this.sendPayloadTimer = monitoring.newTimer("trace.agent.send.time"); this.agentErrorCounter = monitoring.newCounter("trace.agent.error.counter"); - this.metricsEnabled = metricsEnabled; + this.nativeMetricsEnabled = nativeMetricsEnabled; this.headers = new HashMap<>(); this.headers.put(DATADOG_CLIENT_COMPUTED_TOP_LEVEL, "true"); @@ -109,7 +109,8 @@ public Response sendSerializedTraces(final Payload payload) { .addHeader(DATADOG_DROPPED_SPAN_COUNT, Long.toString(payload.droppedSpans())) .addHeader( DATADOG_CLIENT_COMPUTED_STATS, - (metricsEnabled && featuresDiscovery.supportsMetrics()) + Config.get().isOtelTracesSpanMetricsEnabled() + || (nativeMetricsEnabled && featuresDiscovery.supportsMetrics()) // Disabling the computation agent-side of the APM trace metrics by // pretending it was already done by the library || !Config.get().isApmTracingEnabled() diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java index ffd2810a996..74e9066a154 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java @@ -11,9 +11,12 @@ import datadog.trace.api.Config; import datadog.trace.api.ProcessTags; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; /** Provides a canned message for OpenTelemetry's "resource.proto" wire protocol. */ @@ -36,8 +39,15 @@ private OtlpResourceProto() {} /** Prefix applied to {@code datadog.runtime_id} and process-tag resource attributes. */ private static final String DATADOG_PREFIX = "datadog."; - /** Vendor-neutral resource (no {@code datadog.*}). Used by the OTLP trace/metric export. */ - public static final byte[] RESOURCE_MESSAGE = buildResourceMessage(Config.get()); + /** + * Resource attribute added to OTLP trace metrics to ensure calculations are not re-computed in + * the Agent + */ + private static final String STATS_COMPUTED_KEY = "_dd.stats_computed"; + + /** Vendor-neutral resource (no {@code datadog.*}). Used by the OTLP metric export. */ + public static final byte[] RESOURCE_MESSAGE = + buildResourceMessage(Config.get(), Collections.emptyMap()); /** * Resource that additionally carries {@code datadog.runtime_id} and process tags (each prefixed @@ -45,13 +55,17 @@ private OtlpResourceProto() {} * mode. */ public static final byte[] RESOURCE_MESSAGE_WITH_DATADOG_ATTRS = - buildResourceMessage(Config.get(), true); + buildResourceMessage(Config.get(), datadogResourceAttributes(Config.get())); - static byte[] buildResourceMessage(Config config) { - return buildResourceMessage(config, false); - } + /** + * Resource used by the OTLP trace export. Identical to {@link #RESOURCE_MESSAGE} but adds the + * {@code _dd.stats_computed} marker when the SDK is computing OTLP span metrics, so a downstream + * Agent does not recompute them from the exported spans. + */ + public static final byte[] TRACE_RESOURCE_MESSAGE = + buildResourceMessage(Config.get(), traceResourceAttributes(Config.get())); - static byte[] buildResourceMessage(Config config, boolean includeDatadogResourceAttributes) { + static byte[] buildResourceMessage(Config config, Map extraAttributes) { GrowableBuffer buf = new GrowableBuffer(512); String serviceName = config.getServiceName(); @@ -85,8 +99,8 @@ static byte[] buildResourceMessage(Config config, boolean includeDatadogResource } }); - if (includeDatadogResourceAttributes) { - writeDatadogResourceAttributes(buf, config); + for (Map.Entry attribute : extraAttributes.entrySet()) { + writeResourceAttribute(buf, attribute.getKey(), attribute.getValue()); } OtlpProtoBuffer protobuf = new OtlpProtoBuffer(buf.capacity()); @@ -97,10 +111,24 @@ static byte[] buildResourceMessage(Config config, boolean includeDatadogResource return resourceMessage; } - private static void writeDatadogResourceAttributes(StreamingBuffer buf, Config config) { + /** + * Builds the extra resource attributes for the OTLP trace export: the {@code _dd.stats_computed} + * marker when the SDK is computing OTLP span metrics, so a downstream Agent does not recompute + * them from the exported spans. + */ + static Map traceResourceAttributes(Config config) { + Map attributes = new LinkedHashMap<>(); + if (config.isOtelTracesSpanMetricsEnabled()) { + attributes.put(STATS_COMPUTED_KEY, "true"); + } + return attributes; + } + + static Map datadogResourceAttributes(Config config) { + Map attributes = new LinkedHashMap<>(); String runtimeId = config.getRuntimeId(); if (runtimeId != null && !runtimeId.isEmpty()) { - writeResourceAttribute(buf, DATADOG_PREFIX + "runtime_id", runtimeId); + attributes.put(DATADOG_PREFIX + "runtime_id", runtimeId); } // Process tags arrive as "key:value" pairs; emit each as datadog. = value. List processTags = ProcessTags.getTagsAsStringList(); @@ -108,11 +136,11 @@ private static void writeDatadogResourceAttributes(StreamingBuffer buf, Config c for (String tag : processTags) { int colon = tag.indexOf(':'); if (colon > 0) { - writeResourceAttribute( - buf, DATADOG_PREFIX + tag.substring(0, colon), tag.substring(colon + 1)); + attributes.put(DATADOG_PREFIX + tag.substring(0, colon), tag.substring(colon + 1)); } } } + return attributes; } private static void writeResourceAttribute(StreamingBuffer buf, String key, String value) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java index 43582fdfa33..be13f66ac6b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriter.java @@ -7,6 +7,7 @@ import datadog.metrics.api.Histogram; import datadog.trace.api.Config; import datadog.trace.api.time.SystemTimeSource; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor; import datadog.trace.bootstrap.otlp.metrics.OtlpDataPoint; @@ -40,6 +41,7 @@ public final class OtlpStatsMetricWriter implements MetricWriter { private static final OtelInstrumentationScope SCOPE = new OtelInstrumentationScope("datadog.trace.metrics", null, null); + private static final String SERVICE_NAME = "service.name"; private static final String SPAN_NAME = "span.name"; private static final String SPAN_KIND = "span.kind"; private static final String HTTP_REQUEST_METHOD = "http.request.method"; @@ -57,8 +59,9 @@ public final class OtlpStatsMetricWriter implements MetricWriter { @Nullable private final OtlpSender sender; private final boolean otelSemanticsMode; + @Nullable private final String defaultService; + // own single-thread collector; forced to DELTA since trace-stats buckets are per-interval deltas. - // Initialized in the constructor so the resource chunk can be chosen per semantics mode. private final OtlpMetricsProtoCollector collector; // data points snapshotted during add(), replayed through the visitor in finishBucket() @@ -84,7 +87,10 @@ private static final class PendingPoint { public OtlpStatsMetricWriter(Config config) { // shared protocol-based sender selection so both OTLP metrics export paths agree - this(OtlpMetricsSenderFactory.create(config), config.isTraceOtelSemanticsEnabled()); + this( + OtlpMetricsSenderFactory.create(config), + config.isTraceOtelSemanticsEnabled(), + config.getServiceName()); if (this.sender == null) { // HTTP_JSON has no protobuf-free encoder yet; JSON transport is deferred per the plan. log.warn( @@ -95,10 +101,12 @@ public OtlpStatsMetricWriter(Config config) { } // visible for testing: lets tests inject a capturing sender to decode the emitted protobuf and - // control the semantics mode - OtlpStatsMetricWriter(@Nullable OtlpSender sender, boolean otelSemanticsMode) { + // control the semantics mode and default service + OtlpStatsMetricWriter( + @Nullable OtlpSender sender, boolean otelSemanticsMode, @Nullable String defaultService) { this.sender = sender; this.otelSemanticsMode = otelSemanticsMode; + this.defaultService = defaultService; // Default mode carries datadog.runtime_id / process tags on the Resource; OTel-semantics mode // uses the plain vendor-neutral resource (no datadog.*). byte[] resourceMessage = @@ -185,6 +193,11 @@ private void emitDataPointAttributes( // OTel semconv attrs are emitted in both modes emitStringAttribute(metric, SPAN_NAME, entry.getResource()); emitStringAttribute(metric, SPAN_KIND, entry.getSpanKind()); + // service.name on the point only when the span's service differs from the resource's default + UTF8BytesString service = entry.getService(); + if (service != null && service.length() > 0 && !service.toString().equals(defaultService)) { + emitStringAttribute(metric, SERVICE_NAME, service); + } if (entry.hasHttpMethod()) { emitStringAttribute(metric, HTTP_REQUEST_METHOD, entry.getHttpMethod()); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java index f37b7a5aea2..fa1efa87fba 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java @@ -1,6 +1,6 @@ package datadog.trace.core.otlp.trace; -import static datadog.trace.core.otlp.common.OtlpResourceProto.RESOURCE_MESSAGE; +import static datadog.trace.core.otlp.common.OtlpResourceProto.TRACE_RESOURCE_MESSAGE; import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordScopedSpansMessage; import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordSpanLinkMessage; import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordSpanMessage; @@ -138,7 +138,7 @@ private OtlpPayload completePayload() { } // prepend the canned resource chunk - payloadBytes += protobuf.recordMessage(RESOURCE_MESSAGE); + payloadBytes += protobuf.recordMessage(TRACE_RESOURCE_MESSAGE); // finally prepend the total length of all collected chunks protobuf.recordMessage(buf, 1, payloadBytes); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.groovy deleted file mode 100644 index dc9eb86fde3..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.groovy +++ /dev/null @@ -1,33 +0,0 @@ -package datadog.trace.common.metrics - -import datadog.communication.ddagent.SharedCommunicationObjects -import datadog.trace.api.Config -import datadog.trace.core.monitor.HealthMetrics -import datadog.trace.test.util.DDSpecification -import okhttp3.HttpUrl - -class MetricsAggregatorFactoryTest extends DDSpecification { - - def "when metrics disabled no-op aggregator created"() { - setup: - Config config = Mock(Config) - config.isTracerMetricsEnabled() >> false - def sco = Mock(SharedCommunicationObjects) - sco.agentUrl = HttpUrl.parse("http://localhost:8126") - expect: - def aggregator = MetricsAggregatorFactory.createMetricsAggregator(config, sco, HealthMetrics.NO_OP,) - assert aggregator instanceof NoOpMetricsAggregator - } - - def "when metrics enabled conflating aggregator created"() { - setup: - Config config = Spy(Config.get()) - config.isTracerMetricsEnabled() >> true - def sco = Mock(SharedCommunicationObjects) - sco.agentUrl = HttpUrl.parse("http://localhost:8126") - expect: - def aggregator = MetricsAggregatorFactory.createMetricsAggregator(config, sco, HealthMetrics.NO_OP, - ) - assert aggregator instanceof ClientStatsAggregator - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy index e4cc7d88fd3..0306de5ae39 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/writer/DDAgentApiTest.groovy @@ -18,6 +18,7 @@ import datadog.metrics.impl.MonitoringImpl import datadog.trace.api.Config import datadog.trace.api.ProcessTags import datadog.trace.api.ProtocolVersion +import datadog.trace.api.config.OtlpConfig import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.common.sampling.RateByServiceTraceSampler import datadog.trace.common.writer.ddagent.DDAgentApi @@ -454,6 +455,50 @@ class DDAgentApiTest extends DDCoreSpecification { return maps } + def "Datadog-Client-Computed-Stats header set when either stats pipeline is enabled (otlpSpanMetrics=#otlpSpanMetrics, nativeMetrics=#nativeMetrics)"() { + setup: + injectSysConfig(OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED, "$otlpSpanMetrics") + + def agent = httpServer { + handlers { + put("v0.4/traces") { + response.status(200).send() + } + } + } + def agentUrl = HttpUrl.get(agent.address.toString()) + + // Mock feature discovery so the native-stats pipeline signal can be controlled independently of + // what the (embedded) agent advertises. supportsMetrics() reflects agent-side client stats support. + def discovery = Mock(DDAgentFeaturesDiscovery) + discovery.getTraceEndpoint() >> "v0.4/traces" + discovery.supportsMetrics() >> nativeMetrics + discovery.state() >> null + + def client = OkHttpUtils.buildHttpClient(agentUrl, 1000) + // nativeMetricsEnabled is the constructor flag WriterFactory sets from Config.isTracerMetricsEnabled(). + def api = new DDAgentApi(client, agentUrl, discovery, monitoring, nativeMetrics) + def payload = prepareTraces("v0.4/traces", []) + + when: + // Named clientResponse (not response) to avoid shadowing the httpServer handler closure's `response`. + def clientResponse = api.sendSerializedTraces(payload) + + then: + clientResponse.success() + (agent.lastRequest.headers.get("Datadog-Client-Computed-Stats") == "true") == expectedComputesStats + + cleanup: + agent.close() + + where: + otlpSpanMetrics | nativeMetrics | expectedComputesStats + true | false | true // gap case: OTLP span metrics on, native stats off + false | false | false // neither pipeline computes stats + false | true | true // native stats on (regression guard) + true | true | true // both on + } + Payload prepareTraces(String agentVersion, List> traces) { Traces traceCapture = new Traces() def packer = new MsgPackWriter(new FlushingBuffer(1 << 20, traceCapture)) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ClientStatsAggregatorOtlpExportTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ClientStatsAggregatorOtlpExportTest.java new file mode 100644 index 00000000000..b8d56bde658 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ClientStatsAggregatorOtlpExportTest.java @@ -0,0 +1,85 @@ +package datadog.trace.common.metrics; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.core.CoreSpan; +import datadog.trace.core.SpanKindFilter; +import datadog.trace.core.monitor.HealthMetrics; +import datadog.trace.core.otlp.metrics.OtlpStatsMetricWriter; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +/** + * Coverage for the OTLP-export aggregation path added alongside {@link OtlpStatsMetricWriter}. The + * OTLP path (selected because the injected writer {@code instanceof OtlpStatsMetricWriter}) probes + * every known gRPC status-code convention with no span-type gate, unlike the native v0.6 path's + * single-key {@code rpc.grpc.status_code} lookup gated on {@code rpc}-typed spans. + */ +class ClientStatsAggregatorOtlpExportTest { + + @Test + void grpcStatusExtractedFromGrpcTypedSpanOnOtlpPath() throws Exception { + OtlpStatsMetricWriter writer = mock(OtlpStatsMetricWriter.class); + DDAgentFeaturesDiscovery features = mock(DDAgentFeaturesDiscovery.class); + when(features.peerTags()).thenReturn(Collections.emptySet()); + Sink sink = mock(Sink.class); + + ClientStatsAggregator aggregator = + new ClientStatsAggregator( + Collections.emptySet(), + features, + HealthMetrics.NO_OP, + sink, + writer, + /* maxAggregates */ 16, + /* queueSize */ 16, + /* reportingInterval */ 10, + SECONDS, + /* includeEndpointInMetrics */ false); + aggregator.start(); + try { + // A span typed "grpc" (not "rpc") carrying grpc.status.code -- a key + type combo the native + // v0.6 path would not surface. On the OTLP path it must still be extracted. + aggregator.publish(Collections.>singletonList(grpcSpan("grpc.status.code", "0"))); + aggregator.forceReport().get(5, SECONDS); + + ArgumentCaptor captor = ArgumentCaptor.forClass(AggregateEntry.class); + verify(writer).add(captor.capture()); + assertEquals("0", captor.getValue().getGrpcStatusCode().toString()); + } finally { + aggregator.close(); + } + } + + /** A metrics-eligible, top-level span typed {@code grpc} carrying a single tag. */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static CoreSpan grpcSpan(String tagKey, String tagValue) { + CoreSpan span = mock(CoreSpan.class); + when(span.isMeasured()).thenReturn(false); + when(span.isTopLevel()).thenReturn(true); + when(span.isKind(any(SpanKindFilter.class))).thenReturn(false); + when(span.getLongRunningVersion()).thenReturn(0); + when(span.getDurationNano()).thenReturn(SECONDS.toNanos(1)); + when(span.getError()).thenReturn(0); + when(span.getResourceName()).thenReturn("grpc.request"); + when(span.getServiceName()).thenReturn("svc"); + when(span.getOperationName()).thenReturn("grpc.request"); + when(span.getServiceNameSource()).thenReturn(null); + when(span.getType()).thenReturn("grpc"); + when(span.getHttpStatusCode()).thenReturn((short) 0); + when(span.getParentId()).thenReturn(0L); + when(span.getOrigin()).thenReturn(null); + when(span.unsafeGetTag(eq(Tags.SPAN_KIND), any(CharSequence.class))).thenReturn(""); + when(span.unsafeGetTag(tagKey)).thenReturn(tagValue); + return span; + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java new file mode 100644 index 00000000000..e9d981df389 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/MetricsAggregatorFactoryTest.java @@ -0,0 +1,89 @@ +package datadog.trace.common.metrics; + +import static datadog.trace.api.config.GeneralConfig.TRACE_STATS_COMPUTATION_ENABLED; +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; +import static datadog.trace.api.config.OtlpConfig.OTLP_METRICS_PROTOCOL; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.core.monitor.HealthMetrics; +import java.util.Properties; +import okhttp3.HttpUrl; +import org.junit.jupiter.api.Test; + +/** + * Tests the native-vs-OTLP XOR writer selection in {@link MetricsAggregatorFactory}. The selected + * writer is not exposed directly, but {@link ClientStatsAggregator} publishes the selection outcome + * via {@code isOtlpStatsExportEnabled()} plus the reporting cadence getters. + */ +class MetricsAggregatorFactoryTest { + + private static SharedCommunicationObjects sharedCommunicationObjects() { + SharedCommunicationObjects sco = mock(SharedCommunicationObjects.class); + sco.agentUrl = HttpUrl.parse("http://localhost:8126"); + when(sco.featuresDiscovery(any())).thenReturn(mock(DDAgentFeaturesDiscovery.class)); + return sco; + } + + private static Properties props(String... keyValues) { + Properties props = new Properties(); + for (int i = 0; i < keyValues.length; i += 2) { + props.setProperty(keyValues[i], keyValues[i + 1]); + } + return props; + } + + @Test + void whenAllMetricsDisabledNoOpAggregatorCreated() { + Config config = Config.get(props(TRACE_STATS_COMPUTATION_ENABLED, "false")); + + MetricsAggregator aggregator = + MetricsAggregatorFactory.createMetricsAggregator( + config, sharedCommunicationObjects(), HealthMetrics.NO_OP); + + assertInstanceOf(NoOpMetricsAggregator.class, aggregator); + } + + @Test + void whenNativeTracerMetricsEnabledSerializingWriterSelected() { + // tracer metrics default to enabled; OTLP span metrics default off (no OTLP trace export). + Config config = Config.get(props()); + + MetricsAggregator aggregator = + MetricsAggregatorFactory.createMetricsAggregator( + config, sharedCommunicationObjects(), HealthMetrics.NO_OP); + + ClientStatsAggregator conflating = assertInstanceOf(ClientStatsAggregator.class, aggregator); + assertFalse(conflating.isOtlpStatsExportEnabled()); + // native path uses a hardcoded 10s cadence, not trace.stats.interval. + assertEquals(10, conflating.reportingInterval()); + assertEquals(SECONDS, conflating.reportingIntervalTimeUnit()); + } + + @Test + void whenOtlpTraceMetricsEnabledOtlpStatsMetricWriterSelected() { + Config config = + Config.get( + props(OTEL_TRACES_SPAN_METRICS_ENABLED, "true", OTLP_METRICS_PROTOCOL, "http/json")); + + MetricsAggregator aggregator = + MetricsAggregatorFactory.createMetricsAggregator( + config, sharedCommunicationObjects(), HealthMetrics.NO_OP); + + ClientStatsAggregator conflating = assertInstanceOf(ClientStatsAggregator.class, aggregator); + assertTrue(conflating.isOtlpStatsExportEnabled()); + // OTLP path sources the cadence from trace.stats.interval (ms), default 10s. + assertEquals(config.getTraceStatsInterval(), conflating.reportingInterval()); + assertEquals(MILLISECONDS, conflating.reportingIntervalTimeUnit()); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceProtoTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceProtoTest.java index 80740dc7e54..9c901d9c00c 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceProtoTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceProtoTest.java @@ -5,6 +5,7 @@ import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME; import static datadog.trace.api.config.GeneralConfig.TAGS; import static datadog.trace.api.config.GeneralConfig.VERSION; +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.TracerConfig.TRACE_REPORT_HOSTNAME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -14,6 +15,7 @@ import com.google.protobuf.WireFormat; import datadog.trace.api.Config; import java.io.IOException; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; @@ -149,25 +151,28 @@ void testBuildResourceMessage( String caseName, Properties properties, Map expectedAttributes) throws IOException { Config config = Config.get(properties); - byte[] bytes = OtlpResourceProto.buildResourceMessage(config); + byte[] bytes = OtlpResourceProto.buildResourceMessage(config, Collections.emptyMap()); Map actualAttributes = parseResourceAttributes(bytes); assertEquals(expectedAttributes, actualAttributes, "For case: " + caseName); } /** - * The datadog-attrs variant ({@code buildResourceMessage(config, true)}) carries {@code - * datadog.runtime_id}; the plain variant omits it. (Process tags are emitted only when the - * experimental process-tag propagation is enabled, so they aren't asserted here.) + * The datadog-attrs variant ({@code buildResourceMessage(config, datadogResourceAttributes)}) + * carries {@code datadog.runtime_id}; the plain variant omits it. (Process tags are emitted only + * when the experimental process-tag propagation is enabled, so they aren't asserted here.) */ @Test void datadogResourceAttributesVariantCarriesRuntimeId() throws IOException { Config config = Config.get(props(SERVICE_NAME, "my-service")); Map withDatadog = - parseResourceAttributes(OtlpResourceProto.buildResourceMessage(config, true)); + parseResourceAttributes( + OtlpResourceProto.buildResourceMessage( + config, OtlpResourceProto.datadogResourceAttributes(config))); Map plain = - parseResourceAttributes(OtlpResourceProto.buildResourceMessage(config, false)); + parseResourceAttributes( + OtlpResourceProto.buildResourceMessage(config, Collections.emptyMap())); assertTrue( withDatadog.containsKey("datadog.runtime_id"), @@ -179,6 +184,26 @@ void datadogResourceAttributesVariantCarriesRuntimeId() throws IOException { assertFalse(plain.containsKey("datadog.runtime_id"), "plain variant omits datadog.runtime_id"); } + @Test + void statsComputedVariantCarriesMarker() throws IOException { + Config withMetrics = + Config.get(props(SERVICE_NAME, "my-service", OTEL_TRACES_SPAN_METRICS_ENABLED, "true")); + Config withoutMetrics = Config.get(props(SERVICE_NAME, "my-service")); + + Map withMarker = + parseResourceAttributes( + OtlpResourceProto.buildResourceMessage( + withMetrics, OtlpResourceProto.traceResourceAttributes(withMetrics))); + Map without = + parseResourceAttributes( + OtlpResourceProto.buildResourceMessage( + withoutMetrics, OtlpResourceProto.traceResourceAttributes(withoutMetrics))); + + assertEquals( + "true", withMarker.get("_dd.stats_computed"), "marker present when stats computed"); + assertFalse(without.containsKey("_dd.stats_computed"), "marker absent when stats not computed"); + } + // ── parsing helpers ─────────────────────────────────────────────────────── /** diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java index 9f0b3672eef..670f9098a14 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/metrics/OtlpStatsMetricWriterTest.java @@ -311,7 +311,7 @@ private static Object readAnyValue(CodedInputStream any) throws IOException { private static DecodedMetric writeAndDecode(boolean otelSemanticsMode, AggregateEntry entry) throws IOException { CapturingSender sender = new CapturingSender(); - OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, otelSemanticsMode); + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, otelSemanticsMode, null); writer.startBucket(1, BUCKET_START, BUCKET_DURATION); writer.add(entry); writer.finishBucket(); @@ -369,7 +369,7 @@ void okPlusErrorEntryProducesTwoDataPointsWithErrorStatus() throws IOException { @Test void errorSeriesDoesNotLingerAfterClearWhenBucketHasOnlyOkHits() throws IOException { CapturingSender sender = new CapturingSender(); - OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false); + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false, null); // Bucket 1: the entry sees an error, so its error histogram is allocated and emits a point. AggregateEntry e = entry("GET /users", false, 0, null, null, null); @@ -424,10 +424,67 @@ void httpAndGrpcAttributesAppearOnlyWhenSet() throws IOException { assertFalse(bareAttrs.containsKey("rpc.response.status_code")); } + @Test + void serviceNameEmittedOnlyForNonDefaultService() throws IOException { + CapturingSender sender = new CapturingSender(); + // The configured default service ("web") is reported on the resource; only a span whose service + // differs from it repeats service.name on its own data point. + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false, "web"); + + long start = SECONDS.toNanos(1_700_000_000L); + writer.startBucket(2, start, SECONDS.toNanos(10)); + writer.add(serviceEntry("web.request", "web")); // default service + writer.add(serviceEntry("db.query", "postgres")); // custom service + writer.finishBucket(); + + DecodedMetric metric = decode(sender.lastPayload); + assertEquals(2, metric.dataPoints.size()); + + Map defaultAttrs = null; + Map customAttrs = null; + for (DataPoint dp : metric.dataPoints) { + if ("db.query".equals(dp.attributes.get("datadog.operation.name"))) { + customAttrs = dp.attributes; + } else { + defaultAttrs = dp.attributes; + } + } + assertNotNull(customAttrs, "custom-service data point present"); + assertNotNull(defaultAttrs, "default-service data point present"); + assertEquals( + "postgres", + customAttrs.get("service.name"), + "non-default service is carried on its own data point"); + assertFalse( + defaultAttrs.containsKey("service.name"), + "default service must not be repeated on its data point"); + } + + /** An ok-only entry on the given service and operation name, recording a single 1s hit. */ + private static AggregateEntry serviceEntry(String operationName, String service) { + AggregateEntry e = + AggregateEntryTestUtils.of( + "GET /users", + service, + operationName, + null, + "web", + 0, + false, + true, + "server", + null, + null, + null, + null); + AggregateEntryTestUtils.recordOk(e, SECONDS.toNanos(1)); + return e; + } + @Test void emptyBucketSendsNothing() { CapturingSender sender = new CapturingSender(); - OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false); + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false, null); writer.startBucket(0, BUCKET_START, BUCKET_DURATION); writer.finishBucket(); // no add() @@ -439,7 +496,7 @@ void emptyBucketSendsNothing() { @Test void nullSenderDoesNotThrowOnNonEmptyBucket() { // mirrors the HTTP_JSON path where createSender(config) returns null. - OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(null, false); + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(null, false, null); writer.startBucket(1, BUCKET_START, BUCKET_DURATION); writer.add(okEntry(SECONDS.toNanos(1), 2)); try { @@ -513,7 +570,7 @@ void snapshotsEntryDataBeforeAggregatorClearsIt() throws IOException { // (and the top-level count) at add() time; if it deferred reading to finishBucket() it would // encode the already-cleared (empty, zero-count) entry. CapturingSender sender = new CapturingSender(); - OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false); + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false, null); AggregateEntry e = entry("servlet.request", false, 0, null, null, null); AggregateEntryTestUtils.recordTopLevel(e, SECONDS.toNanos(1)); @@ -541,7 +598,7 @@ void defaultModeResourceCarriesRuntimeId() throws IOException { // runtime-id is enabled by default, so default-mode payloads carry datadog.runtime_id on the // Resource. CapturingSender sender = new CapturingSender(); - OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false); + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, false, null); writer.startBucket(1, SECONDS.toNanos(1_700_000_000L), SECONDS.toNanos(10)); writer.add(okEntry(SECONDS.toNanos(1), 1)); writer.finishBucket(); @@ -558,7 +615,7 @@ void defaultModeResourceCarriesRuntimeId() throws IOException { @Test void otelSemanticsModeResourceOmitsDatadogAttributes() throws IOException { CapturingSender sender = new CapturingSender(); - OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, true); + OtlpStatsMetricWriter writer = new OtlpStatsMetricWriter(sender, true, null); writer.startBucket(1, SECONDS.toNanos(1_700_000_000L), SECONDS.toNanos(10)); writer.add(okEntry(SECONDS.toNanos(1), 1)); writer.finishBucket(); diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 70ff04932e7..e3b5a8a2371 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -479,6 +479,7 @@ import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_EXPORTER; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_INTERVAL; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_TIMEOUT; +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.OtlpConfig.OTLP_LOGS_COMPRESSION; import static datadog.trace.api.config.OtlpConfig.OTLP_LOGS_ENDPOINT; import static datadog.trace.api.config.OtlpConfig.OTLP_LOGS_HEADERS; @@ -495,7 +496,6 @@ import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_HEADERS; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_PROTOCOL; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_TIMEOUT; -import static datadog.trace.api.config.OtlpConfig.TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; import static datadog.trace.api.config.ProfilingConfig.PROFILING_AGENTLESS; import static datadog.trace.api.config.ProfilingConfig.PROFILING_AGENTLESS_DEFAULT; @@ -1006,7 +1006,7 @@ public static String getHostName() { private final int otlpMetricsTimeout; private final OtlpConfig.Temporality otlpMetricsTemporalityPreference; - private final boolean tracesSpanMetricsEnabled; + private final boolean otelTracesSpanMetricsEnabled; private final boolean traceOtelSemanticsEnabled; private final String traceOtelExporter; @@ -2141,9 +2141,9 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins traceOtelSemanticsEnabled = configProvider.getBoolean(TRACE_OTEL_SEMANTICS_ENABLED, false); // Tri-state default: when unset, SDK-computed OTLP span metrics are emitted iff OTLP trace // export and OTLP metrics export are both enabled. - tracesSpanMetricsEnabled = + otelTracesSpanMetricsEnabled = configProvider.getBoolean( - TRACES_SPAN_METRICS_ENABLED, + OTEL_TRACES_SPAN_METRICS_ENABLED, isTraceOtlpExporterEnabled() && isMetricsOtelEnabled() && isMetricsOtlpExporterEnabled()); @@ -5634,8 +5634,8 @@ public OtlpConfig.Temporality getOtlpMetricsTemporalityPreference() { return otlpMetricsTemporalityPreference; } - public boolean isTracesSpanMetricsEnabled() { - return tracesSpanMetricsEnabled; + public boolean isOtelTracesSpanMetricsEnabled() { + return otelTracesSpanMetricsEnabled; } public boolean isTraceOtelSemanticsEnabled() { @@ -6748,8 +6748,8 @@ public String toString() { + otlpMetricsTimeout + ", otlpMetricsTemporalityPreference=" + otlpMetricsTemporalityPreference - + ", tracesSpanMetricsEnabled=" - + tracesSpanMetricsEnabled + + ", otelTracesSpanMetricsEnabled=" + + otelTracesSpanMetricsEnabled + ", traceOtelSemanticsEnabled=" + traceOtelSemanticsEnabled + ", traceStatsInterval=" diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 7db955e49ae..11f100efc42 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -150,7 +150,7 @@ import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_ENDPOINT import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_HEADERS import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_PROTOCOL import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_TIMEOUT -import static datadog.trace.api.config.OtlpConfig.TRACES_SPAN_METRICS_ENABLED +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_ENABLED import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER import datadog.trace.config.inversion.ConfigHelper @@ -560,7 +560,7 @@ class ConfigTest extends DDSpecification { config.otlpTracesProtocol == HTTP_PROTOBUF config.otlpTracesTimeout == 10000 - !config.tracesSpanMetricsEnabled + !config.otelTracesSpanMetricsEnabled !config.traceOtelSemanticsEnabled config.traceStatsInterval == 10000 } @@ -731,7 +731,7 @@ class ConfigTest extends DDSpecification { System.setProperty(PREFIX + METRICS_OTEL_EXPORTER, metricsExporter) } if (override != null) { - System.setProperty(PREFIX + TRACES_SPAN_METRICS_ENABLED, override) + System.setProperty(PREFIX + OTEL_TRACES_SPAN_METRICS_ENABLED, override) } when: @@ -739,8 +739,8 @@ class ConfigTest extends DDSpecification { then: // Unset: emit iff OTLP trace export and OTLP metrics export are both on. An explicit - // dd.traces.span.metrics.enabled always wins. - config.tracesSpanMetricsEnabled == expected + // dd.otel.traces.span.metrics.enabled always wins. + config.otelTracesSpanMetricsEnabled == expected where: exporter | metricsEnabled | metricsExporter | override | expected @@ -994,7 +994,7 @@ class ConfigTest extends DDSpecification { config.otlpTracesTimeout == 5002 config.traceOtelSemanticsEnabled - config.tracesSpanMetricsEnabled // tri-state default: OTLP trace export + OTel metrics both on + config.otelTracesSpanMetricsEnabled // tri-state default: OTLP trace export + OTel metrics both on } def "specify overrides via env vars"() { diff --git a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSourceTest.groovy b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSourceTest.groovy index 783df9e76f4..977bba8f6c6 100644 --- a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSourceTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSourceTest.groovy @@ -9,7 +9,7 @@ import static datadog.trace.api.config.GeneralConfig.RUNTIME_METRICS_ENABLED import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.VERSION -import static datadog.trace.api.config.OtlpConfig.TRACES_SPAN_METRICS_ENABLED +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_ENABLED import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED @@ -268,7 +268,7 @@ class OtelEnvironmentConfigSourceTest extends DDSpecification { def source = new OtelEnvironmentConfigSource() then: - source.get(TRACES_SPAN_METRICS_ENABLED) == value + source.get(OTEL_TRACES_SPAN_METRICS_ENABLED) == value where: value << ['true', 'false'] @@ -283,7 +283,7 @@ class OtelEnvironmentConfigSourceTest extends DDSpecification { def source = new OtelEnvironmentConfigSource() then: - source.get(TRACES_SPAN_METRICS_ENABLED) == value + source.get(OTEL_TRACES_SPAN_METRICS_ENABLED) == value where: value << ['true', 'false'] @@ -298,7 +298,7 @@ class OtelEnvironmentConfigSourceTest extends DDSpecification { def source = new OtelEnvironmentConfigSource() then: - source.get(TRACES_SPAN_METRICS_ENABLED) == null + source.get(OTEL_TRACES_SPAN_METRICS_ENABLED) == null } def "otel traces exporter none still disables tracing"() { diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 733f3736cbb..ca70112b07e 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -8657,7 +8657,7 @@ "aliases": [] } ], - "DD_TRACES_SPAN_METRICS_ENABLED": [ + "DD_OTEL_TRACES_SPAN_METRICS_ENABLED": [ { "version": "A", "type": "boolean", diff --git a/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java b/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java index 1aea89c9a2d..28ef45ff9de 100644 --- a/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java +++ b/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java @@ -21,6 +21,7 @@ import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_EXPORTER; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_INTERVAL; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_TIMEOUT; +import static datadog.trace.api.config.OtlpConfig.OTEL_TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.OtlpConfig.OTLP_LOGS_COMPRESSION; import static datadog.trace.api.config.OtlpConfig.OTLP_LOGS_ENDPOINT; import static datadog.trace.api.config.OtlpConfig.OTLP_LOGS_HEADERS; @@ -37,7 +38,6 @@ import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_HEADERS; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_PROTOCOL; import static datadog.trace.api.config.OtlpConfig.OTLP_TRACES_TIMEOUT; -import static datadog.trace.api.config.OtlpConfig.TRACES_SPAN_METRICS_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_ENABLED; import static datadog.trace.api.config.OtlpConfig.TRACE_OTEL_EXPORTER; import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED; @@ -163,8 +163,9 @@ private void setupTraceOtelEnvironment() { capture(RESPONSE_HEADER_TAGS, mapHeaderTags("http.response.header.", responseHeaders)); capture(TRACE_EXTENSIONS_PATH, extensions); capture( - TRACES_SPAN_METRICS_ENABLED, - getOtelProperty("otel.traces.span.metrics.enabled", "dd." + TRACES_SPAN_METRICS_ENABLED)); + OTEL_TRACES_SPAN_METRICS_ENABLED, + getOtelProperty( + "otel.traces.span.metrics.enabled", "dd." + OTEL_TRACES_SPAN_METRICS_ENABLED)); String exporter = getOtelProperty("otel.traces.exporter"); if ("otlp".equalsIgnoreCase(exporter)) { // traces defaults to non-OTLP (i.e. datadog)