fix(telemetry): stop a filtered span from becoming the parent a remot…

framework/src/BBT.Aether.AspNetCore/BBT/Aether/AspNetCore/Telemetry/DaprDiagnosticRequest.cs

@@ -0,0 +1,49 @@
+using System;
+
+namespace BBT.Aether.AspNetCore.Telemetry;
+
+/// <summary>
+/// The single definition of "a Dapr call that is diagnostic noise rather than business work" —
+/// state store, lock, secret and configuration operations, in both their gRPC and HTTP API spellings.
+/// </summary>
+/// <remarks>
+/// <para>
+/// It lives in its own type because two very different places must agree on it, and a copy in either
+/// would be a silent trap. The tracing filter uses it to decide that a span is not worth exporting;
+/// <see cref="FilteredSpanParentTextMapPropagator"/> uses it to decide that the same span must not
+/// become the parent a remote service sees. If those two lists ever disagreed, a request would be
+/// filtered but still propagated — which is exactly the orphan this pairing exists to prevent.
+/// </para>
+/// </remarks>
+internal static class DaprDiagnosticRequest
+{
+    internal static bool Matches(Uri? uri)
+    {
+        var path = uri?.AbsolutePath;
+        if (string.IsNullOrEmpty(path))
+        {
+            return false;
+        }
+
+        if (path.StartsWith("/dapr.proto.runtime.v1.Dapr/", StringComparison.OrdinalIgnoreCase))
+        {
+            return path.EndsWith("/GetState", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/GetBulkState", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/SaveState", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/DeleteState", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/ExecuteStateTransaction", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/GetSecret", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/GetBulkSecret", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/GetConfiguration", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/SubscribeConfiguration", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/TryLockAlpha1", StringComparison.OrdinalIgnoreCase)
+                   || path.EndsWith("/UnlockAlpha1", StringComparison.OrdinalIgnoreCase);
+        }
+
+        return path.StartsWith("/v1.0/state/", StringComparison.OrdinalIgnoreCase)
+               || path.StartsWith("/v1.0-alpha1/state/", StringComparison.OrdinalIgnoreCase)
+               || path.StartsWith("/v1.0-alpha1/lock/", StringComparison.OrdinalIgnoreCase)
+               || path.StartsWith("/v1.0/secrets/", StringComparison.OrdinalIgnoreCase)
+               || path.StartsWith("/v1.0/configuration/", StringComparison.OrdinalIgnoreCase);
+    }
+}

framework/src/BBT.Aether.AspNetCore/BBT/Aether/AspNetCore/Telemetry/FilteredSpanParentPropagator.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace BBT.Aether.AspNetCore.Telemetry;
+
+/// <summary>
+/// Wraps the ambient <see cref="DistributedContextPropagator"/> so a span the tracing filter is
+/// about to drop never becomes the parent a remote service sees.
+/// </summary>
+/// <remarks>
+/// <para>
+/// The decision itself lives in <see cref="FilteredSpanRedirect"/>, which also carries the
+/// measurements: why the obvious "is it recorded?" test cannot work, and which three narrower fixes
+/// were tried and refuted first.
+/// </para>
+/// <para>
+/// This layer is .NET's own injection, which <c>DiagnosticsHandler</c> performs before it raises the
+/// DiagnosticSource start event. In a host where OpenTelemetry's HttpClient instrumentation is also
+/// active, the second injection overwrites this one — so
+/// <see cref="FilteredSpanParentTextMapPropagator"/> exists as well and the two must agree. This one
+/// is what protects a host that has HttpClient instrumentation turned off.
+/// </para>
+/// </remarks>
+/// <param name="inner">The propagator to delegate the actual header work to.</param>
+public sealed class FilteredSpanParentPropagator(DistributedContextPropagator inner) : DistributedContextPropagator
+{
+    private readonly DistributedContextPropagator _inner =
+        inner ?? throw new ArgumentNullException(nameof(inner));
+
+    /// <inheritdoc />
+    public override IReadOnlyCollection<string> Fields => _inner.Fields;
+
+    /// <inheritdoc />
+    public override void Inject(Activity? activity, object? carrier, PropagatorSetterCallback? setter) =>
+        _inner.Inject(FilteredSpanRedirect.Target(activity, carrier), carrier, setter);
+
+    /// <inheritdoc />
+    public override void ExtractTraceIdAndState(
+        object? carrier,
+        PropagatorGetterCallback? getter,
+        out string? traceId,
+        out string? traceState) =>
+        _inner.ExtractTraceIdAndState(carrier, getter, out traceId, out traceState);
+
+    /// <inheritdoc />
+    public override IEnumerable<KeyValuePair<string, string?>>? ExtractBaggage(
+        object? carrier,
+        PropagatorGetterCallback? getter) =>
+        _inner.ExtractBaggage(carrier, getter);
+
+}

framework/src/BBT.Aether.AspNetCore/BBT/Aether/AspNetCore/Telemetry/FilteredSpanParentTextMapPropagator.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using OpenTelemetry;
+using OpenTelemetry.Context.Propagation;
+
+namespace BBT.Aether.AspNetCore.Telemetry;
+
+/// <summary>
+/// The OpenTelemetry counterpart of <see cref="FilteredSpanParentPropagator"/>: stops a filtered span
+/// from becoming the parent a remote service sees.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Both wrappers are needed, and which one lands on the wire is not obvious. .NET's
+/// <c>DiagnosticsHandler</c> injects through <see cref="DistributedContextPropagator"/> and *then*
+/// raises the DiagnosticSource start event; OpenTelemetry's HttpClient instrumentation handles that
+/// event and injects again, with the just-created <c>System.Net.Http.HttpRequestOut</c> context —
+/// overwriting whatever was already there. Measured on 2026-09-13: with only the
+/// <see cref="DistributedContextPropagator"/> wrapper installed, the header written was verifiably
+/// the recorded gRPC span's id, and the Dapr sidecar still reported a parent that matched no
+/// exported span. The ids it used were the filtered activity's, i.e. the second injection's.
+/// </para>
+/// <para>
+/// So the rule has to be enforced at both layers. This one applies it where OpenTelemetry injects:
+/// when the context handed to it is not <see cref="ActivityTraceFlags.Recorded"/>, walk
+/// <see cref="Activity.Current"/> up to the nearest ancestor that is, and propagate that instead.
+/// </para>
+/// <para>
+/// Baggage travels unchanged. Only the parent id and its trace flags are redirected — the trace id
+/// is the same on every ancestor, so a redirect can never move the call into a different trace.
+/// </para>
+/// </remarks>
+/// <param name="inner">The propagator that does the actual header work, normally the W3C one.</param>
+public sealed class FilteredSpanParentTextMapPropagator(TextMapPropagator inner) : TextMapPropagator
+{
+    private readonly TextMapPropagator _inner = inner ?? throw new ArgumentNullException(nameof(inner));
+
+    /// <inheritdoc />
+    public override ISet<string> Fields => _inner.Fields!;
+
+    /// <inheritdoc />
+    public override PropagationContext Extract<T>(
+        PropagationContext context,
+        T carrier,
+        Func<T, string, IEnumerable<string>?> getter) =>
+        _inner.Extract(context, carrier, getter);
+
+    /// <inheritdoc />
+    public override void Inject<T>(
+        PropagationContext context,
+        T carrier,
+        Action<T, string, string> setter)
+    {
+        var target = FilteredSpanRedirect.Target(Activity.Current, carrier);
+
+        // Only redirect when the current activity really is the one being injected; otherwise the
+        // context came from somewhere we cannot reason about and must travel untouched.
+        var redirected = target is not null
+                         && Activity.Current is not null
+                         && Activity.Current.Context.SpanId == context.ActivityContext.SpanId
+                         && target != Activity.Current
+            ? new PropagationContext(target.Context, context.Baggage)
+            : context;
+
+        _inner.Inject(redirected, carrier, setter);
+    }
+
+}

framework/src/BBT.Aether.AspNetCore/BBT/Aether/AspNetCore/Telemetry/FilteredSpanRedirect.cs

@@ -0,0 +1,59 @@
+using System.Diagnostics;
+using System.Net.Http;
+
+namespace BBT.Aether.AspNetCore.Telemetry;
+
+/// <summary>
+/// The one decision both propagator wrappers make: when the request about to go out is a Dapr
+/// diagnostic call — a span the tracing filter will drop — the parent written to the wire must be
+/// the enclosing activity, not the one being dropped.
+/// </summary>
+/// <remarks>
+/// <para>
+/// <b>The defect.</b> Filtering a span does not remove its <see cref="Activity"/>. The activity is
+/// created, its id is written into the outgoing <c>traceparent</c>, and only afterwards is it marked
+/// unrecorded. The receiver cannot know the id names a span nobody will write: a Dapr sidecar with
+/// <c>samplingRate: "1"</c> samples on its own terms and records a span whose parent document never
+/// arrives. Elastic APM resolves nesting strictly through <c>parent.id</c> and re-roots such a span
+/// to the trace root; OpenObserve groups by trace id and hides it, which is why the same deployment
+/// looks healthy on one backend and littered on the other.
+/// </para>
+/// <para>
+/// <b>Why the obvious test does not work.</b> The natural rule — "if this context is not recorded,
+/// propagate its nearest recorded ancestor" — is unusable, because the flag has not been cleared
+/// yet. Measured on 2026-09-13 by logging every injection: the context handed to the propagator for
+/// a filtered Dapr state call arrives as <c>Recorded</c>, with
+/// <c>Activity.Current = System.Net.Http.HttpRequestOut</c>. OpenTelemetry's HttpClient
+/// instrumentation injects the headers first and applies <c>FilterHttpRequestMessage</c> afterwards.
+/// At injection time nothing about the activity distinguishes it from one that will be exported.
+/// </para>
+/// <para>
+/// <b>What does work.</b> The carrier is the <see cref="HttpRequestMessage"/> itself, so the same
+/// predicate the filter will apply can be applied here, before the header is written — and
+/// <see cref="DaprDiagnosticRequest"/> is that predicate, shared rather than copied precisely so the
+/// two decisions cannot drift apart. The parent then becomes the enclosing activity, which for a
+/// Dapr state or lock call is the <c>dapr.proto.runtime.v1.Dapr/GetState</c> gRPC client span the
+/// framework already exports.
+/// </para>
+/// </remarks>
+internal static class FilteredSpanRedirect
+{
+    /// <summary>
+    /// The activity whose id should go on the wire for <paramref name="carrier"/>, or
+    /// <paramref name="current"/> when nothing needs redirecting.
+    /// </summary>
+    /// <remarks>
+    /// Both guards are load-bearing. A carrier that is not an <see cref="HttpRequestMessage"/> is
+    /// some other transport we know nothing about; and a redirect is only safe while
+    /// <paramref name="current"/> really is the activity for this request, which is what makes its
+    /// <see cref="Activity.Parent"/> the enclosing span rather than an unrelated one. A parentless
+    /// activity is left alone: propagating nothing would start a fresh trace at the receiver and
+    /// break the correlation this exists to protect.
+    /// </remarks>
+    internal static Activity? Target(Activity? current, object? carrier) =>
+        carrier is HttpRequestMessage request
+        && DaprDiagnosticRequest.Matches(request.RequestUri)
+        && current is { Parent: not null }
+            ? current.Parent
+            : current;
+}

framework/src/BBT.Aether.AspNetCore/Microsoft/Extensions/DependencyInjection/AetherTelemetryServiceCollectionExtensions.cs

@@ -13,6 +13,8 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using OpenTelemetry;
+using OpenTelemetry.Context.Propagation;
 using OpenTelemetry.Logs;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
@@ -100,6 +102,13 @@ public static IServiceCollection AddAetherTelemetry(
             {
                 if (!opts.TracingEnabled) return;
 
+                // A filtered span must not become the parent a remote service sees. Instrumentation
+                // filters run after the Activity exists and only clear the Recorded flag, so the id
+                // that goes on the wire names a span nobody will export — and a receiver that
+                // samples on its own terms (a Dapr sidecar does) then records an orphan. See
+                // FilteredSpanParentPropagator for the measurements and the rejected alternatives.
+                InstallFilteredSpanParentPropagator();
+
                 var excludedPatterns = CompileRegex(opts.Logging.ExcludedPaths
                     .Concat(opts.Tracing.ExcludedPaths));
 
@@ -293,44 +302,36 @@ private static bool IsExcluded(string? value, List<Regex> patterns)
         return false;
     }
 
-    private static bool ShouldTraceHttpRequest(HttpRequestMessage request, List<Regex> excludedPatterns)
+    /// <summary>
+    /// Wraps both injection layers once per process. Idempotent on purpose: AddAetherTelemetry can
+    /// be called more than once in tests and in hosts that compose several modules, and stacking
+    /// wrappers would re-run the same decision for no gain.
+    /// </summary>
+    private static void InstallFilteredSpanParentPropagator()
     {
-        if (IsExcluded(request.RequestUri?.ToString(), excludedPatterns))
+        if (DistributedContextPropagator.Current is not FilteredSpanParentPropagator)
         {
-            return false;
+            DistributedContextPropagator.Current =
+                new FilteredSpanParentPropagator(DistributedContextPropagator.Current);
         }
 
-        return AetherTracingRuntime.IsVerbose || !IsDaprDiagnosticRequest(request.RequestUri);
+        // Both layers inject and OpenTelemetry's runs last, overwriting what .NET wrote — so
+        // wrapping only the .NET propagator leaves the defect in place. Measured, not assumed.
+        if (Propagators.DefaultTextMapPropagator is not FilteredSpanParentTextMapPropagator)
+        {
+            Sdk.SetDefaultTextMapPropagator(
+                new FilteredSpanParentTextMapPropagator(Propagators.DefaultTextMapPropagator));
+        }
     }
 
-    private static bool IsDaprDiagnosticRequest(Uri? uri)
+    private static bool ShouldTraceHttpRequest(HttpRequestMessage request, List<Regex> excludedPatterns)
     {
-        var path = uri?.AbsolutePath;
-        if (string.IsNullOrEmpty(path))
+        if (IsExcluded(request.RequestUri?.ToString(), excludedPatterns))
         {
             return false;
         }
 
-        if (path.StartsWith("/dapr.proto.runtime.v1.Dapr/", StringComparison.OrdinalIgnoreCase))
-        {
-            return path.EndsWith("/GetState", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/GetBulkState", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/SaveState", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/DeleteState", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/ExecuteStateTransaction", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/GetSecret", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/GetBulkSecret", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/GetConfiguration", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/SubscribeConfiguration", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/TryLockAlpha1", StringComparison.OrdinalIgnoreCase)
-                   || path.EndsWith("/UnlockAlpha1", StringComparison.OrdinalIgnoreCase);
-        }
-
-        return path.StartsWith("/v1.0/state/", StringComparison.OrdinalIgnoreCase)
-               || path.StartsWith("/v1.0-alpha1/state/", StringComparison.OrdinalIgnoreCase)
-               || path.StartsWith("/v1.0-alpha1/lock/", StringComparison.OrdinalIgnoreCase)
-               || path.StartsWith("/v1.0/secrets/", StringComparison.OrdinalIgnoreCase)
-               || path.StartsWith("/v1.0/configuration/", StringComparison.OrdinalIgnoreCase);
+        return AetherTracingRuntime.IsVerbose || !DaprDiagnosticRequest.Matches(request.RequestUri);
     }
 
     private static void EnrichHttpClientActivity(Activity activity, HttpRequestMessage request)