From 0928bda7ac215262c194c81818580edc0371838f Mon Sep 17 00:00:00 2001 From: woksin Date: Thu, 12 Nov 2026 20:22:32 +0100 Subject: [PATCH 1/2] Harden DPoP replay freshness and add Redis integration specs --- .github/workflows/pull-requests.yml | 25 +++- .github/workflows/redis-replay-specs.yml | 38 ++++++ AuthProxy.slnx | 1 + Directory.Packages.props | 1 + Documentation/configuration/dpop.md | 4 +- .../AuthProxy.Integration.Specs.csproj | 20 +++ .../GlobalUsings.cs | 5 + .../RedisReplayCollection.cs | 12 ++ .../RedisReplayFixture.cs | 46 +++++++ .../given_a_redis_replay_store.cs | 75 ++++++++++++ ...en_a_replica_clock_disagrees_with_redis.cs | 47 +++++++ ..._redis_restarts_without_replay_evidence.cs | 32 +++++ .../when_replay_coverage_is_quarantined.cs | 41 +++++++ .../when_reserving_a_proof_across_replicas.cs | 42 +++++++ .../when_caching_replay_store_health.cs | 115 ++++++++++++++++++ .../when_mapping_exact_target_paths.cs | 39 ++++++ .../when_proof_precedes_scope_step_up.cs | 31 ++++- .../BearerRoutes/Dpop/DpopEnforcement.cs | 1 + .../BearerRoutes/Dpop/RedisDpopReplayStore.cs | 34 ++++-- Source/AuthProxy/Properties/AssemblyInfo.cs | 1 + 20 files changed, 598 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/redis-replay-specs.yml create mode 100644 Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj create mode 100644 Source/AuthProxy.Integration.Specs/GlobalUsings.cs create mode 100644 Source/AuthProxy.Integration.Specs/RedisReplayCollection.cs create mode 100644 Source/AuthProxy.Integration.Specs/RedisReplayFixture.cs create mode 100644 Source/AuthProxy.Integration.Specs/given_a_redis_replay_store.cs create mode 100644 Source/AuthProxy.Integration.Specs/when_a_replica_clock_disagrees_with_redis.cs create mode 100644 Source/AuthProxy.Integration.Specs/when_redis_restarts_without_replay_evidence.cs create mode 100644 Source/AuthProxy.Integration.Specs/when_replay_coverage_is_quarantined.cs create mode 100644 Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 75da8c75..a7016571 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -31,6 +31,7 @@ jobs: prerelease: ${{ steps.filter.outputs.prerelease }} azure-docs: ${{ steps.filter.outputs.azure-docs }} e2e: ${{ steps.filter.outputs.e2e }} + redis-replay: ${{ steps.filter.outputs.redis-replay }} steps: # The pull request files API lists at most 3,000 files, so a larger pull request could hide # changed paths from the filter and skip checks. Fail closed instead of filtering a partial list. @@ -46,6 +47,21 @@ jobs: uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 with: filters: | + redis-replay: + - "Source/AuthProxy/BearerRoutes/**" + - "Source/AuthProxy/Configuration/Dpop*.cs" + - "Source/AuthProxy/Configuration/BearerRoute*.cs" + - "Source/AuthProxy/Configuration/AuthProxy.cs" + - "Source/AuthProxy/AuthProxy.csproj" + - "Source/AuthProxy/Properties/AssemblyInfo.cs" + - "Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/**" + - "Source/AuthProxy.Integration.Specs/**" + - "Directory.*" + - "global.json" + - "NuGet.config" + - "AuthProxy.slnx" + - ".github/workflows/redis-replay-specs.yml" + - ".github/workflows/pull-requests.yml" dotnet: - "Source/**" - ".github/workflows/**" @@ -91,6 +107,13 @@ jobs: permissions: contents: read + redis-replay-specs: + needs: changes + if: ${{ needs.changes.outputs.redis-replay == 'true' }} + uses: ./.github/workflows/redis-replay-specs.yml + permissions: + contents: read + # No filter: the security checks must always run (their old trigger had no paths). security: uses: ./.github/workflows/security.yml @@ -108,7 +131,7 @@ jobs: gate: name: gate if: ${{ always() }} - needs: [changes, dotnet-build, prerelease, azure-docs-specs, consumer-e2e, security, policy] + needs: [changes, dotnet-build, prerelease, azure-docs-specs, consumer-e2e, redis-replay-specs, security, policy] runs-on: ${{ vars.RUNNER_GATE || 'ubuntu-latest' }} timeout-minutes: 5 permissions: {} diff --git a/.github/workflows/redis-replay-specs.yml b/.github/workflows/redis-replay-specs.yml new file mode 100644 index 00000000..b4726e14 --- /dev/null +++ b/.github/workflows/redis-replay-specs.yml @@ -0,0 +1,38 @@ +# Copyright (c) Cratis. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +name: Redis replay specs + +on: + workflow_call: + +permissions: + contents: read + +jobs: + redis-replay-specs: + name: Redis replay specs + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + global-json-file: global.json + - name: Restore integration specs + run: dotnet restore Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj + - name: Build integration specs + run: dotnet build Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj -c Release --no-restore -warnaserror + - name: Run Redis integration specs + run: dotnet test Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj -c Release --no-build --logger trx --results-directory TestResults/redis-replay + - name: Upload test results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: redis-replay-results + path: TestResults/redis-replay + if-no-files-found: warn diff --git a/AuthProxy.slnx b/AuthProxy.slnx index 8ecf6935..1aa01dc9 100644 --- a/AuthProxy.slnx +++ b/AuthProxy.slnx @@ -6,6 +6,7 @@ + diff --git a/Directory.Packages.props b/Directory.Packages.props index bb8df3fa..017d4bcd 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -34,6 +34,7 @@ + diff --git a/Documentation/configuration/dpop.md b/Documentation/configuration/dpop.md index 7e0c44be..6eb65258 100644 --- a/Documentation/configuration/dpop.md +++ b/Documentation/configuration/dpop.md @@ -150,7 +150,9 @@ create a new proof and `jti` when retrying a nonce challenge. DPoP refusals are non-cacheable machine responses, never redirects or cookie challenges. Proof/store/clock/replay failures use HTTP 401 `invalid_dpop_proof`; -valid proofs lacking scope use HTTP 403 `insufficient_scope`. DPoP challenges +valid proofs lacking scope use HTTP 403 `insufficient_scope` after replay-store +health and any required nonce are checked, without reserving the proof identifier. +Successful authorization still requires an atomic replay reservation. DPoP challenges advertise `algs="ES256"` and do not suggest downgrading to Bearer. Logs contain fixed reason codes only, not tokens, proofs, keys, nonces or full request URLs. diff --git a/Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj b/Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj new file mode 100644 index 00000000..40da3842 --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj @@ -0,0 +1,20 @@ + + + net11.0 + false + true + Exe + true + + + + + + + + + + + + + diff --git a/Source/AuthProxy.Integration.Specs/GlobalUsings.cs b/Source/AuthProxy.Integration.Specs/GlobalUsings.cs new file mode 100644 index 00000000..04f5f5c1 --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/GlobalUsings.cs @@ -0,0 +1,5 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +global using Cratis.Specifications; +global using Xunit; diff --git a/Source/AuthProxy.Integration.Specs/RedisReplayCollection.cs b/Source/AuthProxy.Integration.Specs/RedisReplayCollection.cs new file mode 100644 index 00000000..54f0385d --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/RedisReplayCollection.cs @@ -0,0 +1,12 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Cratis.AuthProxy.Integration.Specs; + +/// Shares one Redis container across serialized replay scenarios. +[CollectionDefinition(Name)] +public class RedisReplayCollection : ICollectionFixture +{ + /// The shared collection name. + public const string Name = "Redis DPoP replay"; +} diff --git a/Source/AuthProxy.Integration.Specs/RedisReplayFixture.cs b/Source/AuthProxy.Integration.Specs/RedisReplayFixture.cs new file mode 100644 index 00000000..c9ad0f99 --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/RedisReplayFixture.cs @@ -0,0 +1,46 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using StackExchange.Redis; +using Testcontainers.Redis; + +namespace Cratis.AuthProxy.Integration.Specs; + +/// Owns a disposable Redis process with persistence disabled. +public sealed class RedisReplayFixture : IAsyncLifetime +{ + readonly RedisContainer _container = new RedisBuilder("redis:7.4.2-alpine") + .WithCommand("redis-server", "--save", string.Empty, "--appendonly", "no", "--maxmemory-policy", "noeviction") + .Build(); + ConnectionMultiplexer _connection = null!; + + /// Gets the endpoint for independent production-store connections. + public string Endpoint => _container.GetConnectionString(); + + /// Gets the database used to inspect replay evidence. + public IDatabase Database => _connection.GetDatabase(); + + /// + public async ValueTask InitializeAsync() + { + await _container.StartAsync(); + _connection = await ConnectionMultiplexer.ConnectAsync(Endpoint); + } + + /// Restarts Redis without preserving in-memory replay evidence. + /// The restart operation. + public async Task Restart() + { + await _container.StopAsync(); + await _container.StartAsync(); + await _connection.DisposeAsync(); + _connection = await ConnectionMultiplexer.ConnectAsync(Endpoint); + } + + /// + public async ValueTask DisposeAsync() + { + await _connection.DisposeAsync(); + await _container.DisposeAsync(); + } +} diff --git a/Source/AuthProxy.Integration.Specs/given_a_redis_replay_store.cs b/Source/AuthProxy.Integration.Specs/given_a_redis_replay_store.cs new file mode 100644 index 00000000..c2620b1e --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/given_a_redis_replay_store.cs @@ -0,0 +1,75 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.AuthProxy.BearerRoutes.Dpop; +using Microsoft.Extensions.Options; +using StackExchange.Redis; +using C = Cratis.AuthProxy.Configuration; + +namespace Cratis.AuthProxy.Integration.Specs; + +/// Isolates each replay scenario behind a unique Redis key prefix. +/// The shared Redis process. +public class given_a_redis_replay_store(RedisReplayFixture redis) : Specification +{ + const string Origin = "https://api.example.test"; + const string Thumbprint = "synthetic-thumbprint"; + readonly List _stores = []; + readonly string _prefix = "dpop-spec-" + Guid.NewGuid().ToString("N"); + + /// Gets the Redis process lifecycle. + protected RedisReplayFixture Redis => redis; + + /// Gets the database for evidence inspection and coverage setup. + protected IDatabase Database => redis.Database; + + /// Gets this scenario's coverage marker. + protected RedisKey CoverageKey => "{" + _prefix + "}:coverage"; + + /// Creates an independent production replay-store replica. + /// The replica clock, or the system clock. + /// The replay store. + protected IDpopReplayStore Replica(TimeProvider? clock = null) + { + var options = new C.AuthProxy(); + options.Dpop.ReplayStore.Redis = new() { ConnectionString = redis.Endpoint, KeyPrefix = _prefix }; + var store = new RedisDpopReplayStore(Options.Create(options), clock ?? TimeProvider.System); + _stores.Add(store); + return store; + } + + /// Disposes a replica without changing Redis evidence. + /// The replica to close. + protected void StopReplica(IDpopReplayStore store) + { + var instance = (RedisDpopReplayStore)store; + _stores.Remove(instance); + instance.Dispose(); + } + + /// Sets coverage age relative to Redis TIME, without waiting a retention window. + /// The age of the coverage marker. + /// The marker write operation. + protected async Task SeedCoverage(TimeSpan age) + { + var milliseconds = (long)await Database.ScriptEvaluateAsync("local t = redis.call('TIME'); return t[1] * 1000 + math.floor(t[2] / 1000)"); + await Database.StringSetAsync(CoverageKey, milliseconds - (long)age.TotalMilliseconds); + } + + /// Gets the production-format key for a synthetic proof identifier. + /// The synthetic proof identifier. + /// The replay key. + protected RedisKey ReplayKey(string jti) => "{" + _prefix + "}:" + DpopEncoding.Hash("resource\0" + Origin + "\0" + Thumbprint + "\0" + jti); + + /// Reserves a synthetic proof through the production Lua path. + /// The independent replica. + /// The synthetic proof identifier. + /// The reservation result. + protected Task Consume(IDpopReplayStore store, string jti = "synthetic-proof") => store.TryConsume(Origin, Thumbprint, jti, CancellationToken.None); + + /// Closes all scenario-owned replica connections. + protected void Destroy() + { + foreach (var store in _stores) store.Dispose(); + } +} diff --git a/Source/AuthProxy.Integration.Specs/when_a_replica_clock_disagrees_with_redis.cs b/Source/AuthProxy.Integration.Specs/when_a_replica_clock_disagrees_with_redis.cs new file mode 100644 index 00000000..3dad3979 --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/when_a_replica_clock_disagrees_with_redis.cs @@ -0,0 +1,47 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.AuthProxy.BearerRoutes.Dpop; + +namespace Cratis.AuthProxy.Integration.Specs; + +/// Exercises Redis TIME checks independently of coverage readiness. +/// The shared Redis process. +[Collection(RedisReplayCollection.Name)] +public class when_a_replica_clock_disagrees_with_redis(RedisReplayFixture redis) : given_a_redis_replay_store(redis) +{ + /// Skew beyond thirty seconds refuses health and reservation without evidence. + /// The clock offset in seconds. + /// The scenario operation. + [Theory] + [InlineData(31)] + [InlineData(-31)] + public async Task should_refuse_health_and_reservation_outside_thirty_seconds(int seconds) + { + await SeedCoverage(TimeSpan.FromSeconds(391)); + var store = Replica(new OffsetClock(seconds)); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Unavailable); + (await Consume(store)).ShouldEqual(DpopReplayResult.Unavailable); + (await Database.KeyExistsAsync(ReplayKey("synthetic-proof"))).ShouldBeFalse(); + } + + /// A reservation rechecks skew even when cached health remains ready. + /// The scenario operation. + [Fact] + public async Task should_check_clock_skew_again_despite_cached_health() + { + await SeedCoverage(TimeSpan.FromSeconds(391)); + var clock = new OffsetClock(0); + var store = Replica(clock); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + clock.Seconds = 31; + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + (await Consume(store)).ShouldEqual(DpopReplayResult.Unavailable); + } + + sealed class OffsetClock(int seconds) : TimeProvider + { + public int Seconds { get; set; } = seconds; + public override DateTimeOffset GetUtcNow() => DateTimeOffset.UtcNow.AddSeconds(Seconds); + } +} diff --git a/Source/AuthProxy.Integration.Specs/when_redis_restarts_without_replay_evidence.cs b/Source/AuthProxy.Integration.Specs/when_redis_restarts_without_replay_evidence.cs new file mode 100644 index 00000000..023459ed --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/when_redis_restarts_without_replay_evidence.cs @@ -0,0 +1,32 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.AuthProxy.BearerRoutes.Dpop; + +namespace Cratis.AuthProxy.Integration.Specs; + +/// Exercises loss of replay evidence during a real Redis restart. +/// The shared Redis process. +[Collection(RedisReplayCollection.Name)] +public class when_redis_restarts_without_replay_evidence(RedisReplayFixture redis) : given_a_redis_replay_store(redis) +{ + /// Old and new replicas fail closed until full coverage is restored. + /// The scenario operation. + [Fact] + public async Task should_quarantine_restarted_redis_instead_of_reusing_cached_health() + { + await SeedCoverage(TimeSpan.FromSeconds(391)); + var warmed = Replica(); + (await warmed.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + (await Consume(warmed)).ShouldEqual(DpopReplayResult.Accepted); + await Redis.Restart(); + (await Database.KeyExistsAsync(CoverageKey)).ShouldBeFalse(); + (await Database.KeyExistsAsync(ReplayKey("synthetic-proof"))).ShouldBeFalse(); + (await Consume(warmed)).ShouldEqual(DpopReplayResult.Unavailable); + var restarted = Replica(); + (await restarted.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Unavailable); + (await Consume(restarted)).ShouldEqual(DpopReplayResult.Unavailable); + await SeedCoverage(TimeSpan.FromSeconds(391)); + (await Consume(restarted)).ShouldEqual(DpopReplayResult.Accepted); + } +} diff --git a/Source/AuthProxy.Integration.Specs/when_replay_coverage_is_quarantined.cs b/Source/AuthProxy.Integration.Specs/when_replay_coverage_is_quarantined.cs new file mode 100644 index 00000000..c05c5e7b --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/when_replay_coverage_is_quarantined.cs @@ -0,0 +1,41 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.AuthProxy.BearerRoutes.Dpop; + +namespace Cratis.AuthProxy.Integration.Specs; + +/// Exercises coverage quarantine on the real Redis script. +/// The shared Redis process. +[Collection(RedisReplayCollection.Name)] +public class when_replay_coverage_is_quarantined(RedisReplayFixture redis) : given_a_redis_replay_store(redis) +{ + /// Reservations require a known full replay-retention window. + /// The scenario operation. + [Fact] + public async Task should_refuse_missing_and_young_coverage_until_a_full_retention_window_is_known() + { + var store = Replica(); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Unavailable); + (await Database.KeyExistsAsync(CoverageKey)).ShouldBeTrue(); + (await Consume(store)).ShouldEqual(DpopReplayResult.Unavailable); + await SeedCoverage(TimeSpan.FromSeconds(380)); + (await Consume(store)).ShouldEqual(DpopReplayResult.Unavailable); + await SeedCoverage(TimeSpan.FromSeconds(391)); + (await Consume(store)).ShouldEqual(DpopReplayResult.Accepted); + } + + /// Cached readiness never bypasses the atomic coverage check. + /// The scenario operation. + [Fact] + public async Task should_not_let_cached_ready_health_bypass_lost_coverage() + { + await SeedCoverage(TimeSpan.FromSeconds(391)); + var store = Replica(); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + await Database.KeyDeleteAsync(CoverageKey); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + (await Consume(store)).ShouldEqual(DpopReplayResult.Unavailable); + (await Database.KeyExistsAsync(ReplayKey("synthetic-proof"))).ShouldBeFalse(); + } +} diff --git a/Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs b/Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs new file mode 100644 index 00000000..4e546c7c --- /dev/null +++ b/Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs @@ -0,0 +1,42 @@ +// Copyright (c) Cratis. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Cratis.AuthProxy.BearerRoutes.Dpop; + +namespace Cratis.AuthProxy.Integration.Specs; + +/// Exercises atomic replay admission through independent connections. +/// The shared Redis process. +[Collection(RedisReplayCollection.Name)] +public class when_reserving_a_proof_across_replicas(RedisReplayFixture redis) : given_a_redis_replay_store(redis) +{ + /// Only one competing reservation succeeds and replays do not extend its TTL. + /// The scenario operation. + [Fact] + public async Task should_admit_exactly_one_contender_and_preserve_the_reservation_ttl() + { + await SeedCoverage(TimeSpan.FromSeconds(391)); + var replicas = new[] { Replica(), Replica() }; + var results = await Task.WhenAll(Enumerable.Range(0, 32).Select(index => Consume(replicas[index % 2]))); + results.Count(result => result == DpopReplayResult.Accepted).ShouldEqual(1); + results.Count(result => result == DpopReplayResult.Replay).ShouldEqual(31); + var ttl = await Database.KeyTimeToLiveAsync(ReplayKey("synthetic-proof")); + ttl.ShouldNotBeNull(); + (ttl.GetValueOrDefault() > TimeSpan.Zero && ttl.GetValueOrDefault() <= TimeSpan.FromSeconds(390)).ShouldBeTrue(); + (await Consume(replicas[1])).ShouldEqual(DpopReplayResult.Replay); + var remaining = await Database.KeyTimeToLiveAsync(ReplayKey("synthetic-proof")); + (remaining <= ttl).ShouldBeTrue(); + } + + /// A new replica observes evidence left by a disposed replica. + /// The scenario operation. + [Fact] + public async Task should_preserve_replay_evidence_when_a_replica_is_recreated() + { + await SeedCoverage(TimeSpan.FromSeconds(391)); + var original = Replica(); + (await Consume(original, "recreated-proof")).ShouldEqual(DpopReplayResult.Accepted); + StopReplica(original); + (await Consume(Replica(), "recreated-proof")).ShouldEqual(DpopReplayResult.Replay); + } +} diff --git a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs index ff820e34..e05470cc 100644 --- a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs +++ b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs @@ -60,6 +60,121 @@ public async Task should_not_let_cached_health_bypass_reservation_guards(int res await database.Received(2).ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } + [Theory] + [InlineData(false, 10, 0)] + [InlineData(true, 10, 0)] + [InlineData(false, 0, 31)] + [InlineData(true, 0, 31)] + [InlineData(false, 0, -31)] + [InlineData(true, 0, -31)] + public async Task should_refuse_replies_that_arrive_after_the_clock_or_freshness_boundary(bool reservation, int elapsed, int offset) + { + var database = Substitute.For(); + var clock = new ManualClock(); + var reply = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(reply.Task); + using var store = CreateStore(database, clock); + var pending = reservation + ? store.TryConsume("https://api.example.test", "synthetic-key", "synthetic-proof", CancellationToken.None) + : store.CheckHealth(CancellationToken.None); + clock.Elapsed = elapsed; + clock.Offset = offset; + reply.SetResult(Result(1)); + (await pending).ShouldEqual(DpopReplayResult.Unavailable); + } + + [Fact] + public async Task should_measure_cached_health_from_the_operation_start() + { + var database = Substitute.For(); + var clock = new ManualClock(); + var reply = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(reply.Task); + using var store = CreateStore(database, clock); + var pending = store.CheckHealth(CancellationToken.None); + clock.Elapsed = 9; + reply.SetResult(Result(1)); + (await pending).ShouldEqual(DpopReplayResult.Accepted); + clock.Elapsed = 10; + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result(1, Now + 10))); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + await database.Received(2).ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Theory] + [InlineData(null, "1800000000000")] + [InlineData("1", null)] + [InlineData("invalid", "1800000000000")] + [InlineData("1", "invalid")] + [InlineData("99999999999999999999999999", "1800000000000")] + [InlineData("1", "99999999999999999999999999")] + public async Task should_refuse_null_or_non_numeric_reply_fields(string? status, string? serverTime) + { + var database = Substitute.For(); + var result = RedisResult.Create([RedisResult.Create((RedisValue)status), RedisResult.Create((RedisValue)serverTime)]); + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(Task.FromResult(result)); + using var store = CreateStore(database, new ManualClock()); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Unavailable); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(3)] + public async Task should_refuse_an_unexpected_reply_shape(int length) + { + var database = Substitute.For(); + var result = RedisResult.Create(Enumerable.Repeat(RedisResult.Create((RedisValue)1), length).ToArray()); + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(Task.FromResult(result)); + using var store = CreateStore(database, new ManualClock()); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Unavailable); + } + + [Theory] + [InlineData(0, false)] + [InlineData(0, true)] + [InlineData(1, false)] + [InlineData(1, true)] + [InlineData(2, false)] + [InlineData(2, true)] + public async Task should_fail_closed_on_transport_or_script_errors_and_recover(int failure, bool reservation) + { + var database = Substitute.For(); + Exception error = failure switch + { + 0 => new RedisConnectionException(ConnectionFailureType.UnableToConnect, "synthetic outage"), + 1 => new TimeoutException("synthetic timeout"), + _ => new RedisServerException("synthetic script refusal"), + }; + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(Task.FromException(error)); + var clock = new ManualClock(); + using var store = CreateStore(database, clock); + var result = reservation + ? await store.TryConsume("https://api.example.test", "synthetic-key", "synthetic-proof", CancellationToken.None) + : await store.CheckHealth(CancellationToken.None); + result.ShouldEqual(DpopReplayResult.Unavailable); + clock.Elapsed = 10; + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result(1, Now + 10))); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + } + + [Fact] + public async Task should_recover_after_a_cancelled_probe_without_publishing_health() + { + var database = Substitute.For(); + var reply = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(reply.Task); + var clock = new ManualClock(); + using var store = CreateStore(database, clock); + using var cancellation = new CancellationTokenSource(); + var pending = store.CheckHealth(cancellation.Token); + await cancellation.CancelAsync(); + (await pending).ShouldEqual(DpopReplayResult.Unavailable); + reply.SetResult(Result(1)); + clock.Elapsed = 10; + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + } + static RedisResult Result(int status, long seconds = Now) => RedisResult.Create([RedisResult.Create((RedisValue)status), RedisResult.Create((RedisValue)(seconds * 1000))]); static RedisDpopReplayStore CreateStore(IDatabase database, TimeProvider clock) diff --git a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_mapping_exact_target_paths.cs b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_mapping_exact_target_paths.cs index ea0679d9..989514e7 100644 --- a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_mapping_exact_target_paths.cs +++ b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_mapping_exact_target_paths.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Cratis.AuthProxy.BearerRoutes.Dpop; +using Microsoft.AspNetCore.Http.Features; namespace Cratis.AuthProxy.BearerRoutes.for_DpopProofVerifier; @@ -23,6 +24,44 @@ public void should_only_substitute_an_exact_mount_boundary(string path, string? if (target is not null) actual.ShouldEqual(target); } + [Theory] + [InlineData("/oauth", "/oauth?query=ignored", "https://api.example.test/mcp")] + [InlineData("/oauth/", "/oauth/", "https://api.example.test/mcp/")] + [InlineData("/oauth/a%2Fb", "/oauth/a%2Fb", "https://api.example.test/mcp/a%2Fb")] + [InlineData("/oauth/a%2fb", "/oauth/a%2fb", "https://api.example.test/mcp/a%2fb")] + [InlineData("/oauth/a", "/oauth/../a", null)] + [InlineData("/oauth/a", "/oauth/%2e%2e/a", null)] + [InlineData("/oauth/a", "/oauth//a", null)] + [InlineData("/oauth/a", "/oauth/a%", null)] + [InlineData("/oauth/a", "/oauth/a%GG", null)] + [InlineData("/oauth/a", "/oauth/a%5C", null)] + [InlineData("/oauth/a", "/oauth/a%1F", null)] + public void should_build_from_the_trusted_origin_and_exact_escaped_path(string path, string rawTarget, string? target) + { + var context = Request(); + context.Request.Path = path; + context.Request.Host = new("attacker.example.test"); + context.Request.Headers["Forwarded"] = "host=attacker.example.test;proto=http"; + context.Request.Headers["X-Forwarded-Host"] = "attacker.example.test"; + context.Features.Get()!.RawTarget = rawTarget; + DpopTargetUri.TryBuild(context.Request, Policy, out var actual).ShouldEqual(target is not null); + if (target is not null) actual.ShouldEqual(target); + } + + [Fact] + public void should_include_path_base_without_an_explicit_mount_mapping() + { + var context = Request(); + Policy.Dpop!.PublicPathPrefix = null; + context.Request.PathBase = "/gateway"; + context.Request.Path = "/oauth/a%2Fb"; + DpopTargetUri.TryBuild(context.Request, Policy, out var target).ShouldBeTrue(); + target.ShouldEqual("https://api.example.test/gateway/oauth/a%2Fb"); + DpopTargetUri.Matches("https://api.example.test/gateway/oauth/a%2fb", target).ShouldBeFalse(); + DpopTargetUri.Matches(target + "?query=ignored", target).ShouldBeFalse(); + DpopTargetUri.Matches(target + "#fragment", target).ShouldBeFalse(); + } + [Theory] [InlineData("/OAUTH")] [InlineData("/OAUTH/action")] diff --git a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_proof_precedes_scope_step_up.cs b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_proof_precedes_scope_step_up.cs index a8ec1589..b715c166 100644 --- a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_proof_precedes_scope_step_up.cs +++ b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_proof_precedes_scope_step_up.cs @@ -14,11 +14,40 @@ void Establish() [Fact] public async Task should_use_a_dpop_scope_challenge_only_after_a_valid_proof() { + var checks = Store.HealthChecks; var request = Request(); await Gate.InvokeAsync(request); Refused(request, 403, "insufficient_scope"); request.Response.Headers.WWWAuthenticate.ToString().ShouldContain("DPoP"); - Store.Count.ShouldEqual(1); + Store.HealthChecks.ShouldEqual(checks + 1); + Store.Consumes.ShouldEqual(0); + Store.Count.ShouldEqual(0); + } + + [Fact] + public async Task should_require_a_ready_store_before_suggesting_broader_scope() + { + Store.Available = false; + var request = Request(); + await Gate.InvokeAsync(request); + Refused(request, 401); + Store.Consumes.ShouldEqual(0); + Store.Available = true; + } + + [Fact] + public async Task should_require_a_nonce_before_suggesting_broader_scope() + { + Options.Dpop.NonceRequired = true; + var request = Request(); + await Gate.InvokeAsync(request); + Refused(request, 401, "use_dpop_nonce"); + Payload["nonce"] = request.Response.Headers["DPoP-Nonce"].ToString(); + var retry = Request(); + await Gate.InvokeAsync(retry); + Refused(retry, 403, "insufficient_scope"); + Store.Consumes.ShouldEqual(0); + Options.Dpop.NonceRequired = false; } [Fact] diff --git a/Source/AuthProxy/BearerRoutes/Dpop/DpopEnforcement.cs b/Source/AuthProxy/BearerRoutes/Dpop/DpopEnforcement.cs index 33ab5fce..3d09fff9 100644 --- a/Source/AuthProxy/BearerRoutes/Dpop/DpopEnforcement.cs +++ b/Source/AuthProxy/BearerRoutes/Dpop/DpopEnforcement.cs @@ -82,6 +82,7 @@ internal static void PrepareForward(HttpContext context, BearerTokenValidation v DpopChallenge.Refuse(context, route, "use_dpop_nonce", new(Nonce: _nonce.Create(route, jkt))); return null; } + if (validation.Status == BearerTokenValidationStatus.InsufficientScope) return validation; return await Consume(context, route, validation, proof); } diff --git a/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs b/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs index aa1030c4..4057c327 100644 --- a/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs +++ b/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs @@ -41,8 +41,9 @@ public async Task CheckHealth(CancellationToken cancellationTo try { if (_healthCheckedAt is { } checkedAt && clock.GetElapsedTime(checkedAt, clock.GetTimestamp()) < _healthInterval) return _healthResult; + var startedAt = clock.GetTimestamp(); _healthResult = await Execute(null, cancellationToken); - _healthCheckedAt = clock.GetTimestamp(); + _healthCheckedAt = startedAt; return _healthResult; } finally @@ -90,23 +91,38 @@ async Task Execute(string? key, CancellationToken cancellation var replayDatabase = database ?? (await Connection(cancellationToken)).GetDatabase(); if (database is null && _connection?.IsConnected != true) return DpopReplayResult.Unavailable; var prefix = "{" + settings.KeyPrefix + "}:"; + var startedAt = clock.GetTimestamp(); var local = clock.GetUtcNow().ToUnixTimeMilliseconds(); var operation = replayDatabase.ScriptEvaluateAsync( Script, [prefix + "coverage", prefix + (key ?? "health")], [local, key is null ? "0" : "1"]); var result = (RedisResult[]?)await operation.WaitAsync(cancellationToken); - if (result is not { Length: 2 } || Math.Abs((decimal)(long)result[1] - local) > 30000) return DpopReplayResult.Unavailable; - return (int)result[0] switch - { - 1 => DpopReplayResult.Accepted, - 2 => DpopReplayResult.Replay, - _ => DpopReplayResult.Unavailable, - }; + return ReadResult(result, local, startedAt); } - catch (Exception exception) when (exception is RedisException or TimeoutException or ArgumentException or InvalidOperationException or OperationCanceledException or InvalidCastException) + catch (Exception exception) when (exception is RedisException or TimeoutException or ArgumentException or InvalidOperationException or OperationCanceledException or InvalidCastException or FormatException or OverflowException) { return DpopReplayResult.Unavailable; } } + + DpopReplayResult ReadResult(RedisResult[]? result, long local, long startedAt) + { + if (result is not { Length: 2 }) return DpopReplayResult.Unavailable; + if (result.Any(field => field.IsNull)) return DpopReplayResult.Unavailable; + if (clock.GetElapsedTime(startedAt, clock.GetTimestamp()) >= _healthInterval) return DpopReplayResult.Unavailable; + if (!ClockIsCurrent((long)result[1], local)) return DpopReplayResult.Unavailable; + return (int)result[0] switch + { + 1 => DpopReplayResult.Accepted, + 2 => DpopReplayResult.Replay, + _ => DpopReplayResult.Unavailable, + }; + } + + bool ClockIsCurrent(decimal serverTime, long startedAt) + { + var completedAt = clock.GetUtcNow().ToUnixTimeMilliseconds(); + return Math.Abs(serverTime - startedAt) <= 30000 && Math.Abs(serverTime - completedAt) <= 30000; + } } diff --git a/Source/AuthProxy/Properties/AssemblyInfo.cs b/Source/AuthProxy/Properties/AssemblyInfo.cs index a6c447ec..bf40da70 100644 --- a/Source/AuthProxy/Properties/AssemblyInfo.cs +++ b/Source/AuthProxy/Properties/AssemblyInfo.cs @@ -5,3 +5,4 @@ [assembly: InternalsVisibleTo("AuthProxy.Specs")] [assembly: InternalsVisibleTo("AuthProxy.Security.Specs")] +[assembly: InternalsVisibleTo("AuthProxy.Integration.Specs")] From 111b4ad2cb0010bd9a120ebc2cbd90e4596767c9 Mon Sep 17 00:00:00 2001 From: woksin Date: Thu, 12 Nov 2026 20:35:21 +0100 Subject: [PATCH 2/2] Bound DPoP health-lock waits and isolate Redis test execution --- .github/workflows/redis-replay-specs.yml | 6 +++--- .../AuthProxy.Integration.Specs.csproj | 4 +++- .../when_reserving_a_proof_across_replicas.cs | 5 ++++- .../when_caching_replay_store_health.cs | 21 +++++++++++++++++-- .../BearerRoutes/Dpop/RedisDpopReplayStore.cs | 6 ++++-- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/redis-replay-specs.yml b/.github/workflows/redis-replay-specs.yml index b4726e14..eaf2eeac 100644 --- a/.github/workflows/redis-replay-specs.yml +++ b/.github/workflows/redis-replay-specs.yml @@ -24,11 +24,11 @@ jobs: with: global-json-file: global.json - name: Restore integration specs - run: dotnet restore Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj + run: dotnet restore Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj -p:RunRedisReplaySpecs=true - name: Build integration specs - run: dotnet build Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj -c Release --no-restore -warnaserror + run: dotnet build Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj -c Release --no-restore -warnaserror -p:RunRedisReplaySpecs=true - name: Run Redis integration specs - run: dotnet test Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj -c Release --no-build --logger trx --results-directory TestResults/redis-replay + run: dotnet test Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj -c Release --no-build -p:RunRedisReplaySpecs=true --logger trx --results-directory TestResults/redis-replay - name: Upload test results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj b/Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj index 40da3842..7d323c65 100644 --- a/Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj +++ b/Source/AuthProxy.Integration.Specs/AuthProxy.Integration.Specs.csproj @@ -2,7 +2,9 @@ net11.0 false - true + + false + true Exe true diff --git a/Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs b/Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs index 4e546c7c..31ef6035 100644 --- a/Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs +++ b/Source/AuthProxy.Integration.Specs/when_reserving_a_proof_across_replicas.cs @@ -23,9 +23,12 @@ public async Task should_admit_exactly_one_contender_and_preserve_the_reservatio var ttl = await Database.KeyTimeToLiveAsync(ReplayKey("synthetic-proof")); ttl.ShouldNotBeNull(); (ttl.GetValueOrDefault() > TimeSpan.Zero && ttl.GetValueOrDefault() <= TimeSpan.FromSeconds(390)).ShouldBeTrue(); + var shortenedRetention = TimeSpan.FromSeconds(60); + (await Database.KeyExpireAsync(ReplayKey("synthetic-proof"), shortenedRetention)).ShouldBeTrue(); (await Consume(replicas[1])).ShouldEqual(DpopReplayResult.Replay); var remaining = await Database.KeyTimeToLiveAsync(ReplayKey("synthetic-proof")); - (remaining <= ttl).ShouldBeTrue(); + remaining.ShouldNotBeNull(); + (remaining > TimeSpan.Zero && remaining <= shortenedRetention).ShouldBeTrue(); } /// A new replica observes evidence left by a disposed replica. diff --git a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs index e05470cc..df9188a2 100644 --- a/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs +++ b/Source/AuthProxy.Specs/BearerRoutes/for_DpopProofVerifier/when_caching_replay_store_health.cs @@ -41,6 +41,23 @@ public async Task should_share_the_inflight_health_check_between_concurrent_requ await database.Received(1).ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } + [Fact] + public async Task should_fail_closed_when_the_health_lock_times_out_without_releasing_the_owner() + { + var database = Substitute.For(); + var reply = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(reply.Task); + using var store = CreateStore(database, new ManualClock(), TimeSpan.Zero); + var owner = store.CheckHealth(CancellationToken.None); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Unavailable); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Unavailable); + owner.IsCompleted.ShouldBeFalse(); + reply.SetResult(Result(1)); + (await owner).ShouldEqual(DpopReplayResult.Accepted); + (await store.CheckHealth(CancellationToken.None)).ShouldEqual(DpopReplayResult.Accepted); + await database.Received(1).ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); + } + [Theory] [InlineData(0, 0)] [InlineData(1, 31)] @@ -177,11 +194,11 @@ public async Task should_recover_after_a_cancelled_probe_without_publishing_heal static RedisResult Result(int status, long seconds = Now) => RedisResult.Create([RedisResult.Create((RedisValue)status), RedisResult.Create((RedisValue)(seconds * 1000))]); - static RedisDpopReplayStore CreateStore(IDatabase database, TimeProvider clock) + static RedisDpopReplayStore CreateStore(IDatabase database, TimeProvider clock, TimeSpan? healthLockTimeout = null) { var options = new C.AuthProxy(); options.Dpop.ReplayStore.Redis = new() { ConnectionString = "replay.example.test:6379", KeyPrefix = "synthetic-prefix" }; - return new(Microsoft.Extensions.Options.Options.Create(options), clock, database); + return new(Microsoft.Extensions.Options.Options.Create(options), clock, database, healthLockTimeout); } sealed class ManualClock : TimeProvider diff --git a/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs b/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs index 4057c327..821064ae 100644 --- a/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs +++ b/Source/AuthProxy/BearerRoutes/Dpop/RedisDpopReplayStore.cs @@ -11,7 +11,8 @@ namespace Cratis.AuthProxy.BearerRoutes.Dpop; /// The deployment's dedicated Redis settings. /// The local clock compared with Redis on every operation. /// An optional database supplied by deterministic specifications. -internal sealed class RedisDpopReplayStore(IOptions options, TimeProvider clock, IDatabase? database = null) : IDpopReplayStore, IDisposable +/// An optional health-lock timeout supplied by deterministic specifications. +internal sealed class RedisDpopReplayStore(IOptions options, TimeProvider clock, IDatabase? database = null, TimeSpan? healthLockTimeout = null) : IDpopReplayStore, IDisposable { /// The atomic Redis time, coverage, quarantine and SET NX operation; every key shares a cluster slot. internal const string Script = @@ -30,6 +31,7 @@ internal sealed class RedisDpopReplayStore(IOptions options, TimePr static readonly TimeSpan _healthInterval = TimeSpan.FromSeconds(10); readonly SemaphoreSlim _connect = new(1, 1); readonly SemaphoreSlim _healthLock = new(1, 1); + readonly TimeSpan _healthLockTimeout = healthLockTimeout ?? TimeSpan.FromSeconds(5); long? _healthCheckedAt; DpopReplayResult _healthResult = DpopReplayResult.Unavailable; ConnectionMultiplexer? _connection; @@ -37,7 +39,7 @@ internal sealed class RedisDpopReplayStore(IOptions options, TimePr /// public async Task CheckHealth(CancellationToken cancellationToken) { - await _healthLock.WaitAsync(cancellationToken); + if (!await _healthLock.WaitAsync(_healthLockTimeout, cancellationToken)) return DpopReplayResult.Unavailable; try { if (_healthCheckedAt is { } checkedAt && clock.GetElapsedTime(checkedAt, clock.GetTimestamp()) < _healthInterval) return _healthResult;