Retain OAuth certs across cert-manager renewal without mass logout

backend/LexBoxApi/Auth/AuthKernel.cs

@@ -1,6 +1,5 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Net.Http.Headers;
-using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using LexBoxApi.Auth.Attributes;
 using LexBoxApi.Auth.Requirements;
@@ -283,6 +282,9 @@ public static void AddOpenId(IServiceCollection services, IHostEnvironment envir
                 });
 
                 options.SetAccessTokenLifetime(TimeSpan.FromHours(1));
+                //note, if we want to change this, then we also need to change the certificate lifetime so we don't have tokens that last longer than the certificate
+                //basically this means that the renewBefore should be increased to the token lifetime + 1, just be careful we don't get to close to the
+                //certificate legnth. If the cert is only good for 90 days and we renew too soon then it won't actually be valid for very long.
                 options.SetRefreshTokenLifetime(TimeSpan.FromDays(14));
                 options.AllowAuthorizationCodeFlow()
                     .AllowRefreshTokenFlow();
@@ -294,23 +296,9 @@ public static void AddOpenId(IServiceCollection services, IHostEnvironment envir
                     options.AddDevelopmentEncryptionCertificate();
                     options.AddDevelopmentSigningCertificate();
                 }
-                else
-                {
-                    //see docs: https://documentation.openiddict.com/configuration/encryption-and-signing-credentials.html
-                    //todo, handle certificate rotation, right now these certs will be replaced 15 days before they expire
-                    //however we need to start signing with the new cert before the old one expires
-                    //this means we need 2 certs for each use case, an old one for tokens that have not expired yet, and a new one to sign all new tokens
-                    var encryptionCert = X509Certificate2.CreateFromPemFile(
-                        "/oauth-certs/encryption/tls.crt",
-                        "/oauth-certs/encryption/tls.key");
-                    options.AddEncryptionCertificate(encryptionCert);
-
-                    var signingCert = X509Certificate2.CreateFromPemFile(
-                        "/oauth-certs/signing/tls.crt",
-                        "/oauth-certs/signing/tls.key");
-
-                    options.AddSigningCertificate(signingCert);
-                }
+                // Non-development PEMs: OauthPemOpenIddictServerConfigurer registers current + optional
+                // last-seen/previous slots. See docs:
+                // https://documentation.openiddict.com/configuration/encryption-and-signing-credentials.html
 
                 var aspnetCoreBuilder = options.UseAspNetCore()
                     .EnableAuthorizationEndpointPassthrough()
@@ -329,6 +317,21 @@ public static void AddOpenId(IServiceCollection services, IHostEnvironment envir
                 options.AddAudiences(Enum.GetValues<LexboxAudience>().Where(a => a != LexboxAudience.Unknown).Select(a => a.ToString()).ToArray());
                 options.EnableAuthorizationEntryValidation();
             });
+
+        if (!environment.IsDevelopment())
+        {
+            services.AddSingleton<IConfigureOptions<OpenIddictServerOptions>, OauthPemOpenIddictServerConfigurer>();
+
+            // Content-hash poll invalidates server + validation options together so UseLocalServer()
+            // re-imports keys after PEM mount updates (k8s Secret ..data swaps).
+            services.AddSingleton<OauthPemOptionsChangeTokenSource>();
+            services.AddSingleton<IOptionsChangeTokenSource<OpenIddictServerOptions>>(
+                sp => sp.GetRequiredService<OauthPemOptionsChangeTokenSource>());
+            services.AddSingleton<IOptionsChangeTokenSource<OpenIddictValidationOptions>>(
+                sp => sp.GetRequiredService<OauthPemOptionsChangeTokenSource>());
+            services.AddHostedService(sp => sp.GetRequiredService<OauthPemOptionsChangeTokenSource>());
+        }
+
         //ensure that validation happens on startup, not on the first request which requires authentication
         services.AddOptions<OpenIddictCoreOptions>().ValidateOnStart();
         services.AddOptions<OpenIddictServerOptions>().ValidateOnStart();

backend/LexBoxApi/Auth/OauthPemCertificateLoader.cs

@@ -0,0 +1,40 @@
+using System.Security.Cryptography.X509Certificates;
+
+namespace LexBoxApi.Auth;
+
+/// <summary>
+/// Loads distinct X.509 certificates from PEM directory slots (<c>tls.crt</c> + <c>tls.key</c>).
+/// Missing directories or companion files are skipped so a single current mount still works.
+/// </summary>
+public static class OauthPemCertificateLoader
+{
+    public static IReadOnlyList<X509Certificate2> LoadDistinctFromDirectories(IEnumerable<string> directories)
+    {
+        ArgumentNullException.ThrowIfNull(directories);
+
+        var certificates = new List<X509Certificate2>();
+        var seenThumbprints = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        foreach (var directory in directories)
+        {
+            if (string.IsNullOrWhiteSpace(directory))
+                continue;
+
+            var certPath = System.IO.Path.Combine(directory, OauthPemCertificatePaths.CertFileName);
+            var keyPath = System.IO.Path.Combine(directory, OauthPemCertificatePaths.KeyFileName);
+            if (!File.Exists(certPath) || !File.Exists(keyPath))
+                continue;
+
+            var certificate = X509Certificate2.CreateFromPemFile(certPath, keyPath);
+            if (!seenThumbprints.Add(certificate.Thumbprint))
+            {
+                certificate.Dispose();
+                continue;
+            }
+
+            certificates.Add(certificate);
+        }
+
+        return certificates;
+    }
+}

backend/LexBoxApi/Auth/OauthPemCertificatePaths.cs

@@ -0,0 +1,25 @@
+namespace LexBoxApi.Auth;
+
+/// <summary>
+/// Default mount paths for OAuth signing/encryption PEM pairs (k8s TLS secret layout).
+/// Companion last-seen/previous slots are optional until the retainer CronJob creates them.
+/// </summary>
+public static class OauthPemCertificatePaths
+{
+    public const string CertFileName = "tls.crt";
+    public const string KeyFileName = "tls.key";
+
+    public static readonly string[] SigningDirectories =
+    [
+        "/oauth-certs/signing",
+        "/oauth-certs/signing-last-seen",
+        "/oauth-certs/signing-previous",
+    ];
+
+    public static readonly string[] EncryptionDirectories =
+    [
+        "/oauth-certs/encryption",
+        "/oauth-certs/encryption-last-seen",
+        "/oauth-certs/encryption-previous",
+    ];
+}

backend/LexBoxApi/Auth/OauthPemOpenIddictServerConfigurer.cs

@@ -0,0 +1,59 @@
+using System.Security.Cryptography.X509Certificates;
+using Microsoft.Extensions.Options;
+using Microsoft.IdentityModel.Tokens;
+using OpenIddict.Server;
+
+namespace LexBoxApi.Auth;
+
+/// <summary>
+/// Registers all distinct OAuth signing/encryption PEMs into OpenIddict server options.
+/// Loads inside <see cref="Configure"/> so <see cref="OauthPemOptionsChangeTokenSource"/> can recreate options on PEM change.
+/// </summary>
+public sealed class OauthPemOpenIddictServerConfigurer : IConfigureOptions<OpenIddictServerOptions>
+{
+    private readonly IReadOnlyList<string> _signingDirectories;
+    private readonly IReadOnlyList<string> _encryptionDirectories;
+
+    public OauthPemOpenIddictServerConfigurer()
+        : this(OauthPemCertificatePaths.SigningDirectories, OauthPemCertificatePaths.EncryptionDirectories)
+    {
+    }
+
+    public OauthPemOpenIddictServerConfigurer(
+        IReadOnlyList<string> signingDirectories,
+        IReadOnlyList<string> encryptionDirectories)
+    {
+        _signingDirectories = signingDirectories;
+        _encryptionDirectories = encryptionDirectories;
+    }
+
+    public void Configure(OpenIddictServerOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+
+        foreach (var certificate in OauthPemCertificateLoader.LoadDistinctFromDirectories(_signingDirectories))
+        {
+            EnsurePrivateKey(certificate);
+            options.SigningCredentials.Add(
+                new SigningCredentials(new X509SecurityKey(certificate), SecurityAlgorithms.RsaSha256));
+        }
+
+        foreach (var certificate in OauthPemCertificateLoader.LoadDistinctFromDirectories(_encryptionDirectories))
+        {
+            EnsurePrivateKey(certificate);
+            options.EncryptionCredentials.Add(new EncryptingCredentials(
+                new X509SecurityKey(certificate),
+                SecurityAlgorithms.RsaOAEP,
+                SecurityAlgorithms.Aes256CbcHmacSha512));
+        }
+    }
+
+    private static void EnsurePrivateKey(X509Certificate2 certificate)
+    {
+        if (!certificate.HasPrivateKey)
+        {
+            throw new InvalidOperationException(
+                $"OAuth certificate '{certificate.Thumbprint}' is missing a private key.");
+        }
+    }
+}

backend/LexBoxApi/Auth/OauthPemOptionsChangeTokenSource.cs

@@ -0,0 +1,145 @@
+using System.Security.Cryptography;
+using System.Text;
+using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Primitives;
+using OpenIddict.Server;
+using OpenIddict.Validation;
+
+namespace LexBoxApi.Auth;
+
+/// <summary>
+/// Polls OAuth PEM mount paths for content changes and invalidates OpenIddict server
+/// and validation options. Polling (content hash) is used instead of a naive
+/// <see cref="FileSystemWatcher"/> because Kubernetes Secret volume updates swap
+/// the <c>..data</c> symlink and often produce no usable watch events.
+/// </summary>
+public sealed class OauthPemOptionsChangeTokenSource :
+    BackgroundService,
+    IOptionsChangeTokenSource<OpenIddictServerOptions>,
+    IOptionsChangeTokenSource<OpenIddictValidationOptions>
+{
+    public static readonly TimeSpan DefaultPollingInterval = TimeSpan.FromSeconds(30);
+
+    private readonly IReadOnlyList<string> _directories;
+    private readonly TimeSpan _pollingInterval;
+    private readonly ILogger<OauthPemOptionsChangeTokenSource>? _logger;
+    private CancellationTokenSource _serverCts = new();
+    private CancellationTokenSource _validationCts = new();
+    private string _lastFingerprint;
+
+    public OauthPemOptionsChangeTokenSource(
+        IReadOnlyList<string> signingDirectories,
+        IReadOnlyList<string> encryptionDirectories,
+        TimeSpan? pollingInterval = null,
+        ILogger<OauthPemOptionsChangeTokenSource>? logger = null)
+    {
+        ArgumentNullException.ThrowIfNull(signingDirectories);
+        ArgumentNullException.ThrowIfNull(encryptionDirectories);
+
+        _directories = signingDirectories.Concat(encryptionDirectories).ToArray();
+        _pollingInterval = pollingInterval ?? DefaultPollingInterval;
+        _logger = logger;
+        _lastFingerprint = ComputeFingerprint();
+    }
+
+    public OauthPemOptionsChangeTokenSource(ILogger<OauthPemOptionsChangeTokenSource> logger)
+        : this(
+            OauthPemCertificatePaths.SigningDirectories,
+            OauthPemCertificatePaths.EncryptionDirectories,
+            DefaultPollingInterval,
+            logger)
+    {
+    }
+
+    string IOptionsChangeTokenSource<OpenIddictServerOptions>.Name => Options.DefaultName;
+    string IOptionsChangeTokenSource<OpenIddictValidationOptions>.Name => Options.DefaultName;
+
+    IChangeToken IOptionsChangeTokenSource<OpenIddictServerOptions>.GetChangeToken() =>
+        new CancellationChangeToken(_serverCts.Token);
+
+    IChangeToken IOptionsChangeTokenSource<OpenIddictValidationOptions>.GetChangeToken() =>
+        new CancellationChangeToken(_validationCts.Token);
+
+    /// <summary>
+    /// Recomputes the PEM content fingerprint and, when it changed, signals options reload.
+    /// Exposed for tests and for an immediate check outside the poll loop.
+    /// </summary>
+    /// <returns><see langword="true"/> if a reload was signaled.</returns>
+    public bool CheckForChanges()
+    {
+        string fingerprint;
+        try
+        {
+            fingerprint = ComputeFingerprint();
+        }
+        catch (Exception ex)
+        {
+            _logger?.LogWarning(ex, "Failed to fingerprint OAuth PEM mounts; will retry on next poll");
+            return false;
+        }
+
+        if (string.Equals(fingerprint, _lastFingerprint, StringComparison.Ordinal))
+            return false;
+
+        _logger?.LogInformation("OAuth PEM mount content changed; reloading OpenIddict credentials");
+        _lastFingerprint = fingerprint;
+        Reload();
+        return true;
+    }
+
+    /// <summary>
+    /// Forces OpenIddict server and validation options to recreate on next access.
+    /// Server is invalidated before validation so <c>UseLocalServer()</c> re-import sees fresh credentials.
+    /// </summary>
+    public void Reload()
+    {
+        var previousServer = Interlocked.Exchange(ref _serverCts, new CancellationTokenSource());
+        previousServer.Cancel();
+
+        var previousValidation = Interlocked.Exchange(ref _validationCts, new CancellationTokenSource());
+        previousValidation.Cancel();
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // Establish baseline without signaling a reload on process start.
+        _lastFingerprint = ComputeFingerprint();
+
+        using var timer = new PeriodicTimer(_pollingInterval);
+        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
+        {
+            CheckForChanges();
+        }
+    }
+
+    private string ComputeFingerprint()
+    {
+        var orderedPaths = _directories
+            .Where(static d => !string.IsNullOrWhiteSpace(d))
+            .SelectMany(static directory => new[]
+            {
+                System.IO.Path.Combine(directory, OauthPemCertificatePaths.CertFileName),
+                System.IO.Path.Combine(directory, OauthPemCertificatePaths.KeyFileName),
+            })
+            .OrderBy(static p => p, StringComparer.Ordinal)
+            .ToArray();
+
+        using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
+        foreach (var path in orderedPaths)
+        {
+            hash.AppendData(Encoding.UTF8.GetBytes(path));
+            hash.AppendData("\0"u8);
+
+            if (!File.Exists(path))
+            {
+                hash.AppendData("missing"u8);
+                continue;
+            }
+
+            // Read resolved file bytes so k8s ..data symlink swaps are visible.
+            hash.AppendData(File.ReadAllBytes(path));
+        }
+
+        return Convert.ToHexString(hash.GetHashAndReset());
+    }
+}