feat: audit trail

SW.Bitween.Api/Data/AuditPolicy.cs

@@ -0,0 +1,115 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.EntityFrameworkCore.ChangeTracking;
+using Microsoft.EntityFrameworkCore.Metadata;
+using SW.Bitween.Domain;
+using SW.Bitween.Domain.Accounts;
+using SW.Bitween.Domain.Gateway;
+using SW.Bitween.Services;
+using SW.EfCoreExtensions;
+
+namespace SW.Bitween;
+
+/// <summary>
+/// What the audit trail records, and what it must never record. Owned by C# in the same way as the
+/// permission catalog, so the rules sit in one readable list rather than being spread across the
+/// handlers that happen to write each entity.
+/// </summary>
+/// <remarks>
+/// <para>
+/// <b>Default deny.</b> An entity absent from <see cref="Audited"/> produces no audit rows at all.
+/// That is deliberate: everything in Bitween shares one <see cref="BitweenDbContext"/>, so an
+/// allow-everything policy would write several rows per message processed and bury the
+/// configuration changes the trail exists to show. It also means <see cref="AuditEntry"/> cannot
+/// audit itself.
+/// </para>
+/// <para>
+/// <b>Credentials.</b> <see cref="Redacted"/> names the properties that are never captured in any
+/// entity state. An audit table is read by more people than the screens that mask these values, so
+/// a property that is hidden in the UI and recorded here in full would be a leak wearing a
+/// different hat.
+/// </para>
+/// </remarks>
+public static class AuditPolicy
+{
+    /// <summary>
+    /// The configuration entities, their owned children, and the account/role records. Runtime
+    /// traffic — every <c>Xchange</c>, result, delivery, receive attempt, retry usage counter and
+    /// refresh token — is deliberately absent.
+    /// </summary>
+    private static readonly HashSet<Type> Audited =
+    [
+        typeof(Subscription),
+        typeof(Schedule),               // owned by Subscription
+        typeof(SubscriptionCategory),
+        typeof(Partner),
+        typeof(ApiCredential),          // owned by Partner
+        typeof(Document),
+        typeof(ApiGateway),
+        typeof(ApiGatewayPartner),
+        typeof(BusGateway),
+        typeof(BusGatewayRoute),
+        typeof(WorkGroup),
+        typeof(RetryPolicy),
+        typeof(RetryAlertOverride),
+        typeof(Notifier),
+        typeof(GlobalAdapterValuesSet),
+        typeof(Setting),
+        typeof(Account),
+        typeof(Role),
+        typeof(AccountRoleLink)
+    ];
+
+    /// <summary>
+    /// Properties whose values never enter the trail. The adapter property bags are here because
+    /// each is stored as a single JSON column, so the change tracker sees one property rather than
+    /// the individual keys — there is no way to keep <c>Host</c> and drop <c>Password</c> without
+    /// cracking the JSON open and asking the adapter which of its keys are <c>[Secure]</c>, which
+    /// would mean starting a serverless adapter in the middle of a save. Recording that a
+    /// subscription was edited without saying which adapter key changed is the deliberate trade.
+    /// </summary>
+    private static readonly Dictionary<Type, HashSet<string>> Redacted = new()
+    {
+        [typeof(Subscription)] =
+        [
+            nameof(Subscription.HandlerProperties),
+            nameof(Subscription.MapperProperties),
+            nameof(Subscription.ReceiverProperties),
+            nameof(Subscription.ValidatorProperties)
+        ],
+        [typeof(Partner)] = [nameof(Partner.AdapterProperties)],
+        [typeof(Notifier)] = [nameof(Notifier.HandlerProperties)],
+        [typeof(RetryPolicy)] = [nameof(RetryPolicy.AlertHandlerProperties)],
+        [typeof(RetryAlertOverride)] = [nameof(RetryAlertOverride.AlertHandlerProperties)],
+
+        // Free-form values an adapter reads by name. Nothing marks which of them is a credential,
+        // so there is no safe subset to keep.
+        [typeof(GlobalAdapterValuesSet)] = [nameof(GlobalAdapterValuesSet.Values)],
+
+        // The key itself is the credential. Name survives, so adding or revoking one is visible.
+        [typeof(ApiCredential)] = [nameof(ApiCredential.Key)],
+
+        [typeof(Account)] = [nameof(Account.Password)]
+    };
+
+    public static readonly AuditOptions Options = new()
+    {
+        ShouldAuditEntity = entry => Audited.Contains(entry.Metadata.ClrType),
+        ShouldAuditProperty = ShouldAuditProperty
+    };
+
+    private static bool ShouldAuditProperty(EntityEntry entry, IProperty property)
+    {
+        if (Redacted.TryGetValue(entry.Metadata.ClrType, out var redacted) &&
+            redacted.Contains(property.Name))
+            return false;
+
+        // A setting's value is only sometimes a secret, and unlike an adapter's properties the
+        // catalog says which — a static lookup, no adapter to start — so the theme colour stays
+        // readable in the trail while the licence key never enters it.
+        if (entry.Entity is Setting setting && property.Name == nameof(Setting.Value))
+            return SettingsCatalog.Find(setting.Id)?.Secret != true;
+
+        return true;
+    }
+}

SW.Bitween.Api/Data/BitweenDbContext.cs

@@ -59,21 +59,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
 
                 b.HasData(new Document(Document.AggregationDocumentId, "Aggregation Document"));
             });
-            modelBuilder.Entity<DocumentTrail>(b =>
-            {
-                b.HasKey(i => i.Id);
-                b.Property(p => p.Id).HasMaxLength(50);
-                b.HasIndex(i => i.CreatedOn);
-                b.HasOne(i => i.Document).WithMany().HasForeignKey(i => i.DocumentId);
-            });
-
-            modelBuilder.Entity<SubscriptionTrail>(b =>
-            {
-                b.HasKey(i => i.Id);
-                b.Property(p => p.Id).HasMaxLength(50);
-                b.HasIndex(i => i.CreatedOn);
-                b.HasOne(i => i.Subscription).WithMany().HasForeignKey(i => i.SubscriptionId);
-            });
             modelBuilder.Entity<RunFlagUpdater.RunningResult>(cr =>
             {
                 cr.HasNoKey().ToView(null);
@@ -450,6 +435,26 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.Id).IsUnicode(false).HasMaxLength(200);
             });
 
+            // ——— Audit trail ———
+            // A table that only ever grows and is only ever appended to. The indexes answer the two
+            // questions asked of it: the history of one row, and everything one save changed.
+            modelBuilder.Entity<AuditEntry>(b =>
+            {
+                b.ToTable("AuditEntries");
+                b.HasKey(p => p.Id);
+                b.Property(p => p.Id).IsUnicode(false).HasMaxLength(32);
+                // 36, not 32: the library builds these with Guid.ToString(), which keeps the hyphens.
+                b.Property(p => p.CorrelationId).IsUnicode(false).HasMaxLength(36).IsRequired();
+                b.Property(p => p.UserId).IsUnicode(false).HasMaxLength(50);
+                b.Property(p => p.EntityName).HasMaxLength(200).IsRequired();
+                b.Property(p => p.EntityKey).HasMaxLength(200);
+                b.Property(p => p.State).IsUnicode(false).HasMaxLength(10).IsRequired();
+
+                b.HasIndex(p => new { p.EntityName, p.EntityKey, p.OccurredOn });
+                b.HasIndex(p => p.CorrelationId);
+                b.HasIndex(p => p.OccurredOn);
+            });
+
         }
 
         /// <summary>
@@ -469,11 +474,63 @@ protected Role[] SystemRoleSeed() =>
                 { CreatedOn = defaultCreatedOn.ToUniversalTime() }
         ];
 
+        /// <summary>
+        /// Refused, because it would save without auditing. Only <see cref="SaveChangesAsync"/>
+        /// captures the trail, and an audit table with a silent hole in it is worse than none —
+        /// nobody would know which changes it had missed. Nothing in Bitween calls this today;
+        /// this makes sure a future caller finds out immediately rather than quietly.
+        /// </summary>
+        public override int SaveChanges() => throw new NotSupportedException(
+            "Use SaveChangesAsync — the synchronous path would skip the audit trail.");
+
+        /// <summary>
+        /// Saves, and records what was saved. The audit rows are written inside the same transaction
+        /// as the change they describe, so the trail can never disagree with the data — a save that
+        /// rolls back takes its audit rows with it.
+        /// </summary>
+        /// <remarks>
+        /// The transaction is opened only when there is something to audit. Runtime traffic — every
+        /// xchange, result and receive attempt — is excluded by <see cref="AuditPolicy"/> and so
+        /// still saves in exactly one round trip, unchanged. When a caller has already opened a
+        /// transaction of its own, that one is used rather than nested.
+        /// </remarks>
         async public override Task<int> SaveChangesAsync(CancellationToken cancellationToken = default)
         {
-            ChangeTracker.ApplyAuditValues(requestContext.GetNameIdentifier());
-            //using var transaction = await Database.BeginTransactionAsync();
-            var affectedRecords = await base.SaveChangesAsync(cancellationToken);
+            var userId = requestContext.GetNameIdentifier();
+            ChangeTracker.ApplyAuditValues(userId);
+
+            var pendingAudit = ChangeTracker.CapturePendingAuditDiffs(userId, AuditPolicy.Options);
+
+            var transaction = pendingAudit.Count > 0 && Database.CurrentTransaction is null
+                ? await Database.BeginTransactionAsync(cancellationToken)
+                : null;
+
+            int affectedRecords;
+            try
+            {
+                affectedRecords = await base.SaveChangesAsync(cancellationToken);
+
+                if (pendingAudit.Count > 0)
+                {
+                    // Finalized after the save because that is when a generated primary key exists;
+                    // an entry for a newly created row would otherwise record no key at all.
+                    foreach (var diff in pendingAudit.FinalizeAuditDiffJson())
+                        Add(new AuditEntry(diff));
+
+                    // base, deliberately: re-entering this override would audit the audit rows and
+                    // publish every domain event a second time.
+                    await base.SaveChangesAsync(cancellationToken);
+                }
+
+                if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+            }
+            finally
+            {
+                if (transaction is not null) await transaction.DisposeAsync();
+            }
+
+            // Published after the commit. An event announcing a change that then rolled back would
+            // send every consumer after a row that never existed.
             //await ChangeTracker.PublishDomainEvents(publish);
             var entitiesWithEvents = ChangeTracker.Entries<IGeneratesDomainEvents>()
                 .Select(e => e.Entity)

SW.Bitween.Api/Domain/Audit/AuditEntry.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Newtonsoft.Json;
+using SW.EfCoreExtensions;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Domain;
+
+/// <summary>
+/// One configuration change, as the change tracker saw it: which entity, which properties, from what
+/// to what, by whom.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Rows are written by <see cref="BitweenDbContext.SaveChangesAsync"/> for the entities
+/// <see cref="AuditPolicy"/> admits, in the same transaction as the change itself — so an entry
+/// exists for every audited change that committed, and for no change that didn't.
+/// </para>
+/// <para>
+/// Nothing edits or deletes an entry. That is the point of the table, and it is why the setters are
+/// private and there is no update path anywhere in the code.
+/// </para>
+/// </remarks>
+public class AuditEntry : BaseEntity<string>
+{
+    private AuditEntry()
+    {
+    }
+
+    public AuditEntry(GenericAuditDiffJson diff)
+    {
+        Id = Guid.NewGuid().ToString("N");
+        CorrelationId = diff.CorrelationId;
+        Sequence = diff.Sequence;
+        // The library hands back a DateTimeOffset. Every other timestamp in Bitween is a UTC
+        // DateTime, and MySQL has no offset type to store one in, so it is flattened here rather
+        // than becoming the one column that behaves differently on one of the three providers.
+        OccurredOn = diff.Timestamp.UtcDateTime;
+        UserId = diff.UserId;
+        EntityName = diff.EntityName;
+        EntityKey = FlattenKey(diff.PrimaryKey);
+        State = diff.State;
+        Changes = JsonConvert.SerializeObject(diff.Changes);
+    }
+
+    /// <summary>Shared by every entry written by the same save, so one request reads as one change.</summary>
+    public string CorrelationId { get; private set; }
+
+    /// <summary>Position within that save, 1-based. Only meaningful alongside <see cref="CorrelationId"/>.</summary>
+    public int Sequence { get; private set; }
+
+    public DateTime OccurredOn { get; private set; }
+
+    /// <summary>
+    /// The account id behind the change, or null for a save with no signed-in user — a bus consumer
+    /// or a scheduled job. Kept as the id rather than a name so that renaming an account doesn't
+    /// rewrite history.
+    /// </summary>
+    public string UserId { get; private set; }
+
+    /// <summary>The entity's display name, e.g. <c>Subscription</c>.</summary>
+    public string EntityName { get; private set; }
+
+    /// <summary>
+    /// The primary key as a single value, so the history of one row is an indexed lookup rather than
+    /// a JSON scan. Composite keys are joined with <c>|</c> in property-name order.
+    /// </summary>
+    public string EntityKey { get; private set; }
+
+    /// <summary><c>Added</c>, <c>Modified</c> or <c>Deleted</c>.</summary>
+    public string State { get; private set; }
+
+    /// <summary>
+    /// The per-property before and after values, as a JSON object of
+    /// <c>{ "PropertyName": { "Old": …, "New": … } }</c>. Properties excluded by
+    /// <see cref="AuditPolicy"/> never reach it.
+    /// </summary>
+    public string Changes { get; private set; }
+
+    private static string FlattenKey(object primaryKey) =>
+        primaryKey is IDictionary<string, object> values
+            ? string.Join("|", values.OrderBy(kv => kv.Key).Select(kv => kv.Value?.ToString()))
+            : primaryKey?.ToString();
+}