Reduce sync work: faster snapshot projection + benchmarks

harmony.sln

@@ -22,6 +22,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		src\.editorconfig = src\.editorconfig
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.Harmony.Benchmarks", "src\SIL.Harmony.Benchmarks\SIL.Harmony.Benchmarks.csproj", "{E5C038C9-AB5B-456C-AFC9-037A8E399486}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -55,5 +57,9 @@ Global
 		{8EB807F6-C548-4016-856E-2ACCA5603036}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E5C038C9-AB5B-456C-AFC9-037A8E399486}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E5C038C9-AB5B-456C-AFC9-037A8E399486}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E5C038C9-AB5B-456C-AFC9-037A8E399486}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E5C038C9-AB5B-456C-AFC9-037A8E399486}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs

@@ -0,0 +1,160 @@
+using System.Diagnostics.CodeAnalysis;
+using BenchmarkDotNet.Attributes;
+using Microsoft.EntityFrameworkCore;
+using SIL.Harmony.Db;
+using SIL.Harmony.Tests;
+
+namespace SIL.Harmony.Benchmarks;
+
+public enum AddSnapshotsWorkload
+{
+    /// <summary>Create many distinct entities (root snapshots, all inserts, no FindAsync hits).</summary>
+    CreateNew,
+
+    /// <summary>Word + Definition per commit — two projected table types.</summary>
+    MultiTypeCreate,
+
+    /// <summary>Word + Tag + WordTag per commit — three types plus a reference graph.</summary>
+    ReferencedCreate,
+
+    /// <summary>Seed created words, then update each once — updates project onto existing rows (FindAsync per snapshot).</summary>
+    UpdateExisting,
+
+    /// <summary>Modify one entity many times — large snapshot count (many intermediates), dedup, few projected rows.</summary>
+    ModifySameEntity,
+
+    /// <summary>Create then delete each entity — delete group + insert group, two SaveChanges.</summary>
+    CreateThenDelete,
+
+    /// <summary>Create, delete, then apply-on-deleted — mixed EntityIsDeleted projection skips.</summary>
+    CreateDeleteModify,
+}
+
+// Isolates CrdtRepository.AddSnapshots (the slow, non-FAST path) from the rest of the sync pipeline.
+// Expensive DB seeding happens once in GlobalSetup; each iteration gets a clean copy via ForkDatabase() and
+// recomputes the snapshot batch so no EF-tracked state leaks across iterations.
+// disable warning about waiting for sync code, benchmarkdotnet does not support async code, and it doesn't deadlock when waiting.
+[SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")]
+public class AddSnapshotsBenchmarks
+{
+    private DataModelTestBase _template = null!;
+    private HashSet<Guid> _measuredCommitIds = null!;
+
+    private DataModelTestBase _local = null!;
+    private CrdtRepository _repository = null!;
+    private ObjectSnapshot[] _snapshotsToAdd = null!;
+
+    [Params(
+        1000
+        // ,        10_000
+        )]
+    public int ChangeCount { get; set; }
+
+    [ParamsAllValues]
+    public AddSnapshotsWorkload Workload { get; set; }
+
+    [GlobalSetup]
+    public void GlobalSetup()
+    {
+        _template = new DataModelTestBase(alwaysValidate: false, performanceTest: true);
+        var clientId = Guid.NewGuid();
+
+        List<Commit> seed;
+        List<Commit> measured;
+        switch (Workload)
+        {
+            case AddSnapshotsWorkload.CreateNew:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildCreateWords(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.MultiTypeCreate:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildWordsWithDefinitions(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.ReferencedCreate:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildWordsWithTags(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.UpdateExisting:
+                (seed, measured) = BenchmarkWorkloadBuilders.BuildUpdateExisting(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.ModifySameEntity:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildModifySameWord(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.CreateThenDelete:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildCreateThenDelete(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.CreateDeleteModify:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildCreateDeleteModify(_template, clientId, ChangeCount);
+                break;
+            default:
+                throw new ArgumentOutOfRangeException();
+        }
+
+        // Seed commits go through the full pipeline so their snapshots and projected rows already exist.
+        if (seed.Count > 0)
+            ((ISyncable)_template.DataModel).AddRangeFromSync(seed).Wait();
+
+        // The measured commits are present in the database but their snapshots are NOT yet persisted; that's the
+        // work AddSnapshots performs. Adding only the commits mirrors the state right before UpdateSnapshots runs.
+        var repository = _template.CreateRepository();
+        repository.AddCommits(measured).GetAwaiter().GetResult();
+        _measuredCommitIds = measured.Select(c => c.Id).ToHashSet();
+    }
+
+    [IterationSetup]
+    public void IterationSetup()
+    {
+        _local = _template.ForkDatabase(alwaysValidate: false);
+        _repository = _local.CreateRepository();
+
+        // Load the measured commits fresh from the fork (tracked) so AddSnapshots resolves their Commit navigation
+        // from the change tracker instead of trying to re-insert them.
+        var measuredCommits = _local.DbContext.Commits
+            .Include(c => c.ChangeEntities)
+            .Where(c => EF.Parameter(_measuredCommitIds).Contains(c.Id))
+            .ToArray()
+            .ToSortedSet();
+
+        // Prepopulate the snapshot lookup the same way DataModel.UpdateSnapshots does: existing snapshots (untracked)
+        // plus null for entities without one, so SnapshotWorker doesn't issue a per-entity query while computing.
+        var entityIds = measuredCommits
+            .SelectMany(c => c.ChangeEntities.Select(ce => ce.EntityId))
+            .ToHashSet();
+        var snapshotLookup = _repository.CurrentSnapshots()
+            .Include(s => s.Commit)
+            .Where(s => EF.Parameter(entityIds).Contains(s.EntityId))
+            .ToDictionary(s => s.EntityId, s => (ObjectSnapshot?)s);
+        foreach (var entityId in entityIds)
+            snapshotLookup.TryAdd(entityId, null);
+
+        var worker = new SnapshotWorker(snapshotLookup, _repository, _local.CrdtConfig);
+        _snapshotsToAdd = worker.ComputeSnapshotsToPersist(measuredCommits).GetAwaiter().GetResult().ToArray();
+    }
+
+    [Benchmark]
+    public void AddSnapshots()
+    {
+        _repository.AddSnapshots(_snapshotsToAdd).Wait();
+    }
+
+    [IterationCleanup]
+    public void IterationCleanup()
+    {
+        _repository.DisposeAsync().AsTask().Wait();
+        _local.DisposeAsync().AsTask().Wait();
+        _repository = null!;
+        _local = null!;
+        _snapshotsToAdd = null!;
+    }
+
+    [GlobalCleanup]
+    public void GlobalCleanup()
+    {
+        _template.DisposeAsync().AsTask().Wait();
+        _template = null!;
+    }
+}

src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs

@@ -0,0 +1,168 @@
+using SIL.Harmony.Changes;
+using SIL.Harmony.Tests;
+
+namespace SIL.Harmony.Benchmarks;
+
+/// <summary>
+/// Shared commit-building helpers for the benchmark suites so <see cref="DataModelSyncBenchmarks"/> (full sync
+/// pipeline) and <see cref="AddSnapshotsBenchmarks"/> (isolated snapshot persist) exercise the same domain scenarios.
+/// Builders are pure: they only use the source <see cref="DataModelTestBase"/> for its change factories and
+/// <see cref="DataModelTestBase.NextDate"/> counter and never touch the database.
+/// </summary>
+public static class BenchmarkWorkloadBuilders
+{
+    public static Commit NewCommit(Guid clientId, DateTimeOffset dateTime, params IChange[] changes)
+    {
+        var commit = new Commit(Guid.NewGuid())
+        {
+            ClientId = clientId,
+            HybridDateTime = new HybridDateTime(dateTime, 0)
+        };
+        for (var i = 0; i < changes.Length; i++)
+        {
+            commit.ChangeEntities.Add(DataModel.ToChangeEntity(changes[i], i, commit.Id));
+        }
+        return commit;
+    }
+
+    /// <summary>Create many distinct words (one SetWord per commit).</summary>
+    public static List<Commit> BuildCreateWords(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(Guid.NewGuid(), $"entity {i}")));
+        }
+        return commits;
+    }
+
+    /// <summary>Create words each with a new definition in the same commit.</summary>
+    public static List<Commit> BuildWordsWithDefinitions(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}"),
+                src.NewDefinition(wordId, $"definition {i}", "noun")));
+        }
+        return commits;
+    }
+
+    /// <summary>Create words each with a tag and WordTag link in the same commit.</summary>
+    public static List<Commit> BuildWordsWithTags(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            var tagId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}"),
+                src.SetTag(tagId, $"tag {i}"),
+                src.TagWord(wordId, tagId)));
+        }
+        return commits;
+    }
+
+    /// <summary>Create one word, then modify that same word repeatedly.</summary>
+    public static List<Commit> BuildModifySameWord(DataModelTestBase src, Guid clientId, int count)
+    {
+        var wordId = Guid.NewGuid();
+        var commits = new List<Commit>(count);
+        commits.Add(NewCommit(clientId, src.NextDate(),
+            src.SetWord(wordId, "entity 0")));
+        for (var i = 1; i < count; i++)
+        {
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}")));
+        }
+        return commits;
+    }
+
+    /// <summary>Create words then delete them.</summary>
+    public static List<Commit> BuildCreateThenDelete(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count * 2);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}")));
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.DeleteWord(wordId)));
+        }
+        return commits;
+    }
+
+    /// <summary>Create, delete, then modify (apply-after-delete) for each word.</summary>
+    public static List<Commit> BuildCreateDeleteModify(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count * 3);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}")));
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.DeleteWord(wordId)));
+            // SetWordNote supports apply-on-existing (including deleted) without undeleting
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWordNote(wordId, $"note {i}")));
+        }
+        return commits;
+    }
+
+    /// <summary>
+    /// Local already has create + late modify; sync inserts a mid-history modify for each word
+    /// (forces stale snapshot deletion / rebuild). Returns the seed commits and the mid-history commits to sync.
+    /// </summary>
+    public static (List<Commit> seed, List<Commit> toSync) BuildOutOfOrderInsert(DataModelTestBase src, Guid clientId, int count)
+    {
+        var seed = new List<Commit>(count * 2);
+        var toSync = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            var createTime = src.NextDate();
+            var midTime = src.NextDate();
+            var lateTime = src.NextDate();
+
+            seed.Add(NewCommit(clientId, createTime,
+                src.SetWord(wordId, $"entity {i}")));
+            // Mid-history change is what gets synced after local already has create + late
+            toSync.Add(NewCommit(clientId, midTime,
+                src.SetWordNote(wordId, $"note {i}")));
+            seed.Add(NewCommit(clientId, lateTime,
+                src.SetWord(wordId, $"entity {i} late")));
+        }
+
+        return (seed, toSync);
+    }
+
+    /// <summary>
+    /// Seed a set of created words, then modify each one exactly once. The updates are the measured batch;
+    /// their snapshots must update the already-projected rows (FindAsync per snapshot in the slow path).
+    /// </summary>
+    public static (List<Commit> seed, List<Commit> measured) BuildUpdateExisting(DataModelTestBase src, Guid clientId, int count)
+    {
+        var wordIds = new Guid[count];
+        var seed = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            wordIds[i] = Guid.NewGuid();
+            seed.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordIds[i], $"entity {i}")));
+        }
+
+        var measured = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            measured.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordIds[i], $"entity {i} updated")));
+        }
+        return (seed, measured);
+    }
+}