From 136982d8d6c007963a440991bdeda0d776f60fd6 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 10:42:59 +0700 Subject: [PATCH 01/12] add some sync perf benchmarks --- harmony.sln | 6 ++ .../DataModelSyncBenchmarks.cs | 82 +++++++++++++++++++ src/SIL.Harmony.Benchmarks/Program.cs | 7 ++ .../SIL.Harmony.Benchmarks.csproj | 26 ++++++ src/SIL.Harmony/DataModel.cs | 2 +- src/SIL.Harmony/SIL.Harmony.csproj | 1 + 6 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs create mode 100644 src/SIL.Harmony.Benchmarks/Program.cs create mode 100644 src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj diff --git a/harmony.sln b/harmony.sln index a44dc0d..f9d4080 100644 --- a/harmony.sln +++ b/harmony.sln @@ -22,6 +22,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution src\.editorconfig = src\.editorconfig EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SIL.Harmony.Benchmarks", "src\SIL.Harmony.Benchmarks\SIL.Harmony.Benchmarks.csproj", "{E5C038C9-AB5B-456C-AFC9-037A8E399486}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -55,5 +57,9 @@ Global {8EB807F6-C548-4016-856E-2ACCA5603036}.Debug|Any CPU.Build.0 = Debug|Any CPU {8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.ActiveCfg = Release|Any CPU {8EB807F6-C548-4016-856E-2ACCA5603036}.Release|Any CPU.Build.0 = Release|Any CPU + {E5C038C9-AB5B-456C-AFC9-037A8E399486}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5C038C9-AB5B-456C-AFC9-037A8E399486}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5C038C9-AB5B-456C-AFC9-037A8E399486}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5C038C9-AB5B-456C-AFC9-037A8E399486}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs new file mode 100644 index 0000000..ca049fc --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs @@ -0,0 +1,82 @@ +using System.Diagnostics.CodeAnalysis; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; +using SIL.Harmony.Tests; + +namespace SIL.Harmony.Benchmarks; + +[SimpleJob(RunStrategy.Monitoring, warmupCount: 2)] +[SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")] +public class DataModelSyncBenchmarks +{ + private DataModelTestBase remote = null!; + private DataModelTestBase local = null!; + + [Params(1000, 10_000, 100_000)] + public int ChangeCount { get; set; } + private Commit[] _commits = null!; + + [IterationSetup] + public void IterationSetup() + { + remote = new DataModelTestBase(alwaysValidate: false, performanceTest: true); + local = remote.ForkDatabase(false); + _ = local.WriteNextChange(local.SetWord(Guid.NewGuid(), "entity1")).Result; + var clientId = Guid.NewGuid(); + var commits = new List(); + Commit? currentCommit = null; + for (int i = 0; i < ChangeCount; i++) + { + if (currentCommit == null) + { + var commitId = Guid.NewGuid(); + currentCommit = new Commit(commitId) + { + ClientId = clientId, + HybridDateTime = new HybridDateTime(remote.NextDate(), 0) + }; + } + + if (i % 100 == 0) + { + var tagId = Guid.NewGuid(); + var wordId = Guid.NewGuid(); + currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetWord(wordId, $"entity {i}"), i, currentCommit.Id)); + currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetTag(tagId, $"tag {i}"), i, currentCommit.Id)); + currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.TagWord(wordId, tagId), i, currentCommit.Id)); + } + else if (i % 25 == 0) + { + var wordId = Guid.NewGuid(); + currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetWord(wordId, $"entity {i}"), i, currentCommit.Id)); + currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.NewDefinition(wordId, $"definition {i}", "noun"), i, currentCommit.Id)); + } + else + { + currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetWord(Guid.NewGuid(), $"entity {i}"), i, currentCommit.Id)); + } + if (i % 5 == 0 || i % 3 == 0) + { + for (var i1 = 0; i1 < currentCommit.ChangeEntities.Count; i1++) + { + currentCommit.ChangeEntities[i1].Index = i1; + } + commits.Add(currentCommit); + currentCommit = null; + } + } + if (currentCommit != null) + { + commits.Add(currentCommit); + } + ((ISyncable)remote.DataModel).AddRangeFromSync(commits).Wait(); + _commits = remote.DataModel.GetChanges(new SyncState([])).Result.MissingFromClient; + } + + [Benchmark] + public void SyncCommits() + { + ((ISyncable)local.DataModel).AddRangeFromSync(_commits) + .Wait(); + } +} diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs new file mode 100644 index 0000000..3f2f74d --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -0,0 +1,7 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; +using SIL.Harmony.Benchmarks; + + +var config = DefaultConfig.Instance; +BenchmarkRunner.Run(config, args); diff --git a/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj new file mode 100644 index 0000000..90275bd --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj @@ -0,0 +1,26 @@ + + + Exe + + + AnyCPU + pdbonly + true + true + true + Release + false + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 02acd36..4c49284 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -123,7 +123,7 @@ public ValueTask DisposeAsync() return ValueTask.CompletedTask; } - private static ChangeEntity ToChangeEntity(IChange change, int index, Guid commitId) + internal static ChangeEntity ToChangeEntity(IChange change, int index, Guid commitId) { return new ChangeEntity() { diff --git a/src/SIL.Harmony/SIL.Harmony.csproj b/src/SIL.Harmony/SIL.Harmony.csproj index 094fe40..4d202a5 100644 --- a/src/SIL.Harmony/SIL.Harmony.csproj +++ b/src/SIL.Harmony/SIL.Harmony.csproj @@ -8,6 +8,7 @@ + From 83ba3598952c74b6996255af404072f79868c6f2 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 11:23:05 +0700 Subject: [PATCH 02/12] fix benchmark reduce work per iteration --- .../DataModelSyncBenchmarks.cs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs index ca049fc..06ef84d 100644 --- a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs @@ -5,7 +5,7 @@ namespace SIL.Harmony.Benchmarks; -[SimpleJob(RunStrategy.Monitoring, warmupCount: 2)] +[SimpleJob(RunStrategy.Monitoring)] [SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")] public class DataModelSyncBenchmarks { @@ -16,12 +16,10 @@ public class DataModelSyncBenchmarks public int ChangeCount { get; set; } private Commit[] _commits = null!; - [IterationSetup] - public void IterationSetup() + [GlobalSetup] + public void GlobalSetup() { remote = new DataModelTestBase(alwaysValidate: false, performanceTest: true); - local = remote.ForkDatabase(false); - _ = local.WriteNextChange(local.SetWord(Guid.NewGuid(), "entity1")).Result; var clientId = Guid.NewGuid(); var commits = new List(); Commit? currentCommit = null; @@ -70,9 +68,24 @@ public void IterationSetup() commits.Add(currentCommit); } ((ISyncable)remote.DataModel).AddRangeFromSync(commits).Wait(); + } + + [IterationSetup] + public void IterationSetup() + { + local = new DataModelTestBase(alwaysValidate: false, performanceTest: true); + _ = local.WriteNextChange(local.SetWord(Guid.NewGuid(), "entity1")).Result; + //cant share commits between iterations, because EF modifies them _commits = remote.DataModel.GetChanges(new SyncState([])).Result.MissingFromClient; } + [IterationCleanup] + public void IterationCleanup() + { + local.DisposeAsync().AsTask().Wait(); + local = null!; + } + [Benchmark] public void SyncCommits() { From d2645394710d3b9e65f8dc14bcc6948eb1f168a4 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 11:53:49 +0700 Subject: [PATCH 03/12] pre load snapshots --- src/SIL.Harmony/DataModel.cs | 14 ++++++++++---- src/SIL.Harmony/SnapshotWorker.cs | 19 +++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 4c49284..f3d8250 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -191,19 +191,25 @@ private async Task UpdateSnapshots(CrdtRepository repo, SortedSet commit if (commitsToApply.Count == 0) return; var oldestAddedCommit = commitsToApply.First(); await repo.DeleteStaleSnapshots(oldestAddedCommit); - Dictionary snapshotLookup = []; + Dictionary snapshotLookup = []; if (commitsToApply.Count > 10) { // Bulk-load relevant snapshots to minimize DB queries var entityIds = commitsToApply .SelectMany(c => c.ChangeEntities.Select(ce => ce.EntityId)) - .Distinct(); + .ToHashSet(); //EF.Parameter forces a single JSON parameter; without it EF 10+ emits one parameter per id and overflows SQLite's parameter limit snapshotLookup = await repo.CurrentSnapshots() + .Include(s => s.Commit) .Where(s => EF.Parameter(entityIds).Contains(s.EntityId)) - .Select(s => new KeyValuePair(s.EntityId, s.Id)) - .ToDictionaryAsync(s => s.Key, s => s.Value); + .ToDictionaryAsync(s => s.EntityId, s => (ObjectSnapshot?)s); + entityIds.ExceptWith(snapshotLookup.Keys); + foreach (Guid entityId in entityIds) + { + //snapshot does not exist, store null to tell SnapshotWorker NOT to attempt to fetch it from the database + snapshotLookup[entityId] = null; + } } var snapshotWorker = new SnapshotWorker(snapshotLookup, repo, _crdtConfig.Value); diff --git a/src/SIL.Harmony/SnapshotWorker.cs b/src/SIL.Harmony/SnapshotWorker.cs index aa78f80..86043a9 100644 --- a/src/SIL.Harmony/SnapshotWorker.cs +++ b/src/SIL.Harmony/SnapshotWorker.cs @@ -10,7 +10,7 @@ namespace SIL.Harmony; /// internal class SnapshotWorker { - private readonly Dictionary _snapshotLookup; + private readonly Dictionary _snapshotCache; private readonly CrdtRepository _crdtRepository; private readonly CrdtConfig _crdtConfig; private readonly Dictionary _pendingSnapshots = []; @@ -18,13 +18,13 @@ internal class SnapshotWorker private readonly List _newIntermediateSnapshots = []; private SnapshotWorker(Dictionary snapshots, - Dictionary snapshotLookup, + Dictionary snapshotCache, CrdtRepository crdtRepository, CrdtConfig crdtConfig) { _pendingSnapshots = snapshots; _crdtRepository = crdtRepository; - _snapshotLookup = snapshotLookup; + _snapshotCache = snapshotCache; _crdtConfig = crdtConfig; } @@ -40,12 +40,12 @@ internal static async Task> ApplyCommitsToSnaps return snapshots; } - /// a dictionary of entity id to latest snapshot id + /// a dictionary of entity id to latest snapshot id /// /// - internal SnapshotWorker(Dictionary snapshotLookup, + internal SnapshotWorker(Dictionary snapshotCache, CrdtRepository crdtRepository, - CrdtConfig crdtConfig) : this([], snapshotLookup, crdtRepository, crdtConfig) + CrdtConfig crdtConfig) : this([], snapshotCache, crdtRepository, crdtConfig) { } @@ -158,14 +158,13 @@ private async ValueTask MarkDeleted(Guid deletedEntityId, ChangeContext context) return rootSnapshot; } - if (_snapshotLookup.TryGetValue(entityId, out var snapshotId)) + if (_snapshotCache.TryGetValue(entityId, out snapshot)) { - if (snapshotId is null) return null; - return await _crdtRepository.FindSnapshot(snapshotId.Value, true); + return snapshot; } snapshot = await _crdtRepository.GetCurrentSnapshotByObjectId(entityId, true); - _snapshotLookup[entityId] = snapshot?.Id; + _snapshotCache[entityId] = snapshot; return snapshot; } From cb988b7ebad86ba0f42d4feab8b2f8d709c389ce Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 10:17:17 +0000 Subject: [PATCH 04/12] Batch-load existing projected rows in AddSnapshots Previously AddSnapshots projected each snapshot by calling FindAsync per entity, which issued one database query per snapshot (and, on an initial sync of new data, every query returned null after a round-trip). Pre-load the projected rows that already exist for the batch with a single tracked query per object type. ProjectSnapshot then resolves existing entities from the change tracker and skips the lookup entirely for entities that have no projected row yet, collapsing N queries down to roughly one per distinct object type. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019zEmz7jPRPF6Lv8h6YAWBW --- src/SIL.Harmony/Db/CrdtDbContextFactory.cs | 2 + src/SIL.Harmony/Db/CrdtRepository.cs | 57 ++++++++++++++++++++-- src/SIL.Harmony/Db/ICrdtDbContext.cs | 2 + 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/SIL.Harmony/Db/CrdtDbContextFactory.cs b/src/SIL.Harmony/Db/CrdtDbContextFactory.cs index 890b1ee..d50a1b1 100644 --- a/src/SIL.Harmony/Db/CrdtDbContextFactory.cs +++ b/src/SIL.Harmony/Db/CrdtDbContextFactory.cs @@ -1,6 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; namespace SIL.Harmony.Db; @@ -65,6 +66,7 @@ public DbSet Set() where TEntity : class return context.Set(); } + public IModel Model => context.Model; public DatabaseFacade Database => context.Database; public ChangeTracker ChangeTracker => context.ChangeTracker; diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index fb23e53..a511304 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -3,6 +3,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -301,8 +302,12 @@ public async Task> GetChanges(SyncState remoteState) public async Task AddSnapshots(IEnumerable snapshots) { + var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); + // pre-load the projected rows that already exist so ProjectSnapshot's FindAsync calls are served from the + // change tracker instead of issuing one query per snapshot + var existingEntityIds = await LoadExistingEntityIds(snapshotList); var latestProjectByEntityId = new Dictionary(); - foreach (var grouping in snapshots.GroupBy(s => s.EntityIsDeleted).OrderByDescending(g => g.Key))//execute deletes first + foreach (var grouping in snapshotList.GroupBy(s => s.EntityIsDeleted).OrderByDescending(g => g.Key))//execute deletes first { foreach (var snapshot in grouping.DefaultOrderDescending()) { @@ -318,7 +323,7 @@ public async Task AddSnapshots(IEnumerable snapshots) } latestProjectByEntityId[snapshot.EntityId] = snapshot.Commit.CompareKey; - await ProjectSnapshot(snapshot); + await ProjectSnapshot(snapshot, existingEntityIds); } try @@ -335,12 +340,55 @@ public async Task AddSnapshots(IEnumerable snapshots) } } - private async ValueTask ProjectSnapshot(ObjectSnapshot objectSnapshot) + /// + /// Loads the projected rows that already exist for the given snapshots so they're tracked by the context. + /// This lets ProjectSnapshot resolve existing entities from the change tracker (and skip lookups for new + /// entities entirely) rather than running one FindAsync query per snapshot. + /// + /// the set of entity ids that already have a projected row + private async Task> LoadExistingEntityIds(IReadOnlyCollection snapshots) + { + var existingEntityIds = new HashSet(); + if (!_crdtConfig.Value.EnableProjectedTables) return existingEntityIds; + foreach (var typeGroup in snapshots.GroupBy(s => s.Entity.DbObject.GetType())) + { + var entityIds = typeGroup.Select(s => s.EntityId).Distinct().ToArray(); + var task = (Task>)loadExistingEntitiesMethod + .MakeGenericMethod(typeGroup.Key) + .Invoke(null, [_dbContext, entityIds])!; + existingEntityIds.UnionWith(await task); + } + return existingEntityIds; + } + + private static readonly MethodInfo loadExistingEntitiesMethod = + new Func, Task>>(LoadExistingEntities) + .Method.GetGenericMethodDefinition(); + + private static async Task> LoadExistingEntities(ICrdtDbContext dbContext, IReadOnlyCollection entityIds) + where T : class + { + var keyName = dbContext.Model.FindEntityType(typeof(T))?.FindPrimaryKey()?.Properties.Single().Name + ?? throw new InvalidOperationException($"No primary key found for projected type {typeof(T).Name}"); + //load (and thereby track) the matching rows so ProjectSnapshot's FindAsync calls hit the change tracker. + //EF.Parameter forces a single JSON parameter; without it EF 10+ emits one parameter per id and overflows SQLite's parameter limit + var entities = await dbContext.Set() + .Where(e => EF.Parameter(entityIds).Contains(EF.Property(e, keyName))) + .ToListAsync(); + return entities + .Select(e => (Guid)dbContext.Entry(e).Property(keyName).CurrentValue!) + .ToList(); + } + + private async ValueTask ProjectSnapshot(ObjectSnapshot objectSnapshot, HashSet existingEntityIds) { if (!_crdtConfig.Value.EnableProjectedTables) return; //need to check if an entry exists already, even if this is the root commit it may have already been added to the db - var existingEntry = await GetEntityEntry(objectSnapshot.Entity.DbObject.GetType(), objectSnapshot.EntityId); + //entities not in existingEntityIds have no projected row yet, so skip the lookup and treat them as new + var existingEntry = existingEntityIds.Contains(objectSnapshot.EntityId) + ? await GetEntityEntry(objectSnapshot.Entity.DbObject.GetType(), objectSnapshot.EntityId) + : null; if (existingEntry is null && objectSnapshot.EntityIsDeleted) return; if (existingEntry is null) // add @@ -505,6 +553,7 @@ public DbSet Set() where TEntity : class throw new NotSupportedException("can not support Set when using scoped db context"); } + public IModel Model => inner.Model; public DatabaseFacade Database => inner.Database; public ChangeTracker ChangeTracker => inner.ChangeTracker; diff --git a/src/SIL.Harmony/Db/ICrdtDbContext.cs b/src/SIL.Harmony/Db/ICrdtDbContext.cs index 438a225..59f52cc 100644 --- a/src/SIL.Harmony/Db/ICrdtDbContext.cs +++ b/src/SIL.Harmony/Db/ICrdtDbContext.cs @@ -1,6 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; namespace SIL.Harmony.Db; @@ -11,6 +12,7 @@ public interface ICrdtDbContext : IDisposable, IAsyncDisposable Task SaveChangesAsync(CancellationToken cancellationToken = default); ValueTask FindAsync(Type entityType, params object?[]? keyValues); DbSet Set() where TEntity : class; + IModel Model { get; } DatabaseFacade Database { get; } ChangeTracker ChangeTracker { get; } EntityEntry Entry(TEntity entity) where TEntity : class; From ed1c19345bdf7191cb6b245569418963b7209621 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 13:31:34 +0700 Subject: [PATCH 05/12] create a bunch of benchmark types --- .../DataModelSyncBenchmarks.cs | 255 ++++++++++++++---- src/SIL.Harmony.Tests/DataModelTestBase.cs | 5 + 2 files changed, 213 insertions(+), 47 deletions(-) diff --git a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs index 06ef84d..32de034 100644 --- a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs @@ -1,10 +1,41 @@ using System.Diagnostics.CodeAnalysis; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; +using SIL.Harmony.Changes; using SIL.Harmony.Tests; namespace SIL.Harmony.Benchmarks; +public enum SyncWorkload +{ + /// Create many distinct words (one SetWord per commit). + CreateWords, + + /// Create words each with a new definition in the same commit. + WordsWithDefinitions, + + /// Create words each with a tag and WordTag link in the same commit. + WordsWithTags, + + /// Create one word, then modify that same word repeatedly. + ModifySameWord, + + /// Create words then delete them. + CreateThenDelete, + + /// Create, delete, then modify (apply-after-delete) for each word. + CreateDeleteModify, + + /// Create, delete, then recreate (undelete via NewEntity) for each word. + CreateDeleteUndelete, + + /// + /// Local already has create + late modify; sync inserts a mid-history modify + /// for each word (forces stale snapshot deletion / rebuild). + /// + OutOfOrderInsert, +} + [SimpleJob(RunStrategy.Monitoring)] [SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")] public class DataModelSyncBenchmarks @@ -12,61 +43,33 @@ public class DataModelSyncBenchmarks private DataModelTestBase remote = null!; private DataModelTestBase local = null!; - [Params(1000, 10_000, 100_000)] + [Params(1000, 10_000)] public int ChangeCount { get; set; } + + [ParamsAllValues] + public SyncWorkload Workload { get; set; } + private Commit[] _commits = null!; + private HashSet? _syncCommitIds; [GlobalSetup] public void GlobalSetup() { + _syncCommitIds = null; remote = new DataModelTestBase(alwaysValidate: false, performanceTest: true); var clientId = Guid.NewGuid(); - var commits = new List(); - Commit? currentCommit = null; - for (int i = 0; i < ChangeCount; i++) + var commits = Workload switch { - if (currentCommit == null) - { - var commitId = Guid.NewGuid(); - currentCommit = new Commit(commitId) - { - ClientId = clientId, - HybridDateTime = new HybridDateTime(remote.NextDate(), 0) - }; - } - - if (i % 100 == 0) - { - var tagId = Guid.NewGuid(); - var wordId = Guid.NewGuid(); - currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetWord(wordId, $"entity {i}"), i, currentCommit.Id)); - currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetTag(tagId, $"tag {i}"), i, currentCommit.Id)); - currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.TagWord(wordId, tagId), i, currentCommit.Id)); - } - else if (i % 25 == 0) - { - var wordId = Guid.NewGuid(); - currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetWord(wordId, $"entity {i}"), i, currentCommit.Id)); - currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.NewDefinition(wordId, $"definition {i}", "noun"), i, currentCommit.Id)); - } - else - { - currentCommit.ChangeEntities.Add(DataModel.ToChangeEntity(remote.SetWord(Guid.NewGuid(), $"entity {i}"), i, currentCommit.Id)); - } - if (i % 5 == 0 || i % 3 == 0) - { - for (var i1 = 0; i1 < currentCommit.ChangeEntities.Count; i1++) - { - currentCommit.ChangeEntities[i1].Index = i1; - } - commits.Add(currentCommit); - currentCommit = null; - } - } - if (currentCommit != null) - { - commits.Add(currentCommit); - } + SyncWorkload.CreateWords => BuildCreateWords(clientId), + SyncWorkload.WordsWithDefinitions => BuildWordsWithDefinitions(clientId), + SyncWorkload.WordsWithTags => BuildWordsWithTags(clientId), + SyncWorkload.ModifySameWord => BuildModifySameWord(clientId), + SyncWorkload.CreateThenDelete => BuildCreateThenDelete(clientId), + SyncWorkload.CreateDeleteModify => BuildCreateDeleteModify(clientId), + SyncWorkload.CreateDeleteUndelete => BuildCreateDeleteUndelete(clientId), + SyncWorkload.OutOfOrderInsert => BuildOutOfOrderInsert(clientId), + _ => throw new ArgumentOutOfRangeException() + }; ((ISyncable)remote.DataModel).AddRangeFromSync(commits).Wait(); } @@ -76,7 +79,26 @@ public void IterationSetup() local = new DataModelTestBase(alwaysValidate: false, performanceTest: true); _ = local.WriteNextChange(local.SetWord(Guid.NewGuid(), "entity1")).Result; //cant share commits between iterations, because EF modifies them - _commits = remote.DataModel.GetChanges(new SyncState([])).Result.MissingFromClient; + var allCommits = remote.DataModel.GetChanges(new SyncState([])).Result.MissingFromClient; + + if (_syncCommitIds is null) + { + _commits = allCommits; + return; + } + + var seed = new List(allCommits.Length); + var toSync = new List(_syncCommitIds.Count); + foreach (var commit in allCommits) + { + if (_syncCommitIds.Contains(commit.Id)) + toSync.Add(commit); + else + seed.Add(commit); + } + + ((ISyncable)local.DataModel).AddRangeFromSync(seed).Wait(); + _commits = toSync.ToArray(); } [IterationCleanup] @@ -92,4 +114,143 @@ public void SyncCommits() ((ISyncable)local.DataModel).AddRangeFromSync(_commits) .Wait(); } + + private List BuildCreateWords(Guid clientId) + { + var commits = new List(ChangeCount); + for (var i = 0; i < ChangeCount; i++) + { + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(Guid.NewGuid(), $"entity {i}"))); + } + return commits; + } + + private List BuildWordsWithDefinitions(Guid clientId) + { + var commits = new List(ChangeCount); + for (var i = 0; i < ChangeCount; i++) + { + var wordId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, $"entity {i}"), + remote.NewDefinition(wordId, $"definition {i}", "noun"))); + } + return commits; + } + + private List BuildWordsWithTags(Guid clientId) + { + var commits = new List(ChangeCount); + for (var i = 0; i < ChangeCount; i++) + { + var wordId = Guid.NewGuid(); + var tagId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, $"entity {i}"), + remote.SetTag(tagId, $"tag {i}"), + remote.TagWord(wordId, tagId))); + } + return commits; + } + + private List BuildModifySameWord(Guid clientId) + { + var wordId = Guid.NewGuid(); + var commits = new List(ChangeCount); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, "entity 0"))); + for (var i = 1; i < ChangeCount; i++) + { + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, $"entity {i}"))); + } + return commits; + } + + private List BuildCreateThenDelete(Guid clientId) + { + var commits = new List(ChangeCount * 2); + for (var i = 0; i < ChangeCount; i++) + { + var wordId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, $"entity {i}"))); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.DeleteWord(wordId))); + } + return commits; + } + + private List BuildCreateDeleteModify(Guid clientId) + { + var commits = new List(ChangeCount * 3); + for (var i = 0; i < ChangeCount; i++) + { + var wordId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, $"entity {i}"))); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.DeleteWord(wordId))); + // SetWordNote supports apply-on-existing (including deleted) without undeleting + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWordNote(wordId, $"note {i}"))); + } + return commits; + } + + private List BuildCreateDeleteUndelete(Guid clientId) + { + var commits = new List(ChangeCount * 3); + for (var i = 0; i < ChangeCount; i++) + { + var wordId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, $"entity {i}"))); + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.DeleteWord(wordId))); + // SetWord supports NewEntity and undeletes + commits.Add(NewCommit(clientId, remote.NextDate(), + remote.SetWord(wordId, $"undeleted {i}"))); + } + return commits; + } + + private List BuildOutOfOrderInsert(Guid clientId) + { + var seed = new List(ChangeCount * 2); + var toSync = new List(ChangeCount); + for (var i = 0; i < ChangeCount; i++) + { + var wordId = Guid.NewGuid(); + var createTime = remote.NextDate(); + var midTime = remote.NextDate(); + var lateTime = remote.NextDate(); + + seed.Add(NewCommit(clientId, createTime, + remote.SetWord(wordId, $"entity {i}"))); + // Mid-history change is what gets synced after local already has create + late + toSync.Add(NewCommit(clientId, midTime, + remote.SetWordNote(wordId, $"note {i}"))); + seed.Add(NewCommit(clientId, lateTime, + remote.SetWord(wordId, $"entity {i} late"))); + } + + _syncCommitIds = toSync.Select(c => c.Id).ToHashSet(); + return [..seed, ..toSync]; + } + + private static Commit NewCommit(Guid clientId, DateTimeOffset dateTime, params IChange[] changes) + { + var commit = new Commit(Guid.NewGuid()) + { + ClientId = clientId, + HybridDateTime = new HybridDateTime(dateTime, 0) + }; + for (var i = 0; i < changes.Length; i++) + { + commit.ChangeEntities.Add(DataModel.ToChangeEntity(changes[i], i, commit.Id)); + } + return commit; + } } diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index 756162d..bd1b177 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -131,6 +131,11 @@ public IChange SetWord(Guid entityId, string value) return new SetWordTextChange(entityId, value); } + public IChange SetWordNote(Guid entityId, string note) + { + return new SetWordNoteChange(entityId, note); + } + public IChange DeleteWord(Guid entityId) { return new DeleteChange(entityId); From bef87400ae0e19a1ad3581641b4a7ac760baccd6 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 14:13:24 +0700 Subject: [PATCH 06/12] enable running new and old code --- .../DataModelSyncBenchmarks.cs | 28 ++++--------------- src/SIL.Harmony.Benchmarks/Program.cs | 7 ++++- src/SIL.Harmony/Db/CrdtRepository.cs | 21 ++++++++++---- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs index 32de034..ab31982 100644 --- a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs @@ -26,9 +26,6 @@ public enum SyncWorkload /// Create, delete, then modify (apply-after-delete) for each word. CreateDeleteModify, - /// Create, delete, then recreate (undelete via NewEntity) for each word. - CreateDeleteUndelete, - /// /// Local already has create + late modify; sync inserts a mid-history modify /// for each word (forces stale snapshot deletion / rebuild). @@ -36,14 +33,17 @@ public enum SyncWorkload OutOfOrderInsert, } -[SimpleJob(RunStrategy.Monitoring)] +// [SimpleJob(RunStrategy.Monitoring)] [SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")] public class DataModelSyncBenchmarks { private DataModelTestBase remote = null!; private DataModelTestBase local = null!; - [Params(1000, 10_000)] + [Params( + 1000 + // , 10_000 + )] public int ChangeCount { get; set; } [ParamsAllValues] @@ -66,7 +66,6 @@ public void GlobalSetup() SyncWorkload.ModifySameWord => BuildModifySameWord(clientId), SyncWorkload.CreateThenDelete => BuildCreateThenDelete(clientId), SyncWorkload.CreateDeleteModify => BuildCreateDeleteModify(clientId), - SyncWorkload.CreateDeleteUndelete => BuildCreateDeleteUndelete(clientId), SyncWorkload.OutOfOrderInsert => BuildOutOfOrderInsert(clientId), _ => throw new ArgumentOutOfRangeException() }; @@ -199,23 +198,6 @@ private List BuildCreateDeleteModify(Guid clientId) return commits; } - private List BuildCreateDeleteUndelete(Guid clientId) - { - var commits = new List(ChangeCount * 3); - for (var i = 0; i < ChangeCount; i++) - { - var wordId = Guid.NewGuid(); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, $"entity {i}"))); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.DeleteWord(wordId))); - // SetWord supports NewEntity and undeletes - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, $"undeleted {i}"))); - } - return commits; - } - private List BuildOutOfOrderInsert(Guid clientId) { var seed = new List(ChangeCount * 2); diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs index 3f2f74d..33eb7e6 100644 --- a/src/SIL.Harmony.Benchmarks/Program.cs +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -1,7 +1,12 @@ using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using SIL.Harmony.Benchmarks; -var config = DefaultConfig.Instance; +var config = DefaultConfig.Instance + .AddJob(Job.MediumRun.WithStrategy(RunStrategy.Monitoring).WithId("DEFAULT").AsBaseline()) + .AddJob(Job.MediumRun.WithMsBuildArguments("/p:DefineConstants=FAST").WithStrategy(RunStrategy.Monitoring) + .WithId("FAST")); BenchmarkRunner.Run(config, args); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index a511304..bde97d8 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Collections.Frozen; using System.Reflection; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.ChangeTracking; @@ -300,12 +301,22 @@ public async Task> GetChanges(SyncState remoteState) return await _dbContext.Commits.GetChanges(remoteState); } +#if FAST + public async Task AddSnapshots(IEnumerable snapshots) +{ + Console.WriteLine("FAST enabled ==========================================="); + _dbContext.AddRange(snapshots); + await _dbContext.SaveChangesAsync(); +} +#else public async Task AddSnapshots(IEnumerable snapshots) { var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); // pre-load the projected rows that already exist so ProjectSnapshot's FindAsync calls are served from the // change tracker instead of issuing one query per snapshot - var existingEntityIds = await LoadExistingEntityIds(snapshotList); + + Console.WriteLine("SLOW enabled ==========================================="); + var latestProjectByEntityId = new Dictionary(); foreach (var grouping in snapshotList.GroupBy(s => s.EntityIsDeleted).OrderByDescending(g => g.Key))//execute deletes first { @@ -323,7 +334,7 @@ public async Task AddSnapshots(IEnumerable snapshots) } latestProjectByEntityId[snapshot.EntityId] = snapshot.Commit.CompareKey; - await ProjectSnapshot(snapshot, existingEntityIds); + await ProjectSnapshot(snapshot, null); } try @@ -339,7 +350,7 @@ public async Task AddSnapshots(IEnumerable snapshots) } } } - +#endif /// /// Loads the projected rows that already exist for the given snapshots so they're tracked by the context. /// This lets ProjectSnapshot resolve existing entities from the change tracker (and skip lookups for new @@ -380,13 +391,13 @@ private static async Task> LoadExistingEntities(ICrdtDbContext dbC .ToList(); } - private async ValueTask ProjectSnapshot(ObjectSnapshot objectSnapshot, HashSet existingEntityIds) + private async ValueTask ProjectSnapshot(ObjectSnapshot objectSnapshot, ISet? existingEntityIds) { if (!_crdtConfig.Value.EnableProjectedTables) return; //need to check if an entry exists already, even if this is the root commit it may have already been added to the db //entities not in existingEntityIds have no projected row yet, so skip the lookup and treat them as new - var existingEntry = existingEntityIds.Contains(objectSnapshot.EntityId) + var existingEntry = existingEntityIds is null || existingEntityIds.Contains(objectSnapshot.EntityId) ? await GetEntityEntry(objectSnapshot.Entity.DbObject.GetType(), objectSnapshot.EntityId) : null; if (existingEntry is null && objectSnapshot.EntityIsDeleted) return; From 0466d2129266ec39c9a7d20a5d7ba4bc5249bb70 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 15:26:47 +0700 Subject: [PATCH 07/12] Add AddSnapshots benchmark suite isolating the snapshot-persist step Adds a BenchmarkDotNet suite that measures CrdtRepository.AddSnapshots on its own, across 7 workloads mirroring DataModelSyncBenchmarks. Expensive DB seeding runs once in a template DB; each iteration forks the DB and recomputes the snapshot batch so no EF-tracked state leaks across iterations. - SnapshotWorker.ComputeSnapshotsToPersist: returns the exact snapshot list UpdateSnapshots would persist, without writing it. - DataModelTestBase: internal CreateRepository() and CrdtConfig accessors. - BenchmarkWorkloadBuilders: shared commit builders extracted from DataModelSyncBenchmarks (+ BuildUpdateExisting). - Program.cs: run both suites via BenchmarkSwitcher (handles --filter/args). - Remove leftover Console.WriteLine debug lines from AddSnapshots. Co-Authored-By: Claude Opus 4.8 --- .../AddSnapshotsBenchmarks.cs | 160 +++++++++++++++++ .../BenchmarkWorkloadBuilders.cs | 168 ++++++++++++++++++ .../DataModelSyncBenchmarks.cs | 160 +++-------------- src/SIL.Harmony.Benchmarks/Program.cs | 4 +- src/SIL.Harmony.Tests/DataModelTestBase.cs | 10 ++ .../SIL.Harmony.Tests.csproj | 4 + src/SIL.Harmony/Db/CrdtRepository.cs | 3 - src/SIL.Harmony/SnapshotWorker.cs | 11 ++ 8 files changed, 384 insertions(+), 136 deletions(-) create mode 100644 src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs create mode 100644 src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs diff --git a/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs new file mode 100644 index 0000000..3a9d503 --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs @@ -0,0 +1,160 @@ +using System.Diagnostics.CodeAnalysis; +using BenchmarkDotNet.Attributes; +using Microsoft.EntityFrameworkCore; +using SIL.Harmony.Db; +using SIL.Harmony.Tests; + +namespace SIL.Harmony.Benchmarks; + +public enum AddSnapshotsWorkload +{ + /// Create many distinct entities (root snapshots, all inserts, no FindAsync hits). + CreateNew, + + /// Word + Definition per commit — two projected table types. + MultiTypeCreate, + + /// Word + Tag + WordTag per commit — three types plus a reference graph. + ReferencedCreate, + + /// Seed created words, then update each once — updates project onto existing rows (FindAsync per snapshot). + UpdateExisting, + + /// Modify one entity many times — large snapshot count (many intermediates), dedup, few projected rows. + ModifySameEntity, + + /// Create then delete each entity — delete group + insert group, two SaveChanges. + CreateThenDelete, + + /// Create, delete, then apply-on-deleted — mixed EntityIsDeleted projection skips. + CreateDeleteModify, +} + +// Isolates CrdtRepository.AddSnapshots (the slow, non-FAST path) from the rest of the sync pipeline. +// Expensive DB seeding happens once in GlobalSetup; each iteration gets a clean copy via ForkDatabase() and +// recomputes the snapshot batch so no EF-tracked state leaks across iterations. +// disable warning about waiting for sync code, benchmarkdotnet does not support async code, and it doesn't deadlock when waiting. +[SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")] +public class AddSnapshotsBenchmarks +{ + private DataModelTestBase _template = null!; + private HashSet _measuredCommitIds = null!; + + private DataModelTestBase _local = null!; + private CrdtRepository _repository = null!; + private ObjectSnapshot[] _snapshotsToAdd = null!; + + [Params( + 1000 + // , 10_000 + )] + public int ChangeCount { get; set; } + + [ParamsAllValues] + public AddSnapshotsWorkload Workload { get; set; } + + [GlobalSetup] + public void GlobalSetup() + { + _template = new DataModelTestBase(alwaysValidate: false, performanceTest: true); + var clientId = Guid.NewGuid(); + + List seed; + List measured; + switch (Workload) + { + case AddSnapshotsWorkload.CreateNew: + seed = []; + measured = BenchmarkWorkloadBuilders.BuildCreateWords(_template, clientId, ChangeCount); + break; + case AddSnapshotsWorkload.MultiTypeCreate: + seed = []; + measured = BenchmarkWorkloadBuilders.BuildWordsWithDefinitions(_template, clientId, ChangeCount); + break; + case AddSnapshotsWorkload.ReferencedCreate: + seed = []; + measured = BenchmarkWorkloadBuilders.BuildWordsWithTags(_template, clientId, ChangeCount); + break; + case AddSnapshotsWorkload.UpdateExisting: + (seed, measured) = BenchmarkWorkloadBuilders.BuildUpdateExisting(_template, clientId, ChangeCount); + break; + case AddSnapshotsWorkload.ModifySameEntity: + seed = []; + measured = BenchmarkWorkloadBuilders.BuildModifySameWord(_template, clientId, ChangeCount); + break; + case AddSnapshotsWorkload.CreateThenDelete: + seed = []; + measured = BenchmarkWorkloadBuilders.BuildCreateThenDelete(_template, clientId, ChangeCount); + break; + case AddSnapshotsWorkload.CreateDeleteModify: + seed = []; + measured = BenchmarkWorkloadBuilders.BuildCreateDeleteModify(_template, clientId, ChangeCount); + break; + default: + throw new ArgumentOutOfRangeException(); + } + + // Seed commits go through the full pipeline so their snapshots and projected rows already exist. + if (seed.Count > 0) + ((ISyncable)_template.DataModel).AddRangeFromSync(seed).Wait(); + + // The measured commits are present in the database but their snapshots are NOT yet persisted; that's the + // work AddSnapshots performs. Adding only the commits mirrors the state right before UpdateSnapshots runs. + var repository = _template.CreateRepository(); + repository.AddCommits(measured).GetAwaiter().GetResult(); + _measuredCommitIds = measured.Select(c => c.Id).ToHashSet(); + } + + [IterationSetup] + public void IterationSetup() + { + _local = _template.ForkDatabase(alwaysValidate: false); + _repository = _local.CreateRepository(); + + // Load the measured commits fresh from the fork (tracked) so AddSnapshots resolves their Commit navigation + // from the change tracker instead of trying to re-insert them. + var measuredCommits = _local.DbContext.Commits + .Include(c => c.ChangeEntities) + .Where(c => EF.Parameter(_measuredCommitIds).Contains(c.Id)) + .ToArray() + .ToSortedSet(); + + // Prepopulate the snapshot lookup the same way DataModel.UpdateSnapshots does: existing snapshots (untracked) + // plus null for entities without one, so SnapshotWorker doesn't issue a per-entity query while computing. + var entityIds = measuredCommits + .SelectMany(c => c.ChangeEntities.Select(ce => ce.EntityId)) + .ToHashSet(); + var snapshotLookup = _repository.CurrentSnapshots() + .Include(s => s.Commit) + .Where(s => EF.Parameter(entityIds).Contains(s.EntityId)) + .ToDictionary(s => s.EntityId, s => (ObjectSnapshot?)s); + foreach (var entityId in entityIds) + snapshotLookup.TryAdd(entityId, null); + + var worker = new SnapshotWorker(snapshotLookup, _repository, _local.CrdtConfig); + _snapshotsToAdd = worker.ComputeSnapshotsToPersist(measuredCommits).GetAwaiter().GetResult().ToArray(); + } + + [Benchmark] + public void AddSnapshots() + { + _repository.AddSnapshots(_snapshotsToAdd).Wait(); + } + + [IterationCleanup] + public void IterationCleanup() + { + _repository.DisposeAsync().AsTask().Wait(); + _local.DisposeAsync().AsTask().Wait(); + _repository = null!; + _local = null!; + _snapshotsToAdd = null!; + } + + [GlobalCleanup] + public void GlobalCleanup() + { + _template.DisposeAsync().AsTask().Wait(); + _template = null!; + } +} diff --git a/src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs b/src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs new file mode 100644 index 0000000..15a4475 --- /dev/null +++ b/src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs @@ -0,0 +1,168 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Tests; + +namespace SIL.Harmony.Benchmarks; + +/// +/// Shared commit-building helpers for the benchmark suites so (full sync +/// pipeline) and (isolated snapshot persist) exercise the same domain scenarios. +/// Builders are pure: they only use the source for its change factories and +/// counter and never touch the database. +/// +public static class BenchmarkWorkloadBuilders +{ + public static Commit NewCommit(Guid clientId, DateTimeOffset dateTime, params IChange[] changes) + { + var commit = new Commit(Guid.NewGuid()) + { + ClientId = clientId, + HybridDateTime = new HybridDateTime(dateTime, 0) + }; + for (var i = 0; i < changes.Length; i++) + { + commit.ChangeEntities.Add(DataModel.ToChangeEntity(changes[i], i, commit.Id)); + } + return commit; + } + + /// Create many distinct words (one SetWord per commit). + public static List BuildCreateWords(DataModelTestBase src, Guid clientId, int count) + { + var commits = new List(count); + for (var i = 0; i < count; i++) + { + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(Guid.NewGuid(), $"entity {i}"))); + } + return commits; + } + + /// Create words each with a new definition in the same commit. + public static List BuildWordsWithDefinitions(DataModelTestBase src, Guid clientId, int count) + { + var commits = new List(count); + for (var i = 0; i < count; i++) + { + var wordId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordId, $"entity {i}"), + src.NewDefinition(wordId, $"definition {i}", "noun"))); + } + return commits; + } + + /// Create words each with a tag and WordTag link in the same commit. + public static List BuildWordsWithTags(DataModelTestBase src, Guid clientId, int count) + { + var commits = new List(count); + for (var i = 0; i < count; i++) + { + var wordId = Guid.NewGuid(); + var tagId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordId, $"entity {i}"), + src.SetTag(tagId, $"tag {i}"), + src.TagWord(wordId, tagId))); + } + return commits; + } + + /// Create one word, then modify that same word repeatedly. + public static List BuildModifySameWord(DataModelTestBase src, Guid clientId, int count) + { + var wordId = Guid.NewGuid(); + var commits = new List(count); + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordId, "entity 0"))); + for (var i = 1; i < count; i++) + { + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordId, $"entity {i}"))); + } + return commits; + } + + /// Create words then delete them. + public static List BuildCreateThenDelete(DataModelTestBase src, Guid clientId, int count) + { + var commits = new List(count * 2); + for (var i = 0; i < count; i++) + { + var wordId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordId, $"entity {i}"))); + commits.Add(NewCommit(clientId, src.NextDate(), + src.DeleteWord(wordId))); + } + return commits; + } + + /// Create, delete, then modify (apply-after-delete) for each word. + public static List BuildCreateDeleteModify(DataModelTestBase src, Guid clientId, int count) + { + var commits = new List(count * 3); + for (var i = 0; i < count; i++) + { + var wordId = Guid.NewGuid(); + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordId, $"entity {i}"))); + commits.Add(NewCommit(clientId, src.NextDate(), + src.DeleteWord(wordId))); + // SetWordNote supports apply-on-existing (including deleted) without undeleting + commits.Add(NewCommit(clientId, src.NextDate(), + src.SetWordNote(wordId, $"note {i}"))); + } + return commits; + } + + /// + /// Local already has create + late modify; sync inserts a mid-history modify for each word + /// (forces stale snapshot deletion / rebuild). Returns the seed commits and the mid-history commits to sync. + /// + public static (List seed, List toSync) BuildOutOfOrderInsert(DataModelTestBase src, Guid clientId, int count) + { + var seed = new List(count * 2); + var toSync = new List(count); + for (var i = 0; i < count; i++) + { + var wordId = Guid.NewGuid(); + var createTime = src.NextDate(); + var midTime = src.NextDate(); + var lateTime = src.NextDate(); + + seed.Add(NewCommit(clientId, createTime, + src.SetWord(wordId, $"entity {i}"))); + // Mid-history change is what gets synced after local already has create + late + toSync.Add(NewCommit(clientId, midTime, + src.SetWordNote(wordId, $"note {i}"))); + seed.Add(NewCommit(clientId, lateTime, + src.SetWord(wordId, $"entity {i} late"))); + } + + return (seed, toSync); + } + + /// + /// Seed a set of created words, then modify each one exactly once. The updates are the measured batch; + /// their snapshots must update the already-projected rows (FindAsync per snapshot in the slow path). + /// + public static (List seed, List measured) BuildUpdateExisting(DataModelTestBase src, Guid clientId, int count) + { + var wordIds = new Guid[count]; + var seed = new List(count); + for (var i = 0; i < count; i++) + { + wordIds[i] = Guid.NewGuid(); + seed.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordIds[i], $"entity {i}"))); + } + + var measured = new List(count); + for (var i = 0; i < count; i++) + { + measured.Add(NewCommit(clientId, src.NextDate(), + src.SetWord(wordIds[i], $"entity {i} updated"))); + } + return (seed, measured); + } +} diff --git a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs index ab31982..6e57514 100644 --- a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs @@ -58,17 +58,35 @@ public void GlobalSetup() _syncCommitIds = null; remote = new DataModelTestBase(alwaysValidate: false, performanceTest: true); var clientId = Guid.NewGuid(); - var commits = Workload switch + List commits; + switch (Workload) { - SyncWorkload.CreateWords => BuildCreateWords(clientId), - SyncWorkload.WordsWithDefinitions => BuildWordsWithDefinitions(clientId), - SyncWorkload.WordsWithTags => BuildWordsWithTags(clientId), - SyncWorkload.ModifySameWord => BuildModifySameWord(clientId), - SyncWorkload.CreateThenDelete => BuildCreateThenDelete(clientId), - SyncWorkload.CreateDeleteModify => BuildCreateDeleteModify(clientId), - SyncWorkload.OutOfOrderInsert => BuildOutOfOrderInsert(clientId), - _ => throw new ArgumentOutOfRangeException() - }; + case SyncWorkload.CreateWords: + commits = BenchmarkWorkloadBuilders.BuildCreateWords(remote, clientId, ChangeCount); + break; + case SyncWorkload.WordsWithDefinitions: + commits = BenchmarkWorkloadBuilders.BuildWordsWithDefinitions(remote, clientId, ChangeCount); + break; + case SyncWorkload.WordsWithTags: + commits = BenchmarkWorkloadBuilders.BuildWordsWithTags(remote, clientId, ChangeCount); + break; + case SyncWorkload.ModifySameWord: + commits = BenchmarkWorkloadBuilders.BuildModifySameWord(remote, clientId, ChangeCount); + break; + case SyncWorkload.CreateThenDelete: + commits = BenchmarkWorkloadBuilders.BuildCreateThenDelete(remote, clientId, ChangeCount); + break; + case SyncWorkload.CreateDeleteModify: + commits = BenchmarkWorkloadBuilders.BuildCreateDeleteModify(remote, clientId, ChangeCount); + break; + case SyncWorkload.OutOfOrderInsert: + var (seed, toSync) = BenchmarkWorkloadBuilders.BuildOutOfOrderInsert(remote, clientId, ChangeCount); + _syncCommitIds = toSync.Select(c => c.Id).ToHashSet(); + commits = [.. seed, .. toSync]; + break; + default: + throw new ArgumentOutOfRangeException(); + } ((ISyncable)remote.DataModel).AddRangeFromSync(commits).Wait(); } @@ -113,126 +131,4 @@ public void SyncCommits() ((ISyncable)local.DataModel).AddRangeFromSync(_commits) .Wait(); } - - private List BuildCreateWords(Guid clientId) - { - var commits = new List(ChangeCount); - for (var i = 0; i < ChangeCount; i++) - { - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(Guid.NewGuid(), $"entity {i}"))); - } - return commits; - } - - private List BuildWordsWithDefinitions(Guid clientId) - { - var commits = new List(ChangeCount); - for (var i = 0; i < ChangeCount; i++) - { - var wordId = Guid.NewGuid(); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, $"entity {i}"), - remote.NewDefinition(wordId, $"definition {i}", "noun"))); - } - return commits; - } - - private List BuildWordsWithTags(Guid clientId) - { - var commits = new List(ChangeCount); - for (var i = 0; i < ChangeCount; i++) - { - var wordId = Guid.NewGuid(); - var tagId = Guid.NewGuid(); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, $"entity {i}"), - remote.SetTag(tagId, $"tag {i}"), - remote.TagWord(wordId, tagId))); - } - return commits; - } - - private List BuildModifySameWord(Guid clientId) - { - var wordId = Guid.NewGuid(); - var commits = new List(ChangeCount); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, "entity 0"))); - for (var i = 1; i < ChangeCount; i++) - { - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, $"entity {i}"))); - } - return commits; - } - - private List BuildCreateThenDelete(Guid clientId) - { - var commits = new List(ChangeCount * 2); - for (var i = 0; i < ChangeCount; i++) - { - var wordId = Guid.NewGuid(); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, $"entity {i}"))); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.DeleteWord(wordId))); - } - return commits; - } - - private List BuildCreateDeleteModify(Guid clientId) - { - var commits = new List(ChangeCount * 3); - for (var i = 0; i < ChangeCount; i++) - { - var wordId = Guid.NewGuid(); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWord(wordId, $"entity {i}"))); - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.DeleteWord(wordId))); - // SetWordNote supports apply-on-existing (including deleted) without undeleting - commits.Add(NewCommit(clientId, remote.NextDate(), - remote.SetWordNote(wordId, $"note {i}"))); - } - return commits; - } - - private List BuildOutOfOrderInsert(Guid clientId) - { - var seed = new List(ChangeCount * 2); - var toSync = new List(ChangeCount); - for (var i = 0; i < ChangeCount; i++) - { - var wordId = Guid.NewGuid(); - var createTime = remote.NextDate(); - var midTime = remote.NextDate(); - var lateTime = remote.NextDate(); - - seed.Add(NewCommit(clientId, createTime, - remote.SetWord(wordId, $"entity {i}"))); - // Mid-history change is what gets synced after local already has create + late - toSync.Add(NewCommit(clientId, midTime, - remote.SetWordNote(wordId, $"note {i}"))); - seed.Add(NewCommit(clientId, lateTime, - remote.SetWord(wordId, $"entity {i} late"))); - } - - _syncCommitIds = toSync.Select(c => c.Id).ToHashSet(); - return [..seed, ..toSync]; - } - - private static Commit NewCommit(Guid clientId, DateTimeOffset dateTime, params IChange[] changes) - { - var commit = new Commit(Guid.NewGuid()) - { - ClientId = clientId, - HybridDateTime = new HybridDateTime(dateTime, 0) - }; - for (var i = 0; i < changes.Length; i++) - { - commit.ChangeEntities.Add(DataModel.ToChangeEntity(changes[i], i, commit.Id)); - } - return commit; - } } diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs index 33eb7e6..a63445d 100644 --- a/src/SIL.Harmony.Benchmarks/Program.cs +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -9,4 +9,6 @@ .AddJob(Job.MediumRun.WithStrategy(RunStrategy.Monitoring).WithId("DEFAULT").AsBaseline()) .AddJob(Job.MediumRun.WithMsBuildArguments("/p:DefineConstants=FAST").WithStrategy(RunStrategy.Monitoring) .WithId("FAST")); -BenchmarkRunner.Run(config, args); +BenchmarkSwitcher + .FromTypes([typeof(DataModelSyncBenchmarks), typeof(AddSnapshotsBenchmarks)]) + .Run(args, config); diff --git a/src/SIL.Harmony.Tests/DataModelTestBase.cs b/src/SIL.Harmony.Tests/DataModelTestBase.cs index bd1b177..3219ef4 100644 --- a/src/SIL.Harmony.Tests/DataModelTestBase.cs +++ b/src/SIL.Harmony.Tests/DataModelTestBase.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; using SIL.Harmony.Changes; using SIL.Harmony.Db; using SIL.Harmony.Sample; @@ -64,6 +65,15 @@ public void SetCurrentDate(DateTime dateTime) currentDate = dateTime; } + /// + /// Creates a repository over this instance's DbContext. Exposed so benchmarks can drive + /// methods (e.g. AddSnapshots) directly without going through the sync pipeline. + /// + internal CrdtRepository CreateRepository() => + _services.GetRequiredService().CreateRepositorySync(); + + internal CrdtConfig CrdtConfig => _services.GetRequiredService>().Value; + private static int _instanceCount = 0; private DateTimeOffset currentDate = new(new DateTime(2000, 1, 1, 0, 0, 0).AddHours(_instanceCount++)); public DateTimeOffset NextDate() => currentDate = currentDate.AddDays(1); diff --git a/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj b/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj index f542263..24f3074 100644 --- a/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj +++ b/src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj @@ -41,4 +41,8 @@ + + + + diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index bde97d8..402fefb 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -304,7 +304,6 @@ public async Task> GetChanges(SyncState remoteState) #if FAST public async Task AddSnapshots(IEnumerable snapshots) { - Console.WriteLine("FAST enabled ==========================================="); _dbContext.AddRange(snapshots); await _dbContext.SaveChangesAsync(); } @@ -315,8 +314,6 @@ public async Task AddSnapshots(IEnumerable snapshots) // pre-load the projected rows that already exist so ProjectSnapshot's FindAsync calls are served from the // change tracker instead of issuing one query per snapshot - Console.WriteLine("SLOW enabled ==========================================="); - var latestProjectByEntityId = new Dictionary(); foreach (var grouping in snapshotList.GroupBy(s => s.EntityIsDeleted).OrderByDescending(g => g.Key))//execute deletes first { diff --git a/src/SIL.Harmony/SnapshotWorker.cs b/src/SIL.Harmony/SnapshotWorker.cs index 86043a9..e43d918 100644 --- a/src/SIL.Harmony/SnapshotWorker.cs +++ b/src/SIL.Harmony/SnapshotWorker.cs @@ -59,6 +59,17 @@ await _crdtRepository.AddSnapshots([ ]); } + /// + /// Applies the commits to snapshots the same way does, but returns the full list + /// of snapshots that would be persisted instead of writing them. Used by benchmarks to isolate the + /// step from commit application. + /// + internal async Task> ComputeSnapshotsToPersist(SortedSet commits) + { + await ApplyCommitChanges(commits); + return [.._rootSnapshots.Values, .._newIntermediateSnapshots, .._pendingSnapshots.Values]; + } + private async ValueTask ApplyCommitChanges(SortedSet commits) { var intermediateSnapshots = new Dictionary(); From 3e9ec1ba7a4f45e1a916da2f92ef42799db1e7e5 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 16:13:38 +0700 Subject: [PATCH 08/12] Add raw-SQL projection fast paths for AddSnapshots Two experimental fast AddSnapshots implementations that keep the EF snapshot insert unchanged but populate projected tables with raw INSERT ... ON CONFLICT upserts instead of going through EF's change tracker: - FAST: one upsert command per entity row - FAST_JSON: one command per entity type, rows passed as a single JSON array expanded with SQLite json_each/json_extract FastProjection derives table/column names, primary key, the SnapshotId shadow FK, and value converters from the EF model (no per-entity code). It dedups to the latest snapshot per entity, runs deletes before upserts (children-first) then upserts (parents-first) for FK/unique-constraint safety, and reuses the caller's transaction. CrdtRepository.AddSnapshots now selects via #if FAST_JSON / #elif FAST / #else. Program.cs adds a third FAST_JSON benchmark job and DataModelSyncBenchmarks enables [MemoryDiagnoser]. Benchmarks (CreateWords, 1000): both fast paths ~35% faster and ~32% fewer allocations than baseline; per-query vs JSON-batch shows no measurable difference against in-memory SQLite. Co-Authored-By: Claude Opus 4.8 --- .../DataModelSyncBenchmarks.cs | 1 + src/SIL.Harmony.Benchmarks/Program.cs | 4 +- src/SIL.Harmony/Db/CrdtRepository.cs | 20 +- src/SIL.Harmony/Db/FastProjection.cs | 340 ++++++++++++++++++ 4 files changed, 358 insertions(+), 7 deletions(-) create mode 100644 src/SIL.Harmony/Db/FastProjection.cs diff --git a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs index 6e57514..3830b72 100644 --- a/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs +++ b/src/SIL.Harmony.Benchmarks/DataModelSyncBenchmarks.cs @@ -34,6 +34,7 @@ public enum SyncWorkload } // [SimpleJob(RunStrategy.Monitoring)] +[MemoryDiagnoser] [SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")] public class DataModelSyncBenchmarks { diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs index a63445d..08cdaef 100644 --- a/src/SIL.Harmony.Benchmarks/Program.cs +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -8,7 +8,9 @@ var config = DefaultConfig.Instance .AddJob(Job.MediumRun.WithStrategy(RunStrategy.Monitoring).WithId("DEFAULT").AsBaseline()) .AddJob(Job.MediumRun.WithMsBuildArguments("/p:DefineConstants=FAST").WithStrategy(RunStrategy.Monitoring) - .WithId("FAST")); + .WithId("FAST")) + .AddJob(Job.MediumRun.WithMsBuildArguments("/p:DefineConstants=FAST_JSON").WithStrategy(RunStrategy.Monitoring) + .WithId("FAST_JSON")); BenchmarkSwitcher .FromTypes([typeof(DataModelSyncBenchmarks), typeof(AddSnapshotsBenchmarks)]) .Run(args, config); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 402fefb..d8bd05a 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -301,12 +301,20 @@ public async Task> GetChanges(SyncState remoteState) return await _dbContext.Commits.GetChanges(remoteState); } -#if FAST - public async Task AddSnapshots(IEnumerable snapshots) -{ - _dbContext.AddRange(snapshots); - await _dbContext.SaveChangesAsync(); -} +#if FAST_JSON + public Task AddSnapshots(IEnumerable snapshots) + { + var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); + return FastProjection.AddSnapshotsRawAsync(_dbContext, snapshotList, + _crdtConfig.Value.EnableProjectedTables, useJsonBatch: true); + } +#elif FAST + public Task AddSnapshots(IEnumerable snapshots) + { + var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); + return FastProjection.AddSnapshotsRawAsync(_dbContext, snapshotList, + _crdtConfig.Value.EnableProjectedTables, useJsonBatch: false); + } #else public async Task AddSnapshots(IEnumerable snapshots) { diff --git a/src/SIL.Harmony/Db/FastProjection.cs b/src/SIL.Harmony/Db/FastProjection.cs new file mode 100644 index 0000000..28c65df --- /dev/null +++ b/src/SIL.Harmony/Db/FastProjection.cs @@ -0,0 +1,340 @@ +using System.Collections.Concurrent; +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; + +namespace SIL.Harmony.Db; + +/// +/// Experimental fast projection path used by the FAST / FAST_JSON benchmark builds. +/// Snapshots are still inserted through EF (unchanged), but the projected tables are populated +/// with hand-written raw SQL `INSERT ... ON CONFLICT(pk) DO UPDATE` instead of going through EF's +/// change tracker. Everything the SQL needs (table/column names, primary key, the SnapshotId shadow +/// FK, value converters) is derived from the EF model, so no per-entity code is required. +/// +/// Two variants: +/// - useJsonBatch == false: one upsert command per entity row (parameters rebound per row). +/// - useJsonBatch == true: one upsert command per entity type; all rows are passed as a +/// single JSON array parameter and expanded with SQLite's json_each / json_extract. +/// +internal static class FastProjection +{ + private static readonly ConcurrentDictionary TableInfoCache = new(); + + public static async Task AddSnapshotsRawAsync( + ICrdtDbContext dbContext, + IReadOnlyCollection snapshots, + bool enableProjectedTables, + bool useJsonBatch) + { + // AddSnapshots is normally called inside a caller-managed transaction; reuse it so the raw + // SQL runs on the same connection/transaction as the snapshot insert. When called outside a + // transaction (e.g. direct repository tests) we open and commit our own. + var ownTransaction = dbContext.Database.CurrentTransaction is null + ? await dbContext.Database.BeginTransactionAsync() + : null; + try + { + // 1. persist the snapshot rows exactly as before + dbContext.AddRange(snapshots); + await dbContext.SaveChangesAsync(); + + if (enableProjectedTables) + { + await ProjectAsync(dbContext, snapshots, useJsonBatch); + } + + if (ownTransaction is not null) await ownTransaction.CommitAsync(); + } + finally + { + if (ownTransaction is not null) await ownTransaction.DisposeAsync(); + } + } + + private static async Task ProjectAsync( + ICrdtDbContext dbContext, + IReadOnlyCollection snapshots, + bool useJsonBatch) + { + // 2. dedup to the latest snapshot per entity (a batch can contain several snapshots for the + // same entity - intermediate + latest - and possibly a deleted and undeleted one). + var latest = new Dictionary(); + foreach (var snapshot in snapshots) + { + if (latest.TryGetValue(snapshot.EntityId, out var existing) && + existing.Commit.CompareKey.CompareTo(snapshot.Commit.CompareKey) >= 0) + { + continue; + } + latest[snapshot.EntityId] = snapshot; + } + + var connection = dbContext.Database.GetDbConnection(); + var transaction = dbContext.Database.CurrentTransaction!.GetDbTransaction(); + var sqlHelper = dbContext.Database.GetService(); + + // 3. group by projected CLR type and order the types so FK parents are written first + var byType = latest.Values + .GroupBy(s => s.Entity.DbObject.GetType()) + .ToDictionary(g => g.Key, g => g.ToList()); + var orderedTypes = OrderTypesByDependency(dbContext.Model, byType.Keys); + + // deletes first (like the slow path) so a unique value can be freed and re-inserted within the + // same batch; children before parents (reverse dependency order) to satisfy FK constraints. + for (var i = orderedTypes.Count - 1; i >= 0; i--) + { + var deleted = byType[orderedTypes[i]].Where(s => s.EntityIsDeleted).ToList(); + if (deleted.Count == 0) continue; + var info = GetTableInfo(dbContext, orderedTypes[i], sqlHelper); + if (useJsonBatch) + await DeleteJsonBatchAsync(connection, transaction, info, deleted); + else + await DeletePerQueryAsync(connection, transaction, info, deleted); + } + + // upserts: parents first + foreach (var type in orderedTypes) + { + var live = byType[type].Where(s => !s.EntityIsDeleted).ToList(); + if (live.Count == 0) continue; + var info = GetTableInfo(dbContext, type, sqlHelper); + if (useJsonBatch) + await UpsertJsonBatchAsync(connection, transaction, info, live); + else + await UpsertPerQueryAsync(connection, transaction, info, live); + } + } + + // ---- per-query variant ------------------------------------------------------------------ + + private static async Task UpsertPerQueryAsync( + DbConnection connection, DbTransaction transaction, ProjectedTableInfo info, List rows) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = info.InsertSql; + var parameters = new DbParameter[info.Columns.Count]; + for (var i = 0; i < info.Columns.Count; i++) + { + var p = command.CreateParameter(); + p.ParameterName = "@p" + i; + command.Parameters.Add(p); + parameters[i] = p; + } + command.Prepare(); + + foreach (var snapshot in rows) + { + var dbObject = snapshot.Entity.DbObject; + for (var i = 0; i < info.Columns.Count; i++) + { + parameters[i].Value = ToParameterValue(GetProviderValue(info.Columns[i], snapshot, dbObject)); + } + await command.ExecuteNonQueryAsync(); + } + } + + private static async Task DeletePerQueryAsync( + DbConnection connection, DbTransaction transaction, ProjectedTableInfo info, List rows) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = info.DeleteSql; + var p = command.CreateParameter(); + p.ParameterName = "@p0"; + command.Parameters.Add(p); + command.Prepare(); + + foreach (var snapshot in rows) + { + p.Value = ToParameterValue(GetProviderValue(info.PrimaryKey, snapshot, snapshot.Entity.DbObject)); + await command.ExecuteNonQueryAsync(); + } + } + + // ---- json-batch variant ----------------------------------------------------------------- + + private static async Task UpsertJsonBatchAsync( + DbConnection connection, DbTransaction transaction, ProjectedTableInfo info, List rows) + { + var array = new List>(rows.Count); + foreach (var snapshot in rows) + { + var dbObject = snapshot.Entity.DbObject; + var obj = new Dictionary(info.Columns.Count); + foreach (var column in info.Columns) + { + obj[column.RawName] = ToJsonValue(GetProviderValue(column, snapshot, dbObject)); + } + array.Add(obj); + } + await ExecuteJsonAsync(connection, transaction, info.JsonInsertSql, array); + } + + private static async Task DeleteJsonBatchAsync( + DbConnection connection, DbTransaction transaction, ProjectedTableInfo info, List rows) + { + var array = new List>(rows.Count); + foreach (var snapshot in rows) + { + array.Add(new Dictionary(1) + { + [info.PrimaryKey.RawName] = ToJsonValue(GetProviderValue(info.PrimaryKey, snapshot, snapshot.Entity.DbObject)) + }); + } + await ExecuteJsonAsync(connection, transaction, info.JsonDeleteSql, array); + } + + private static async Task ExecuteJsonAsync( + DbConnection connection, DbTransaction transaction, string sql, List> array) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = sql; + var p = command.CreateParameter(); + p.ParameterName = "@json"; + p.Value = JsonSerializer.Serialize(array); + command.Parameters.Add(p); + await command.ExecuteNonQueryAsync(); + } + + // ---- value extraction ------------------------------------------------------------------- + + private static object? GetProviderValue(ColumnInfo column, ObjectSnapshot snapshot, object dbObject) + { + if (column.IsShadowSnapshotId) return snapshot.Id; + var raw = column.Property.PropertyInfo is { } pi + ? pi.GetValue(dbObject) + : column.Property.FieldInfo?.GetValue(dbObject); + var converter = column.Property.GetValueConverter(); + return converter is null ? raw : converter.ConvertToProvider(raw); + } + + private static object ToParameterValue(object? value) => value ?? DBNull.Value; + + /// + /// Converts a provider value into the exact text SQLite/Microsoft.Data.Sqlite stores, so values + /// round-tripped through json_extract match what EF writes elsewhere (notably GUID casing for + /// FK comparisons, which are case-sensitive on TEXT columns). + /// + private static object? ToJsonValue(object? value) => value switch + { + null => null, + Guid g => g.ToString().ToUpperInvariant(), + DateTimeOffset dto => dto.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFFzzz", CultureInfo.InvariantCulture), + DateTime dt => dt.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFF", CultureInfo.InvariantCulture), + bool b => b ? 1 : 0, + _ => value + }; + + // ---- model metadata --------------------------------------------------------------------- + + private static ProjectedTableInfo GetTableInfo(ICrdtDbContext dbContext, Type clrType, ISqlGenerationHelper sqlHelper) + { + return TableInfoCache.GetOrAdd(clrType, t => BuildTableInfo(dbContext, t, sqlHelper)); + } + + private static ProjectedTableInfo BuildTableInfo(ICrdtDbContext dbContext, Type clrType, ISqlGenerationHelper sqlHelper) + { + var entityType = dbContext.Model.FindEntityType(clrType) + ?? throw new InvalidOperationException($"No EF entity type found for projected type {clrType.Name}"); + var tableName = entityType.GetTableName() + ?? throw new InvalidOperationException($"No table name found for projected type {clrType.Name}"); + var schema = entityType.GetSchema(); + var storeObject = StoreObjectIdentifier.Table(tableName, schema); + var pkPropertyNames = (entityType.FindPrimaryKey()?.Properties + ?? throw new InvalidOperationException($"No primary key found for projected type {clrType.Name}")) + .Select(p => p.Name) + .ToHashSet(); + + var columns = new List(); + foreach (var property in entityType.GetProperties()) + { + var columnName = property.GetColumnName(storeObject); + if (columnName is null) continue; // not mapped to this table + columns.Add(new ColumnInfo( + columnName, + sqlHelper.DelimitIdentifier(columnName), + property, + property.Name == ObjectSnapshot.ShadowRefName, + pkPropertyNames.Contains(property.Name))); + } + + var pkColumns = columns.Where(c => c.IsPrimaryKey).ToList(); + if (pkColumns.Count != 1) + throw new NotSupportedException($"Fast projection requires a single-column primary key for {clrType.Name}"); + var pk = pkColumns[0]; + + var delimitedTable = sqlHelper.DelimitIdentifier(tableName, schema); + var columnList = string.Join(",", columns.Select(c => c.DelimitedName)); + var parameterList = string.Join(",", columns.Select((_, i) => "@p" + i)); + var conflictTarget = pk.DelimitedName; + var setClause = string.Join(",", columns.Where(c => !c.IsPrimaryKey) + .Select(c => $"{c.DelimitedName}=excluded.{c.DelimitedName}")); + var onConflict = string.IsNullOrEmpty(setClause) + ? $"ON CONFLICT ({conflictTarget}) DO NOTHING" + : $"ON CONFLICT ({conflictTarget}) DO UPDATE SET {setClause}"; + + var jsonSelectList = string.Join(",", columns.Select(c => $"json_extract(value,'$.\"{c.RawName}\"')")); + + return new ProjectedTableInfo( + columns, + pk, + InsertSql: $"INSERT INTO {delimitedTable} ({columnList}) VALUES ({parameterList}) {onConflict};", + DeleteSql: $"DELETE FROM {delimitedTable} WHERE {pk.DelimitedName}=@p0;", + // `WHERE true` disambiguates the trailing ON CONFLICT from a join constraint on the SELECT + // (a SQLite upsert-after-SELECT parser requirement) + JsonInsertSql: $"INSERT INTO {delimitedTable} ({columnList}) SELECT {jsonSelectList} FROM json_each(@json) WHERE true {onConflict};", + JsonDeleteSql: $"DELETE FROM {delimitedTable} WHERE {pk.DelimitedName} IN (SELECT json_extract(value,'$.\"{pk.RawName}\"') FROM json_each(@json));"); + } + + /// + /// Post-order DFS over FK edges so principal (parent) types come before dependents. Self + /// references and FKs to non-projected types (e.g. the SnapshotId FK to Snapshots) are ignored. + /// + private static List OrderTypesByDependency(IModel model, IEnumerable types) + { + var typeSet = types.ToHashSet(); + var ordered = new List(typeSet.Count); + var visited = new HashSet(); + + void Visit(Type type) + { + if (!visited.Add(type)) return; + var entityType = model.FindEntityType(type); + if (entityType is not null) + { + foreach (var fk in entityType.GetForeignKeys()) + { + var principal = fk.PrincipalEntityType.ClrType; + if (principal != type && typeSet.Contains(principal)) Visit(principal); + } + } + ordered.Add(type); + } + + foreach (var type in typeSet) Visit(type); + return ordered; + } + + private sealed record ColumnInfo( + string RawName, + string DelimitedName, + IProperty Property, + bool IsShadowSnapshotId, + bool IsPrimaryKey); + + private sealed record ProjectedTableInfo( + IReadOnlyList Columns, + ColumnInfo PrimaryKey, + string InsertSql, + string DeleteSql, + string JsonInsertSql, + string JsonDeleteSql); +} From 16a1364c60e53f8a172ea672a95e04570d517f18 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 16:42:21 +0700 Subject: [PATCH 09/12] Drop FAST_JSON projection variant, keep per-query fast path The JSON-batch projection benchmarked identically to the per-query path against in-memory SQLite (same time and allocations), so remove it and keep only the per-query raw-SQL upsert path. FastProjection loses the useJsonBatch parameter and all json_each/json_extract code; CrdtRepository.AddSnapshots collapses to #if FAST / #else; the benchmark drops the FAST_JSON job. Co-Authored-By: Claude Opus 4.8 --- src/SIL.Harmony.Benchmarks/Program.cs | 4 +- src/SIL.Harmony/Db/CrdtRepository.cs | 11 +-- src/SIL.Harmony/Db/FastProjection.cs | 121 ++++---------------------- 3 files changed, 18 insertions(+), 118 deletions(-) diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs index 08cdaef..a63445d 100644 --- a/src/SIL.Harmony.Benchmarks/Program.cs +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -8,9 +8,7 @@ var config = DefaultConfig.Instance .AddJob(Job.MediumRun.WithStrategy(RunStrategy.Monitoring).WithId("DEFAULT").AsBaseline()) .AddJob(Job.MediumRun.WithMsBuildArguments("/p:DefineConstants=FAST").WithStrategy(RunStrategy.Monitoring) - .WithId("FAST")) - .AddJob(Job.MediumRun.WithMsBuildArguments("/p:DefineConstants=FAST_JSON").WithStrategy(RunStrategy.Monitoring) - .WithId("FAST_JSON")); + .WithId("FAST")); BenchmarkSwitcher .FromTypes([typeof(DataModelSyncBenchmarks), typeof(AddSnapshotsBenchmarks)]) .Run(args, config); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index d8bd05a..ef9bcca 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -301,19 +301,12 @@ public async Task> GetChanges(SyncState remoteState) return await _dbContext.Commits.GetChanges(remoteState); } -#if FAST_JSON +#if FAST public Task AddSnapshots(IEnumerable snapshots) { var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); return FastProjection.AddSnapshotsRawAsync(_dbContext, snapshotList, - _crdtConfig.Value.EnableProjectedTables, useJsonBatch: true); - } -#elif FAST - public Task AddSnapshots(IEnumerable snapshots) - { - var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); - return FastProjection.AddSnapshotsRawAsync(_dbContext, snapshotList, - _crdtConfig.Value.EnableProjectedTables, useJsonBatch: false); + _crdtConfig.Value.EnableProjectedTables); } #else public async Task AddSnapshots(IEnumerable snapshots) diff --git a/src/SIL.Harmony/Db/FastProjection.cs b/src/SIL.Harmony/Db/FastProjection.cs index 28c65df..7f024f9 100644 --- a/src/SIL.Harmony/Db/FastProjection.cs +++ b/src/SIL.Harmony/Db/FastProjection.cs @@ -1,8 +1,5 @@ using System.Collections.Concurrent; -using System.Data; using System.Data.Common; -using System.Globalization; -using System.Text.Json; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; @@ -11,16 +8,12 @@ namespace SIL.Harmony.Db; /// -/// Experimental fast projection path used by the FAST / FAST_JSON benchmark builds. -/// Snapshots are still inserted through EF (unchanged), but the projected tables are populated -/// with hand-written raw SQL `INSERT ... ON CONFLICT(pk) DO UPDATE` instead of going through EF's -/// change tracker. Everything the SQL needs (table/column names, primary key, the SnapshotId shadow -/// FK, value converters) is derived from the EF model, so no per-entity code is required. -/// -/// Two variants: -/// - useJsonBatch == false: one upsert command per entity row (parameters rebound per row). -/// - useJsonBatch == true: one upsert command per entity type; all rows are passed as a -/// single JSON array parameter and expanded with SQLite's json_each / json_extract. +/// Fast projection path used by the FAST benchmark build. Snapshots are still inserted through EF +/// (unchanged), but the projected tables are populated with hand-written raw SQL +/// `INSERT ... ON CONFLICT(pk) DO UPDATE` (one upsert command per entity row) instead of going +/// through EF's change tracker. Everything the SQL needs (table/column names, primary key, the +/// SnapshotId shadow FK, value converters) is derived from the EF model, so no per-entity code is +/// required. /// internal static class FastProjection { @@ -29,8 +22,7 @@ internal static class FastProjection public static async Task AddSnapshotsRawAsync( ICrdtDbContext dbContext, IReadOnlyCollection snapshots, - bool enableProjectedTables, - bool useJsonBatch) + bool enableProjectedTables) { // AddSnapshots is normally called inside a caller-managed transaction; reuse it so the raw // SQL runs on the same connection/transaction as the snapshot insert. When called outside a @@ -46,7 +38,7 @@ public static async Task AddSnapshotsRawAsync( if (enableProjectedTables) { - await ProjectAsync(dbContext, snapshots, useJsonBatch); + await ProjectAsync(dbContext, snapshots); } if (ownTransaction is not null) await ownTransaction.CommitAsync(); @@ -59,8 +51,7 @@ public static async Task AddSnapshotsRawAsync( private static async Task ProjectAsync( ICrdtDbContext dbContext, - IReadOnlyCollection snapshots, - bool useJsonBatch) + IReadOnlyCollection snapshots) { // 2. dedup to the latest snapshot per entity (a batch can contain several snapshots for the // same entity - intermediate + latest - and possibly a deleted and undeleted one). @@ -92,10 +83,7 @@ private static async Task ProjectAsync( var deleted = byType[orderedTypes[i]].Where(s => s.EntityIsDeleted).ToList(); if (deleted.Count == 0) continue; var info = GetTableInfo(dbContext, orderedTypes[i], sqlHelper); - if (useJsonBatch) - await DeleteJsonBatchAsync(connection, transaction, info, deleted); - else - await DeletePerQueryAsync(connection, transaction, info, deleted); + await DeletePerQueryAsync(connection, transaction, info, deleted); } // upserts: parents first @@ -104,15 +92,10 @@ private static async Task ProjectAsync( var live = byType[type].Where(s => !s.EntityIsDeleted).ToList(); if (live.Count == 0) continue; var info = GetTableInfo(dbContext, type, sqlHelper); - if (useJsonBatch) - await UpsertJsonBatchAsync(connection, transaction, info, live); - else - await UpsertPerQueryAsync(connection, transaction, info, live); + await UpsertPerQueryAsync(connection, transaction, info, live); } } - // ---- per-query variant ------------------------------------------------------------------ - private static async Task UpsertPerQueryAsync( DbConnection connection, DbTransaction transaction, ProjectedTableInfo info, List rows) { @@ -158,54 +141,6 @@ private static async Task DeletePerQueryAsync( } } - // ---- json-batch variant ----------------------------------------------------------------- - - private static async Task UpsertJsonBatchAsync( - DbConnection connection, DbTransaction transaction, ProjectedTableInfo info, List rows) - { - var array = new List>(rows.Count); - foreach (var snapshot in rows) - { - var dbObject = snapshot.Entity.DbObject; - var obj = new Dictionary(info.Columns.Count); - foreach (var column in info.Columns) - { - obj[column.RawName] = ToJsonValue(GetProviderValue(column, snapshot, dbObject)); - } - array.Add(obj); - } - await ExecuteJsonAsync(connection, transaction, info.JsonInsertSql, array); - } - - private static async Task DeleteJsonBatchAsync( - DbConnection connection, DbTransaction transaction, ProjectedTableInfo info, List rows) - { - var array = new List>(rows.Count); - foreach (var snapshot in rows) - { - array.Add(new Dictionary(1) - { - [info.PrimaryKey.RawName] = ToJsonValue(GetProviderValue(info.PrimaryKey, snapshot, snapshot.Entity.DbObject)) - }); - } - await ExecuteJsonAsync(connection, transaction, info.JsonDeleteSql, array); - } - - private static async Task ExecuteJsonAsync( - DbConnection connection, DbTransaction transaction, string sql, List> array) - { - await using var command = connection.CreateCommand(); - command.Transaction = transaction; - command.CommandText = sql; - var p = command.CreateParameter(); - p.ParameterName = "@json"; - p.Value = JsonSerializer.Serialize(array); - command.Parameters.Add(p); - await command.ExecuteNonQueryAsync(); - } - - // ---- value extraction ------------------------------------------------------------------- - private static object? GetProviderValue(ColumnInfo column, ObjectSnapshot snapshot, object dbObject) { if (column.IsShadowSnapshotId) return snapshot.Id; @@ -218,21 +153,6 @@ private static async Task ExecuteJsonAsync( private static object ToParameterValue(object? value) => value ?? DBNull.Value; - /// - /// Converts a provider value into the exact text SQLite/Microsoft.Data.Sqlite stores, so values - /// round-tripped through json_extract match what EF writes elsewhere (notably GUID casing for - /// FK comparisons, which are case-sensitive on TEXT columns). - /// - private static object? ToJsonValue(object? value) => value switch - { - null => null, - Guid g => g.ToString().ToUpperInvariant(), - DateTimeOffset dto => dto.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFFzzz", CultureInfo.InvariantCulture), - DateTime dt => dt.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFF", CultureInfo.InvariantCulture), - bool b => b ? 1 : 0, - _ => value - }; - // ---- model metadata --------------------------------------------------------------------- private static ProjectedTableInfo GetTableInfo(ICrdtDbContext dbContext, Type clrType, ISqlGenerationHelper sqlHelper) @@ -259,7 +179,6 @@ private static ProjectedTableInfo BuildTableInfo(ICrdtDbContext dbContext, Type var columnName = property.GetColumnName(storeObject); if (columnName is null) continue; // not mapped to this table columns.Add(new ColumnInfo( - columnName, sqlHelper.DelimitIdentifier(columnName), property, property.Name == ObjectSnapshot.ShadowRefName, @@ -274,24 +193,17 @@ private static ProjectedTableInfo BuildTableInfo(ICrdtDbContext dbContext, Type var delimitedTable = sqlHelper.DelimitIdentifier(tableName, schema); var columnList = string.Join(",", columns.Select(c => c.DelimitedName)); var parameterList = string.Join(",", columns.Select((_, i) => "@p" + i)); - var conflictTarget = pk.DelimitedName; var setClause = string.Join(",", columns.Where(c => !c.IsPrimaryKey) .Select(c => $"{c.DelimitedName}=excluded.{c.DelimitedName}")); var onConflict = string.IsNullOrEmpty(setClause) - ? $"ON CONFLICT ({conflictTarget}) DO NOTHING" - : $"ON CONFLICT ({conflictTarget}) DO UPDATE SET {setClause}"; - - var jsonSelectList = string.Join(",", columns.Select(c => $"json_extract(value,'$.\"{c.RawName}\"')")); + ? $"ON CONFLICT ({pk.DelimitedName}) DO NOTHING" + : $"ON CONFLICT ({pk.DelimitedName}) DO UPDATE SET {setClause}"; return new ProjectedTableInfo( columns, pk, InsertSql: $"INSERT INTO {delimitedTable} ({columnList}) VALUES ({parameterList}) {onConflict};", - DeleteSql: $"DELETE FROM {delimitedTable} WHERE {pk.DelimitedName}=@p0;", - // `WHERE true` disambiguates the trailing ON CONFLICT from a join constraint on the SELECT - // (a SQLite upsert-after-SELECT parser requirement) - JsonInsertSql: $"INSERT INTO {delimitedTable} ({columnList}) SELECT {jsonSelectList} FROM json_each(@json) WHERE true {onConflict};", - JsonDeleteSql: $"DELETE FROM {delimitedTable} WHERE {pk.DelimitedName} IN (SELECT json_extract(value,'$.\"{pk.RawName}\"') FROM json_each(@json));"); + DeleteSql: $"DELETE FROM {delimitedTable} WHERE {pk.DelimitedName}=@p0;"); } /// @@ -324,7 +236,6 @@ void Visit(Type type) } private sealed record ColumnInfo( - string RawName, string DelimitedName, IProperty Property, bool IsShadowSnapshotId, @@ -334,7 +245,5 @@ private sealed record ProjectedTableInfo( IReadOnlyList Columns, ColumnInfo PrimaryKey, string InsertSql, - string DeleteSql, - string JsonInsertSql, - string JsonDeleteSql); + string DeleteSql); } From ce441afb888c0283f861b1c53525574d2a566ec7 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 16:45:39 +0700 Subject: [PATCH 10/12] Make raw-SQL projection the only AddSnapshots path Remove the EF change-tracker slow path and the #if FAST conditional so AddSnapshots always uses FastProjection. Deletes the now-dead slow-path helpers (ProjectSnapshot, GetEntityEntry, LoadExistingEntityIds, LoadExistingEntities). The benchmark collapses to a single job since FAST vs DEFAULT are now identical. Co-Authored-By: Claude Opus 4.8 --- src/SIL.Harmony.Benchmarks/Program.cs | 4 +- src/SIL.Harmony/Db/CrdtRepository.cs | 124 -------------------------- src/SIL.Harmony/Db/FastProjection.cs | 4 +- 3 files changed, 3 insertions(+), 129 deletions(-) diff --git a/src/SIL.Harmony.Benchmarks/Program.cs b/src/SIL.Harmony.Benchmarks/Program.cs index a63445d..23e438d 100644 --- a/src/SIL.Harmony.Benchmarks/Program.cs +++ b/src/SIL.Harmony.Benchmarks/Program.cs @@ -6,9 +6,7 @@ var config = DefaultConfig.Instance - .AddJob(Job.MediumRun.WithStrategy(RunStrategy.Monitoring).WithId("DEFAULT").AsBaseline()) - .AddJob(Job.MediumRun.WithMsBuildArguments("/p:DefineConstants=FAST").WithStrategy(RunStrategy.Monitoring) - .WithId("FAST")); + .AddJob(Job.MediumRun.WithStrategy(RunStrategy.Monitoring)); BenchmarkSwitcher .FromTypes([typeof(DataModelSyncBenchmarks), typeof(AddSnapshotsBenchmarks)]) .Run(args, config); diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index ef9bcca..ba0a43a 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -301,136 +301,12 @@ public async Task> GetChanges(SyncState remoteState) return await _dbContext.Commits.GetChanges(remoteState); } -#if FAST public Task AddSnapshots(IEnumerable snapshots) { var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); return FastProjection.AddSnapshotsRawAsync(_dbContext, snapshotList, _crdtConfig.Value.EnableProjectedTables); } -#else - public async Task AddSnapshots(IEnumerable snapshots) - { - var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); - // pre-load the projected rows that already exist so ProjectSnapshot's FindAsync calls are served from the - // change tracker instead of issuing one query per snapshot - - var latestProjectByEntityId = new Dictionary(); - foreach (var grouping in snapshotList.GroupBy(s => s.EntityIsDeleted).OrderByDescending(g => g.Key))//execute deletes first - { - foreach (var snapshot in grouping.DefaultOrderDescending()) - { - _dbContext.Add(snapshot); - if (latestProjectByEntityId.TryGetValue(snapshot.EntityId, out var latestProjected)) - { - // there might be a deleted and un-deleted snapshot for the same entity in the same batch - // in that case there's only a 50% chance that they're in the right order, so we need to explicitly only project the latest one - if (snapshot.Commit.CompareKey.CompareTo(latestProjected) < 0) - { - continue; - } - } - latestProjectByEntityId[snapshot.EntityId] = snapshot.Commit.CompareKey; - - await ProjectSnapshot(snapshot, null); - } - - try - { - await _dbContext.SaveChangesAsync(); - } - catch (DbUpdateException e) - { - var entries = string.Join(Environment.NewLine, e.Entries.Select(entry => entry.ToString())); - var message = $"Error saving snapshots (deleted: {grouping.Key}): {e.Message}{Environment.NewLine}{entries}"; - _logger.LogError(e, message); - throw new DbUpdateException(message, e); - } - } - } -#endif - /// - /// Loads the projected rows that already exist for the given snapshots so they're tracked by the context. - /// This lets ProjectSnapshot resolve existing entities from the change tracker (and skip lookups for new - /// entities entirely) rather than running one FindAsync query per snapshot. - /// - /// the set of entity ids that already have a projected row - private async Task> LoadExistingEntityIds(IReadOnlyCollection snapshots) - { - var existingEntityIds = new HashSet(); - if (!_crdtConfig.Value.EnableProjectedTables) return existingEntityIds; - foreach (var typeGroup in snapshots.GroupBy(s => s.Entity.DbObject.GetType())) - { - var entityIds = typeGroup.Select(s => s.EntityId).Distinct().ToArray(); - var task = (Task>)loadExistingEntitiesMethod - .MakeGenericMethod(typeGroup.Key) - .Invoke(null, [_dbContext, entityIds])!; - existingEntityIds.UnionWith(await task); - } - return existingEntityIds; - } - - private static readonly MethodInfo loadExistingEntitiesMethod = - new Func, Task>>(LoadExistingEntities) - .Method.GetGenericMethodDefinition(); - - private static async Task> LoadExistingEntities(ICrdtDbContext dbContext, IReadOnlyCollection entityIds) - where T : class - { - var keyName = dbContext.Model.FindEntityType(typeof(T))?.FindPrimaryKey()?.Properties.Single().Name - ?? throw new InvalidOperationException($"No primary key found for projected type {typeof(T).Name}"); - //load (and thereby track) the matching rows so ProjectSnapshot's FindAsync calls hit the change tracker. - //EF.Parameter forces a single JSON parameter; without it EF 10+ emits one parameter per id and overflows SQLite's parameter limit - var entities = await dbContext.Set() - .Where(e => EF.Parameter(entityIds).Contains(EF.Property(e, keyName))) - .ToListAsync(); - return entities - .Select(e => (Guid)dbContext.Entry(e).Property(keyName).CurrentValue!) - .ToList(); - } - - private async ValueTask ProjectSnapshot(ObjectSnapshot objectSnapshot, ISet? existingEntityIds) - { - if (!_crdtConfig.Value.EnableProjectedTables) return; - - //need to check if an entry exists already, even if this is the root commit it may have already been added to the db - //entities not in existingEntityIds have no projected row yet, so skip the lookup and treat them as new - var existingEntry = existingEntityIds is null || existingEntityIds.Contains(objectSnapshot.EntityId) - ? await GetEntityEntry(objectSnapshot.Entity.DbObject.GetType(), objectSnapshot.EntityId) - : null; - if (existingEntry is null && objectSnapshot.EntityIsDeleted) return; - - if (existingEntry is null) // add - { - // this is a new entity even though it might not be a root snapshot, because we only project the latest snapshot of each entity per sync - - //if we don't make a copy first then the entity will be tracked by the context and be modified - //by future changes in the same session - var entity = objectSnapshot.Entity.Copy().DbObject; - - var newEntry = _dbContext.Entry(entity); - // only mark this single entry as added, rather than the whole graph (this matches the update behaviour below) - newEntry.State = EntityState.Added; - newEntry.Property(ObjectSnapshot.ShadowRefName).CurrentValue = objectSnapshot.Id; - } - else if (objectSnapshot.EntityIsDeleted) // delete - { - _dbContext.Remove(existingEntry.Entity); - } - else // update - { - var entity = objectSnapshot.Entity.DbObject; - existingEntry.CurrentValues.SetValues(entity); - existingEntry.Property(ObjectSnapshot.ShadowRefName).CurrentValue = objectSnapshot.Id; - } - } - - private async ValueTask GetEntityEntry(Type entityType, Guid entityId) - { - if (!_crdtConfig.Value.EnableProjectedTables) return null; - var entity = await _dbContext.FindAsync(entityType, entityId); - return entity is not null ? _dbContext.Entry(entity) : null; - } public CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) { diff --git a/src/SIL.Harmony/Db/FastProjection.cs b/src/SIL.Harmony/Db/FastProjection.cs index 7f024f9..25e42f0 100644 --- a/src/SIL.Harmony/Db/FastProjection.cs +++ b/src/SIL.Harmony/Db/FastProjection.cs @@ -8,8 +8,8 @@ namespace SIL.Harmony.Db; /// -/// Fast projection path used by the FAST benchmark build. Snapshots are still inserted through EF -/// (unchanged), but the projected tables are populated with hand-written raw SQL +/// Projects snapshots into the projected tables. Snapshots are still inserted through EF (unchanged), +/// but the projected tables are populated with hand-written raw SQL /// `INSERT ... ON CONFLICT(pk) DO UPDATE` (one upsert command per entity row) instead of going /// through EF's change tracker. Everything the SQL needs (table/column names, primary key, the /// SnapshotId shadow FK, value converters) is derived from the EF model, so no per-entity code is From 48b7ad3de4f1385975b03a84b8809a00db050b3d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 Jul 2026 16:54:21 +0700 Subject: [PATCH 11/12] Make FastProjection an injectable service FastProjection is now an injected singleton (registered in AddCrdtDataCore and resolved into CrdtRepository via ActivatorUtilities) instead of a static class. Its per-type projected-table SQL metadata cache moves from a static field onto an internal ConcurrentDictionary on CrdtConfig, so it's shared across repositories/contexts and tied to config lifetime. Co-Authored-By: Claude Opus 4.8 --- src/SIL.Harmony/CrdtConfig.cs | 8 ++++++++ src/SIL.Harmony/CrdtKernel.cs | 1 + src/SIL.Harmony/Db/CrdtRepository.cs | 8 +++++--- src/SIL.Harmony/Db/FastProjection.cs | 28 ++++++++++++++++------------ 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/SIL.Harmony/CrdtConfig.cs b/src/SIL.Harmony/CrdtConfig.cs index 01d9d13..e550af3 100644 --- a/src/SIL.Harmony/CrdtConfig.cs +++ b/src/SIL.Harmony/CrdtConfig.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Text.Json; using System.Text.Json.Serialization.Metadata; using Microsoft.EntityFrameworkCore; @@ -31,6 +32,13 @@ public class CrdtConfig private readonly Lazy _lazyJsonSerializerOptions; private readonly Lazy _lazyChangeDiscriminatorMaps; + /// + /// Cache of derived projected-table SQL metadata (keyed by projected CLR type), used by + /// . Stored on the config so it's shared across repositories and + /// db contexts and only built once per type. + /// + internal ConcurrentDictionary ProjectedTableInfoCache { get; } = new(); + public CrdtConfig() { _lazyChangeDiscriminatorMaps = new Lazy(BuildChangeDiscriminatorMaps); diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index 7e4d888..467cb94 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -51,6 +51,7 @@ public static IServiceCollection AddCrdtDataCore(this IServiceCollection service services.AddSingleton(sp => sp.GetRequiredService>().Value.JsonSerializerOptions); services.AddSingleton(TimeProvider.System); services.AddScoped(NewTimeProvider); + services.AddSingleton(); services.AddScoped(); //must use factory method because DataModel constructor is internal services.AddScoped(provider => new DataModel( diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index ba0a43a..6abc82a 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -53,14 +53,17 @@ internal class CrdtRepository : IDisposable, IAsyncDisposable private readonly ICrdtDbContext _dbContext; private readonly IOptions _crdtConfig; private readonly ILogger _logger; + private readonly FastProjection _fastProjection; public CrdtRepository(ICrdtDbContext dbContext, IOptions crdtConfig, ILogger logger, + FastProjection fastProjection, Commit? ignoreChangesAfter = null) { _crdtConfig = crdtConfig; _dbContext = ignoreChangesAfter is not null ? new ScopedDbContext(dbContext, ignoreChangesAfter) : dbContext; _logger = logger; + _fastProjection = fastProjection; //we can't use the scoped db context is it prevents access to the DbSet for the Snapshots, //but since we're using a custom query, we can use it directly and apply the scoped filters manually _currentSnapshotsQueryable = MakeCurrentSnapshotsQuery(dbContext, ignoreChangesAfter); @@ -304,13 +307,12 @@ public async Task> GetChanges(SyncState remoteState) public Task AddSnapshots(IEnumerable snapshots) { var snapshotList = snapshots as IReadOnlyCollection ?? snapshots.ToArray(); - return FastProjection.AddSnapshotsRawAsync(_dbContext, snapshotList, - _crdtConfig.Value.EnableProjectedTables); + return _fastProjection.AddSnapshotsRawAsync(_dbContext, snapshotList); } public CrdtRepository GetScopedRepository(Commit excludeChangesAfterCommit) { - return new CrdtRepository(_dbContext, _crdtConfig, _logger, excludeChangesAfterCommit); + return new CrdtRepository(_dbContext, _crdtConfig, _logger, _fastProjection, excludeChangesAfterCommit); } /// diff --git a/src/SIL.Harmony/Db/FastProjection.cs b/src/SIL.Harmony/Db/FastProjection.cs index 25e42f0..55ed653 100644 --- a/src/SIL.Harmony/Db/FastProjection.cs +++ b/src/SIL.Harmony/Db/FastProjection.cs @@ -1,9 +1,9 @@ -using System.Collections.Concurrent; using System.Data.Common; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Options; namespace SIL.Harmony.Db; @@ -15,14 +15,18 @@ namespace SIL.Harmony.Db; /// SnapshotId shadow FK, value converters) is derived from the EF model, so no per-entity code is /// required. /// -internal static class FastProjection +internal class FastProjection { - private static readonly ConcurrentDictionary TableInfoCache = new(); + private readonly CrdtConfig _crdtConfig; - public static async Task AddSnapshotsRawAsync( + public FastProjection(IOptions crdtConfig) + { + _crdtConfig = crdtConfig.Value; + } + + public async Task AddSnapshotsRawAsync( ICrdtDbContext dbContext, - IReadOnlyCollection snapshots, - bool enableProjectedTables) + IReadOnlyCollection snapshots) { // AddSnapshots is normally called inside a caller-managed transaction; reuse it so the raw // SQL runs on the same connection/transaction as the snapshot insert. When called outside a @@ -36,7 +40,7 @@ public static async Task AddSnapshotsRawAsync( dbContext.AddRange(snapshots); await dbContext.SaveChangesAsync(); - if (enableProjectedTables) + if (_crdtConfig.EnableProjectedTables) { await ProjectAsync(dbContext, snapshots); } @@ -49,7 +53,7 @@ public static async Task AddSnapshotsRawAsync( } } - private static async Task ProjectAsync( + private async Task ProjectAsync( ICrdtDbContext dbContext, IReadOnlyCollection snapshots) { @@ -155,9 +159,9 @@ private static async Task DeletePerQueryAsync( // ---- model metadata --------------------------------------------------------------------- - private static ProjectedTableInfo GetTableInfo(ICrdtDbContext dbContext, Type clrType, ISqlGenerationHelper sqlHelper) + private ProjectedTableInfo GetTableInfo(ICrdtDbContext dbContext, Type clrType, ISqlGenerationHelper sqlHelper) { - return TableInfoCache.GetOrAdd(clrType, t => BuildTableInfo(dbContext, t, sqlHelper)); + return _crdtConfig.ProjectedTableInfoCache.GetOrAdd(clrType, t => BuildTableInfo(dbContext, t, sqlHelper)); } private static ProjectedTableInfo BuildTableInfo(ICrdtDbContext dbContext, Type clrType, ISqlGenerationHelper sqlHelper) @@ -235,13 +239,13 @@ void Visit(Type type) return ordered; } - private sealed record ColumnInfo( + internal sealed record ColumnInfo( string DelimitedName, IProperty Property, bool IsShadowSnapshotId, bool IsPrimaryKey); - private sealed record ProjectedTableInfo( + internal sealed record ProjectedTableInfo( IReadOnlyList Columns, ColumnInfo PrimaryKey, string InsertSql, From 245babbb377c65c4f5d6d7513b0e5c000ec78d38 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 22 Jul 2026 10:05:51 +0700 Subject: [PATCH 12/12] fix formatting --- src/SIL.Harmony/SnapshotWorker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SIL.Harmony/SnapshotWorker.cs b/src/SIL.Harmony/SnapshotWorker.cs index e43d918..185e7b4 100644 --- a/src/SIL.Harmony/SnapshotWorker.cs +++ b/src/SIL.Harmony/SnapshotWorker.cs @@ -67,7 +67,7 @@ await _crdtRepository.AddSnapshots([ internal async Task> ComputeSnapshotsToPersist(SortedSet commits) { await ApplyCommitChanges(commits); - return [.._rootSnapshots.Values, .._newIntermediateSnapshots, .._pendingSnapshots.Values]; + return [.. _rootSnapshots.Values, .. _newIntermediateSnapshots, .. _pendingSnapshots.Values]; } private async ValueTask ApplyCommitChanges(SortedSet commits)