From 138e2b3bb4cc3f5aa32da0f2f6f3ab2a87602691 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sat, 24 May 2025 00:14:35 +0200 Subject: [PATCH 1/4] Jsonb --- .../MySqlBulkInsertProvider.cs | 5 +- .../PostgreSqlBulkInsertProvider.cs | 44 +++++++++++-- .../SqlServerBulkInsertProvider.cs | 5 +- .../SqliteBulkInsertProvider.cs | 61 ++++------------- .../BulkInsertOptionsExtension.cs | 4 +- .../Metadata/ColumnMetadata.cs | 2 + .../Metadata/MetadataProviderExtension.cs | 4 +- .../TestDbContext.cs | 9 +-- .../DbContext/Extensions.cs | 23 +++++++ .../DbContext/TestDbContext.cs | 58 ++++++++++++++++ .../TestEntityWithGeneratedGuidId.cs | 18 +++++ .../DbContext/TestEntityWithGuidId.cs | 4 +- .../DbContext/TestEntityWithJson.cs | 16 +++++ .../Tests/Basic/BasicTestsBase.cs | 66 ++++++++++++------- .../Tests/Basic/BasicTestsMySql.cs | 5 +- .../Tests/Basic/BasicTestsPostgreSql.cs | 5 +- .../Tests/Basic/BasicTestsSqlServer.cs | 5 +- .../Tests/Basic/BasicTestsSqlite.cs | 5 +- 18 files changed, 219 insertions(+), 120 deletions(-) create mode 100644 tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/Extensions.cs create mode 100644 tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGeneratedGuidId.cs create mode 100644 tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithJson.cs diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs index ab08f92..4587997 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs @@ -9,11 +9,8 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.MySql; -internal class MySqlBulkInsertProvider : BulkInsertProviderBase +internal class MySqlBulkInsertProvider(ILogger? logger = null) : BulkInsertProviderBase(logger) { - public MySqlBulkInsertProvider(ILogger? logger = null) : base(logger) - { - } //language=sql /// diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs index 1258845..8f3654b 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs @@ -6,6 +6,9 @@ using Microsoft.Extensions.Logging; using Npgsql; +using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping; + +using NpgsqlTypes; using PhenX.EntityFrameworkCore.BulkInsert.Metadata; using PhenX.EntityFrameworkCore.BulkInsert.Options; @@ -13,11 +16,8 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.PostgreSql; [UsedImplicitly] -internal class PostgreSqlBulkInsertProvider : BulkInsertProviderBase +internal class PostgreSqlBulkInsertProvider(ILogger? logger = null) : BulkInsertProviderBase(logger) { - public PostgreSqlBulkInsertProvider(ILogger? logger = null) : base(logger) - { - } //language=sql /// @@ -51,6 +51,9 @@ protected override async Task BulkInsert( ? connection.BeginBinaryImport(command) : await connection.BeginBinaryImportAsync(command, ctk); + // The type mapping can be null for obvious types like string. + var columnTypes = columns.Select(GetPostgreSqlType).ToArray(); + foreach (var entity in entities) { if (sync) @@ -63,19 +66,40 @@ protected override async Task BulkInsert( await writer.StartRowAsync(ctk); } + var columnIndex = 0; foreach (var column in columns) { var value = column.GetValue(entity); + // Get the actual type, so that the writer can do the conversation to the target type automatically. + var type = columnTypes[columnIndex]; + if (sync) { - // ReSharper disable once MethodHasAsyncOverloadWithCancellation - writer.Write(value); + if (type != null) + { + // ReSharper disable once MethodHasAsyncOverloadWithCancellation + writer.Write(value, type.Value); + } + else + { + // ReSharper disable once MethodHasAsyncOverloadWithCancellation + writer.Write(value); + } } else { - await writer.WriteAsync(value, ctk); + if (type != null) + { + await writer.WriteAsync(value, type.Value, ctk); + } + else + { + await writer.WriteAsync(value, ctk); + } } + + columnIndex++; } } @@ -91,6 +115,12 @@ protected override async Task BulkInsert( await writer.CompleteAsync(ctk); await writer.DisposeAsync(); } + } + + private static NpgsqlDbType? GetPostgreSqlType(ColumnMetadata column) + { + var mapping = column.Property.GetRelationalTypeMapping() as NpgsqlTypeMapping; + return mapping?.NpgsqlDbType; } } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs index 79fc58f..c4a6b56 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs @@ -11,11 +11,8 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.SqlServer; [UsedImplicitly] -internal class SqlServerBulkInsertProvider : BulkInsertProviderBase +internal class SqlServerBulkInsertProvider(ILogger? logger = null) : BulkInsertProviderBase(logger) { - public SqlServerBulkInsertProvider(ILogger? logger = null) : base(logger) - { - } //language=sql /// diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs index bd82de2..8d90c20 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs @@ -13,11 +13,9 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Sqlite; [UsedImplicitly] -internal class SqliteBulkInsertProvider : BulkInsertProviderBase +internal class SqliteBulkInsertProvider(ILogger? logger = null) : BulkInsertProviderBase(logger) { - public SqliteBulkInsertProvider(ILogger? logger = null) : base(logger) - { - } + private const int MaxParams = 1000; /// protected override string BulkInsertId => "rowid"; @@ -34,48 +32,17 @@ protected override Task AddBulkInsertIdColumn( CancellationToken cancellationToken ) where T : class => Task.CompletedTask; - /// - /// Taken from https://github.com/dotnet/efcore/blob/667c569c49a1ab7e142621395d3f14f2af0508b4/src/Microsoft.Data.Sqlite.Core/SqliteValueBinder.cs#L231 - /// As the method is not exposed in the public API, we need to copy it here. - /// - private static readonly Dictionary SqliteTypeMapping = - new() - { - { typeof(bool), SqliteType.Integer }, - { typeof(byte), SqliteType.Integer }, - { typeof(byte[]), SqliteType.Blob }, - { typeof(char), SqliteType.Text }, - { typeof(DateTime), SqliteType.Text }, - { typeof(DateTimeOffset), SqliteType.Text }, - { typeof(DateOnly), SqliteType.Text }, - { typeof(TimeOnly), SqliteType.Text }, - { typeof(DBNull), SqliteType.Text }, - { typeof(decimal), SqliteType.Text }, - { typeof(double), SqliteType.Real }, - { typeof(float), SqliteType.Real }, - { typeof(Guid), SqliteType.Text }, - { typeof(int), SqliteType.Integer }, - { typeof(long), SqliteType.Integer }, - { typeof(sbyte), SqliteType.Integer }, - { typeof(short), SqliteType.Integer }, - { typeof(string), SqliteType.Text }, - { typeof(TimeSpan), SqliteType.Text }, - { typeof(uint), SqliteType.Integer }, - { typeof(ulong), SqliteType.Integer }, - { typeof(ushort), SqliteType.Integer } - }; - - private static SqliteType GetSqliteType(Type clrType) + private static SqliteType GetSqliteType(ColumnMetadata column) { - var type = Nullable.GetUnderlyingType(clrType) ?? clrType; - type = type.IsEnum ? Enum.GetUnderlyingType(type) : type; - - if (SqliteTypeMapping.TryGetValue(type, out var sqliteType)) + var storeType = column.Property.GetRelationalTypeMapping().StoreType; + return storeType switch { - return sqliteType; - } - - throw new InvalidOperationException($"Unknown Sqlite type for {clrType}"); + "INTEGER" => SqliteType.Integer, + "FLOAT" => SqliteType.Real, + "TEXT" => SqliteType.Text, + "BLOB" => SqliteType.Blob, + _ => throw new NotSupportedException($"Invalid store type '{storeType}' for property '{column.PropertyName}'"), + }; } private static DbCommand GetInsertCommand( @@ -141,15 +108,13 @@ protected override async Task BulkInsert( CancellationToken ctk ) where T : class { - const int maxParams = 1000; - var batchSize = options.BatchSize ?? 5; - batchSize = Math.Min(batchSize, maxParams / columns.Count); + var batchSize = Math.Min(options.BatchSize ?? 5, MaxParams / columns.Count); // The StringBuilder can be resuse between the batches. var sb = new StringBuilder(); var columnList = tableInfo.GetColumns(options.CopyGeneratedColumns); - var columnTypes = columnList.Select(c => GetSqliteType(c.ProviderClrType ?? c.ClrType)).ToArray(); + var columnTypes = columnList.Select(GetSqliteType).ToArray(); await using var insertCommand = GetInsertCommand( diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertOptionsExtension.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertOptionsExtension.cs index ebd131b..350ace4 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertOptionsExtension.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertOptionsExtension.cs @@ -20,10 +20,8 @@ public void Validate(IDbContextOptions options) { } - private class BulkInsertOptionsExtensionInfo : DbContextOptionsExtensionInfo + private class BulkInsertOptionsExtensionInfo(IDbContextOptionsExtension extension) : DbContextOptionsExtensionInfo(extension) { - public BulkInsertOptionsExtensionInfo(IDbContextOptionsExtension extension) - : base(extension) { } /// public override int GetServiceProviderHashCode() => 0; diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/ColumnMetadata.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/ColumnMetadata.cs index b640e66..6e06524 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/ColumnMetadata.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/ColumnMetadata.cs @@ -9,6 +9,8 @@ internal sealed class ColumnMetadata(IProperty property, SqlDialectBuilder dial { private readonly PropertyAccessor.Getter _getter = BuildGetter(property); + public IProperty Property { get; } = property; + public string PropertyName { get; } = property.Name; public string ColumnName { get; } = property.GetColumnName(); diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/MetadataProviderExtension.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/MetadataProviderExtension.cs index 3fe2a20..9774f1d 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/MetadataProviderExtension.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/MetadataProviderExtension.cs @@ -17,10 +17,8 @@ public void Validate(IDbContextOptions options) { } - private class MetadataProviderExtensionInfo : DbContextOptionsExtensionInfo + private class MetadataProviderExtensionInfo(IDbContextOptionsExtension extension) : DbContextOptionsExtensionInfo(extension) { - public MetadataProviderExtensionInfo(IDbContextOptionsExtension extension) - : base(extension) { } /// public override int GetServiceProviderHashCode() => 0; diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/TestDbContext.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/TestDbContext.cs index 8b31a5d..0e09a4b 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/TestDbContext.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/TestDbContext.cs @@ -2,17 +2,12 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Benchmark; -public class TestDbContext : DbContext +public class TestDbContext(Action configure) : DbContext { - public Action Configure { get; } + public Action Configure { get; } = configure; public DbSet TestEntities { get; set; } = null!; - public TestDbContext(Action configure) - { - Configure = configure; - } - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { Configure(optionsBuilder); diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/Extensions.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/Extensions.cs new file mode 100644 index 0000000..5ec25a0 --- /dev/null +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/Extensions.cs @@ -0,0 +1,23 @@ +using System.Text.Json; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; + +public static class Extensions +{ + public static PropertyBuilder AsJsonString(this PropertyBuilder propertyBuilder, string? columnType) + where T : class + { + var converter = new ValueConverter( + v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null), + v => JsonSerializer.Deserialize(v, (JsonSerializerOptions?)null)! + ); + + propertyBuilder.HasConversion(converter).HasColumnType(columnType); + return propertyBuilder; + } + +} diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs index 530d42e..9c67ab8 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs @@ -6,12 +6,14 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; public class TestDbContext : TestDbContextBase { public DbSet TestEntities { get; set; } = null!; + public DbSet TestEntitiesWithJson { get; set; } = null!; public DbSet TestEntitiesWithGuidIds { get; set; } = null!; public DbSet TestEntitiesWithConverters { get; set; } = null!; protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); + modelBuilder.Entity(builder => { builder.Property(e => e.CreatedAt) @@ -25,3 +27,59 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) }); } } + +public class TestDbContextPostgreSql : TestDbContext +{ + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.Entity(b => + { + b.Property(x => x.Json).AsJsonString("jsonb"); + }); + } +} + +public class TestDbContextMySql : TestDbContext +{ + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.Entity(b => + { + b.Property(x => x.Json).AsJsonString("json"); + }); + } +} + +public class TestDbContextSqlServer : TestDbContext +{ + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.Entity(b => + { + b.Property(x => x.Json).AsJsonString(null); + }); + } +} + + +public class TestDbContextSqlite : TestDbContext +{ + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.Entity(b => + { + b.Property(x => x.Json).AsJsonString(null); + }); + } +} + + + diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGeneratedGuidId.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGeneratedGuidId.cs new file mode 100644 index 0000000..4a8b23a --- /dev/null +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGeneratedGuidId.cs @@ -0,0 +1,18 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; + +[Table("test_entity_generated_guids")] +public class TestEntityWithGeneratedGuidId +{ + [Key] + public Guid Id { get; set; } + + [Column("name")] + [MaxLength(100)] + public string Name { get; set; } = string.Empty; + + [Column("test_run")] + public Guid TestRun { get; set; } +} diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs index 0bcd218..e15a28e 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs @@ -1,14 +1,12 @@ using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; -using Microsoft.EntityFrameworkCore; - namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; -[PrimaryKey(nameof(Id))] [Table("test_entity_guids")] public class TestEntityWithGuidId { + [Key] public Guid Id { get; set; } [Column("name")] diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithJson.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithJson.cs new file mode 100644 index 0000000..d9915fd --- /dev/null +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithJson.cs @@ -0,0 +1,16 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; + +[Table("test_entity_json")] +public class TestEntityWithJson +{ + [Key] + public int Id { get; set; } + + public List Json { get; set; } = []; + + [Column("test_run")] + public Guid TestRun { get; set; } +} diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs index eaa1976..9721916 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs @@ -7,16 +7,12 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.Tests.Basic; -public abstract class BasicTestsBase : IClassFixture, IAsyncLifetime - where TFixture : TestDbContainer +public abstract class BasicTestsBase(TestDbContainer dbContainer) : IClassFixture, IAsyncLifetime + where TDbContext : TestDbContext, new() + where TFixture : TestDbContainer { private readonly Guid _run = Guid.NewGuid(); - private TestDbContext _context = null!; - - protected BasicTestsBase(TestDbContainer dbContainer) - { - DbContainer = dbContainer; - } + private TDbContext _context = null!; public async Task InitializeAsync() { @@ -29,10 +25,10 @@ public Task DisposeAsync() return Task.CompletedTask; } - protected TestDbContainer DbContainer { get; } + protected TestDbContainer DbContainer { get; } = dbContainer; [Fact] - public async Task InsertsEntitiesSuccessfully() + public async Task InsertsEntities() { // Arrange var entities = new List @@ -52,7 +48,7 @@ public async Task InsertsEntitiesSuccessfully() } [Fact] - public void InsertsEntitiesSuccessfully_Sync() + public void InsertsEntities_Sync() { // Arrange var entities = new List @@ -72,7 +68,7 @@ public void InsertsEntitiesSuccessfully_Sync() } [SkippableFact] - public async Task InsertsEntitiesAndReturn() + public async Task InsertsEntities_AndReturn() { Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); @@ -93,7 +89,27 @@ public async Task InsertsEntitiesAndReturn() } [SkippableFact] - public async Task InsertsEntitiesAndReturnAsyncEnumerable() + public async Task InsertsEntities_WithJson() + { + // Arrange + var entities = new List + { + new TestEntityWithJson { TestRun = _run, Json = [1] }, + new TestEntityWithJson { TestRun = _run, Json = [2] } + }; + + // Act + await _context.ExecuteBulkInsertAsync(entities); + + // Assert + var insertedEntities = _context.TestEntitiesWithJson.Where(x => x.TestRun == _run).ToList(); + Assert.Equal(2, insertedEntities.Count); + Assert.Contains(insertedEntities, e => e.Json[0] == 1); + Assert.Contains(insertedEntities, e => e.Json[0] == 2); + } + + [SkippableFact] + public async Task InsertsEntities_AndReturn_AsyncEnumerable() { Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); @@ -119,7 +135,7 @@ public async Task InsertsEntitiesAndReturnAsyncEnumerable() } [SkippableFact] - public void InsertsEntitiesAndReturn_Sync() + public void InsertsEntities_AndReturn_Sync() { Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); @@ -238,7 +254,7 @@ await _context.ExecuteBulkInsertAsync(insertedEntities0, } [Fact] - public async Task InsertsEntitiesMoveRowsSuccessfully() + public async Task InsertsEntities_MoveRows() { // Arrange var entities = new List @@ -261,7 +277,7 @@ await _context.ExecuteBulkInsertAsync(entities, o => } [SkippableFact] - public async Task InsertsEntitiesWithConflict_SingleColumn() + public async Task InsertsEntities_WithConflict_SingleColumn() { Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); @@ -300,7 +316,7 @@ await _context.ExecuteBulkInsertAsync(entities, o => } [SkippableFact] - public async Task InsertsEntitiesWithConflict_DoNothing() + public async Task InsertsEntities_WithConflict_DoNothing() { Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); @@ -330,7 +346,7 @@ await _context.ExecuteBulkInsertAsync(entities, o => } [SkippableFact] - public async Task InsertsEntitiesWithConflict_Condition() + public async Task InsertsEntities_WithConflict_Condition() { Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); @@ -361,7 +377,7 @@ await _context.ExecuteBulkInsertAsync(entities, o => } [SkippableFact] - public async Task InsertsEntitiesWithConflict_MultipleColumns() + public async Task InsertsEntities_WithConflict_MultipleColumns() { Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); @@ -397,7 +413,7 @@ await _context.ExecuteBulkInsertAsync(entities, o => } [Fact] - public async Task DoesNothingWhenEntitiesAreEmpty() + public async Task InsertsEntities_DoesNothing_WhenEntitiesAreEmpty() { // Arrange var entities = new List(); @@ -439,7 +455,7 @@ await _context.ExecuteBulkInsertAsync(entities, o => } [Fact] - public async Task InsertAndRead_EntityWithValueConverters() + public async Task InsertEntities_AndReturn_WithEntityWithValueConverters() { // Arrange var now = DateTime.UtcNow; @@ -460,7 +476,7 @@ public async Task InsertAndRead_EntityWithValueConverters() } [Fact] - public async Task BulkInsert_WithOpenTransaction_CommitsSuccessfully() + public async Task InsertEntities_WithOpenTransaction_CommitsSuccessfully() { // Arrange var entities = new List @@ -482,7 +498,7 @@ public async Task BulkInsert_WithOpenTransaction_CommitsSuccessfully() } [Fact] - public void BulkInsert_WithOpenTransaction_CommitsSuccessfully_Sync() + public void InsertEntities_WithOpenTransaction_CommitsSuccessfully_Sync() { // Arrange var entities = new List @@ -504,7 +520,7 @@ public void BulkInsert_WithOpenTransaction_CommitsSuccessfully_Sync() } [Fact] - public async Task BulkInsert_WithOpenTransaction_RollsBackOnFailure() + public async Task InsertEntities_WithOpenTransaction_RollsBackOnFailure() { // Arrange var entities = new List @@ -527,7 +543,7 @@ public async Task BulkInsert_WithOpenTransaction_RollsBackOnFailure() } [Fact] - public void BulkInsert_WithOpenTransaction_RollsBackOnFailure_Sync() + public void InsertEntities_WithOpenTransaction_RollsBackOnFailure_Sync() { // Arrange var entities = new List diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsMySql.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsMySql.cs index 9181409..34716be 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsMySql.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsMySql.cs @@ -6,9 +6,6 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.Tests.Basic; [Trait("Category", "MySql")] -public class BasicTestsMySql : BasicTestsBase> +public class BasicTestsMySql(TestDbContainerMySql dbContainer) : BasicTestsBase, TestDbContextMySql>(dbContainer) { - public BasicTestsMySql(TestDbContainerMySql dbContainer) : base(dbContainer) - { - } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsPostgreSql.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsPostgreSql.cs index f1a9cf4..b0ce8d3 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsPostgreSql.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsPostgreSql.cs @@ -6,9 +6,6 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.Tests.Basic; [Trait("Category", "PostgreSql")] -public class BasicTestsPostgreSql : BasicTestsBase> +public class BasicTestsPostgreSql(TestDbContainerPostgreSql dbContainer) : BasicTestsBase, TestDbContextPostgreSql>(dbContainer) { - public BasicTestsPostgreSql(TestDbContainerPostgreSql dbContainer) : base(dbContainer) - { - } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlServer.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlServer.cs index 0d658e4..f07ac40 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlServer.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlServer.cs @@ -6,9 +6,6 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.Tests.Basic; [Trait("Category", "SqlServer")] -public class BasicTestsSqlServer : BasicTestsBase> +public class BasicTestsSqlServer(TestDbContainerSqlServer dbContainer) : BasicTestsBase, TestDbContextSqlServer>(dbContainer) { - public BasicTestsSqlServer(TestDbContainerSqlServer dbContainer) : base(dbContainer) - { - } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlite.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlite.cs index 0aabe0a..2c728fd 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlite.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsSqlite.cs @@ -6,10 +6,7 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.Tests.Basic; [Trait("Category", "Sqlite")] -public class BasicTestsSqlite : BasicTestsBase> +public class BasicTestsSqlite(TestDbContainerSqlite dbContainer) : BasicTestsBase, TestDbContextSqlite>(dbContainer) { - public BasicTestsSqlite(TestDbContainerSqlite dbContainer) : base(dbContainer) - { - } } From 9a10ed239ecb88f522bcd25af373228484579f26 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sat, 24 May 2025 00:20:42 +0200 Subject: [PATCH 2/4] FIx conversion. --- .../MySqlBulkInsertProvider.cs | 1 - .../PostgreSqlBulkInsertProvider.cs | 1 - .../SqliteBulkInsertProvider.cs | 27 ++++++++++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs index 4587997..4d962dc 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs @@ -11,7 +11,6 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.MySql; internal class MySqlBulkInsertProvider(ILogger? logger = null) : BulkInsertProviderBase(logger) { - //language=sql /// protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD {BulkInsertId} INT AUTO_INCREMENT PRIMARY KEY;"; diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs index 8f3654b..42ef251 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs @@ -18,7 +18,6 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.PostgreSql; [UsedImplicitly] internal class PostgreSqlBulkInsertProvider(ILogger? logger = null) : BulkInsertProviderBase(logger) { - //language=sql /// protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD COLUMN {BulkInsertId} SERIAL PRIMARY KEY;"; diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs index 8d90c20..af46f54 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs @@ -35,14 +35,27 @@ CancellationToken cancellationToken private static SqliteType GetSqliteType(ColumnMetadata column) { var storeType = column.Property.GetRelationalTypeMapping().StoreType; - return storeType switch + + if (string.Equals(storeType, "INTEGER", StringComparison.OrdinalIgnoreCase)) + { + return SqliteType.Integer; + } + else if (string.Equals(storeType, "FLOAT", StringComparison.OrdinalIgnoreCase)) { - "INTEGER" => SqliteType.Integer, - "FLOAT" => SqliteType.Real, - "TEXT" => SqliteType.Text, - "BLOB" => SqliteType.Blob, - _ => throw new NotSupportedException($"Invalid store type '{storeType}' for property '{column.PropertyName}'"), - }; + return SqliteType.Real; + } + else if (string.Equals(storeType, "TEXT", StringComparison.OrdinalIgnoreCase)) + { + return SqliteType.Text; + } + else if (string.Equals(storeType, "BLOB", StringComparison.OrdinalIgnoreCase)) + { + return SqliteType.Blob; + } + else + { + throw new NotSupportedException($"Invalid store type '{storeType}' for property '{column.PropertyName}'"); + } } private static DbCommand GetInsertCommand( From 805444b612fd7b87c983d1e79e22f0e6613ee3d1 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sat, 24 May 2025 12:09:04 +0200 Subject: [PATCH 3/4] No result / out mix. --- .../Extensions/PublicExtensions.cs | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.cs index 4dcc850..cdcdf6e 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.cs @@ -11,22 +11,22 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Extensions; /// public static partial class PublicExtensions { - private static async Task> ExecuteBulkInsertReturnEntitiesCoreAsync( - this DbSet dbSet, + private static async Task> ExecuteBulkInsertReturnEntitiesCoreAsync( + this DbSet dbSet, bool sync, - IEnumerable entities, + IEnumerable entities, Action configure, - OnConflictOptions? onConflict, + OnConflictOptions? onConflict, CancellationToken ctk ) - where T : class + where TEntity : class where TOptions : BulkInsertOptions { - var provider = InitProvider(dbSet, configure, out var context, out var options); + var (provider, context, options) = InitProvider(dbSet, configure); - var enumerable = provider.BulkInsertReturnEntities(sync, context, dbSet.GetDbContext().GetTableInfo(), entities, options, onConflict, ctk); + var enumerable = provider.BulkInsertReturnEntities(sync, context, dbSet.GetDbContext().GetTableInfo(), entities, options, onConflict, ctk); - var result = new List(); + var result = new List(); await foreach (var item in enumerable.WithCancellation(ctk)) { result.Add(item); @@ -41,27 +41,23 @@ private static DbContext GetDbContext(this DbSet dbSet) where T : class return (infrastructure.Instance.GetService(typeof(ICurrentDbContext)) as ICurrentDbContext)!.Context; } - private static IBulkInsertProvider InitProvider( + private static (IBulkInsertProvider, DbContext, TOptions) InitProvider( DbSet dbSet, - Action? configure, - out DbContext context, - out TOptions options + Action? configure ) where T : class where TOptions : BulkInsertOptions { - context = dbSet.GetDbContext(); + var context = dbSet.GetDbContext(); var provider = context.GetService(); + var options = provider.InternalCreateDefaultOptions(); - var defaultOptions = provider.InternalCreateDefaultOptions(); - - if (defaultOptions is not TOptions castedOptions) + if (options is not TOptions castedOptions) { - throw new InvalidOperationException($"Options type mismatch. Expected {defaultOptions.GetType().Name}, but got {typeof(TOptions).Name}."); + throw new InvalidOperationException($"Options type mismatch. Expected {options.GetType().Name}, but got {typeof(TOptions).Name}."); } - options = castedOptions; - configure?.Invoke(options); + configure?.Invoke(castedOptions); - return provider; + return (provider, context, castedOptions); } } From a5c761ea4e4b9d8083c4978f9dd50de380b6b609 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sat, 24 May 2025 13:30:10 +0200 Subject: [PATCH 4/4] Fix build --- .../Extensions/PublicExtensions.DbSet.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.DbSet.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.DbSet.cs index c7cd183..5aa9aef 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.DbSet.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/PublicExtensions.DbSet.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using PhenX.EntityFrameworkCore.BulkInsert.Options; @@ -107,7 +107,7 @@ public static IAsyncEnumerable ExecuteBulkInsertReturnEnumerableAsync(), entities, options, onConflict, ctk); @@ -155,7 +155,7 @@ public static async Task ExecuteBulkInsertAsync( where T : class where TOptions : BulkInsertOptions { - var provider = InitProvider(dbSet, configure, out var context, out var options); + var (provider, context, options) = InitProvider(dbSet, configure); await provider.BulkInsert(false, context, dbSet.GetDbContext().GetTableInfo(), entities, options, onConflict, ctk); @@ -202,7 +202,7 @@ public static void ExecuteBulkInsert( where T : class where TOptions : BulkInsertOptions { - var provider = InitProvider(dbSet, configure, out var context, out var options); + var (provider, context, options) = InitProvider(dbSet, configure); provider.BulkInsert(true, context, dbSet.GetDbContext().GetTableInfo(), entities, options, onConflict) .GetAwaiter().GetResult();