diff --git a/.gitignore b/.gitignore index ca85b27..0073c2e 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,6 @@ fabric.properties # Nuget assets /nupkgs + +# Visual Studio Files +.vs diff --git a/PhenX.EntityFrameworkCore.BulkInsert.sln b/PhenX.EntityFrameworkCore.BulkInsert.sln index ae73ad3..c42b00c 100644 --- a/PhenX.EntityFrameworkCore.BulkInsert.sln +++ b/PhenX.EntityFrameworkCore.BulkInsert.sln @@ -1,5 +1,8 @@  Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35707.178 d17.12 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PhenX.EntityFrameworkCore.BulkInsert", "src\PhenX.EntityFrameworkCore.BulkInsert\PhenX.EntityFrameworkCore.BulkInsert.csproj", "{56CA0AE2-6EAB-4394-9E06-132558551251}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PhenX.EntityFrameworkCore.BulkInsert.PostgreSql", "src\PhenX.EntityFrameworkCore.BulkInsert.PostgreSql\PhenX.EntityFrameworkCore.BulkInsert.PostgreSql.csproj", "{F37308A8-1C3C-44D2-9440-670DF76A8C31}" @@ -30,6 +33,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "root", "root", "{45366E91-4 README.md = README.md EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PhenX.EntityFrameworkCore.BulkInsert.MySql", "src\PhenX.EntityFrameworkCore.BulkInsert.MySql\PhenX.EntityFrameworkCore.BulkInsert.MySql.csproj", "{17649766-EA68-4333-8DA8-47B014A8B2CC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -60,6 +65,13 @@ Global {450E859C-411F-4D67-A0B4-4E02C3D30E14}.Debug|Any CPU.Build.0 = Debug|Any CPU {450E859C-411F-4D67-A0B4-4E02C3D30E14}.Release|Any CPU.ActiveCfg = Release|Any CPU {450E859C-411F-4D67-A0B4-4E02C3D30E14}.Release|Any CPU.Build.0 = Release|Any CPU + {17649766-EA68-4333-8DA8-47B014A8B2CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {17649766-EA68-4333-8DA8-47B014A8B2CC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {17649766-EA68-4333-8DA8-47B014A8B2CC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {17649766-EA68-4333-8DA8-47B014A8B2CC}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {56CA0AE2-6EAB-4394-9E06-132558551251} = {CBEBA2A8-79E0-412E-93C1-C88F4473D78B} @@ -68,5 +80,6 @@ Global {EDCCED5F-D456-45E2-81A6-1077977F042B} = {F8A83782-311C-454D-8B97-B3FB86478BF4} {E4EB1C53-575C-45F8-924A-93DC42E8ACCA} = {F8A83782-311C-454D-8B97-B3FB86478BF4} {450E859C-411F-4D67-A0B4-4E02C3D30E14} = {CBEBA2A8-79E0-412E-93C1-C88F4473D78B} + {17649766-EA68-4333-8DA8-47B014A8B2CC} = {CBEBA2A8-79E0-412E-93C1-C88F4473D78B} EndGlobalSection EndGlobal diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs new file mode 100644 index 0000000..d6eae56 --- /dev/null +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs @@ -0,0 +1,80 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Logging; + +using MySqlConnector; + +using PhenX.EntityFrameworkCore.BulkInsert.Options; + +namespace PhenX.EntityFrameworkCore.BulkInsert.MySql; + +internal class MySqlBulkInsertProvider : BulkInsertProviderBase +{ + public MySqlBulkInsertProvider(ILogger? logger = null) : base(logger) + { + } + + //language=sql + /// + protected override string CreateTableCopySql => "CREATE TEMPORARY TABLE {0} SELECT * FROM {1} WHERE 1 = 0;"; + + //language=sql + /// + protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD {BulkInsertId} INT AUTO_INCREMENT PRIMARY KEY;"; + + /// + protected override string GetTempTableName(string tableName) => $"#_temp_bulk_insert_{tableName}"; + + /// + public override Task> BulkInsertReturnEntities( + bool sync, + DbContext context, + IEnumerable entities, + BulkInsertOptions options, + OnConflictOptions? onConflict = null, + CancellationToken ctk = default) + { + throw new NotSupportedException("Provider does not support returning entities."); + } + + /// + protected override async Task BulkInsert( + bool sync, + DbContext context, + IEnumerable entities, + string tableName, + PropertyAccessor[] properties, + BulkInsertOptions options, + CancellationToken ctk + ) + { + var connection = (MySqlConnection)context.Database.GetDbConnection(); + var sqlTransaction = context.Database.CurrentTransaction!.GetDbTransaction() + ?? throw new InvalidOperationException("No open transaction found."); + if (sqlTransaction is not MySqlTransaction mySqlTransaction) + { + throw new InvalidOperationException($"Invalid transaction foud, got {sqlTransaction.GetType()}."); + } + + var bulkCopy = new MySqlBulkCopy(connection, mySqlTransaction); + bulkCopy.DestinationTableName = tableName; + bulkCopy.BulkCopyTimeout = options.GetCopyTimeoutInSeconds(); + + var sourceOrdinal = 0; + foreach (var prop in properties) + { + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(sourceOrdinal, prop.ColumnName)); + sourceOrdinal++; + } + + if (sync) + { + // ReSharper disable once MethodHasAsyncOverloadWithCancellation + bulkCopy.WriteToServer(new EnumerableDataReader(entities, properties)); + } + else + { + await bulkCopy.WriteToServerAsync(new EnumerableDataReader(entities, properties), ctk); + } + } +} diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDbContextOptionsExtensions.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDbContextOptionsExtensions.cs new file mode 100644 index 0000000..9a51ea9 --- /dev/null +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDbContextOptionsExtensions.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace PhenX.EntityFrameworkCore.BulkInsert.MySql; + +/// +/// DbContext options extension for MySql. +/// +public static class MySqlDbContextOptionsExtensions +{ + /// + /// Configures the DbContext to use the MySql bulk insert provider. + /// + public static DbContextOptionsBuilder UseBulkInsertMySql(this DbContextOptionsBuilder optionsBuilder) + { + var extension = optionsBuilder.Options.FindExtension>() ?? new BulkInsertOptionsExtension(); + + ((IDbContextOptionsBuilderInfrastructure)optionsBuilder).AddOrUpdateExtension(extension); + + return optionsBuilder; + } +} diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDialectBuilder.cs new file mode 100644 index 0000000..b4c5fac --- /dev/null +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDialectBuilder.cs @@ -0,0 +1,57 @@ +using System.Text; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; + +using PhenX.EntityFrameworkCore.BulkInsert.Dialect; +using PhenX.EntityFrameworkCore.BulkInsert.Options; + +namespace PhenX.EntityFrameworkCore.BulkInsert.MySql; + +internal class MySqlServerDialectBuilder : SqlDialectBuilder +{ + protected override string OpenDelimiter => "`"; + + protected override string CloseDelimiter => "`"; + + protected override bool SupportsMoveRows => false; + + protected override void AppendConflictCondition(StringBuilder sql, OnConflictOptions onConflictTyped) + { + throw new NotSupportedException("Conflict conditions are not supported in MYSQL"); + } + + protected override void AppendOnConflictUpdate(StringBuilder sql, IEnumerable updates) + { + sql.AppendLine("UPDATE"); + + var i = 0; + foreach (var update in updates) + { + if (i > 0) + { + sql.Append(", "); + } + + sql.Append(update); + i++; + } + } + + protected override void AppendOnConflictStatement(StringBuilder sql) + { + sql.Append("ON DUPLICATE KEY"); + } + + protected override void AppendDoNothing(StringBuilder sql, IProperty[] insertedProperties) + { + var columnName = insertedProperties[0].GetColumnName(); + + sql.Append($"UPDATE {Quote(columnName)} = {GetExcludedColumnName(columnName)}"); + } + + protected override string GetExcludedColumnName(string columnName) + { + return $"VALUES({Quote(columnName)})"; + } +} diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/PhenX.EntityFrameworkCore.BulkInsert.MySql.csproj b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/PhenX.EntityFrameworkCore.BulkInsert.MySql.csproj new file mode 100644 index 0000000..7a0a0e6 --- /dev/null +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/PhenX.EntityFrameworkCore.BulkInsert.MySql.csproj @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs index b4d1e18..51425ff 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs @@ -48,7 +48,7 @@ CancellationToken ctk foreach (var prop in properties) { - bulkCopy.ColumnMappings.Add(prop.Name, SqlDialect.Quote(prop.ColumnName)); + bulkCopy.ColumnMappings.Add(prop.Name, prop.ColumnName); } if (sync) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs index 1ce6a96..4d9f669 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs @@ -1,4 +1,4 @@ -using System.Linq.Expressions; +using System.Linq.Expressions; using System.Text; using Microsoft.EntityFrameworkCore; @@ -31,6 +31,11 @@ public override string BuildMoveDataSql(DbContext context, string source, var q = new StringBuilder(); + if (options.CopyGeneratedColumns) + { + q.AppendLine($"SET IDENTITY_INSERT {target} ON;"); + } + // Merge handling if (onConflict is OnConflictOptions onConflictTyped && onConflictTyped.Match != null) { @@ -39,7 +44,7 @@ public override string BuildMoveDataSql(DbContext context, string source, matchColumns.Select(col => $"TARGET.{col} = SOURCE.{col}")); var updateSet = onConflictTyped.Update != null - ? string.Join(", ", GetUpdates(context, onConflictTyped.Update)) + ? string.Join(", ", GetUpdates(context, insertedProperties, onConflictTyped.Update)) : null; q.AppendLine($"MERGE INTO {target} AS TARGET"); @@ -79,11 +84,16 @@ public override string BuildMoveDataSql(DbContext context, string source, q.AppendLine(";"); + if (options.CopyGeneratedColumns) + { + q.AppendLine($"SET IDENTITY_INSERT {target} OFF;"); + } + return q.ToString(); } - protected override string GetExcludedColumnName(DbContext context, MemberExpression member) + protected override string GetExcludedColumnName(string columnName) { - return $"SOURCE.{GetColumnName(context, member.Member.Name)}"; + return $"SOURCE.{columnName}"; } } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs index 2e3c0f7..0eea33f 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs @@ -82,9 +82,10 @@ private static SqliteType GetSqliteType(Type clrType) } private DbCommand GetInsertCommand(DbContext context, Type entityType, string tableName, + BulkInsertOptions options, int batchSize) { - var columns = context.GetProperties(entityType, false); + var columns = context.GetProperties(entityType, options.CopyGeneratedColumns); var cmd = context.Database.GetDbConnection().CreateCommand(); var sqliteColumns = columns @@ -136,7 +137,7 @@ CancellationToken ctk var batchSize = options.BatchSize ?? 5; batchSize = Math.Min(batchSize, maxParams / properties.Length); - await using var insertCommand = GetInsertCommand(context, typeof(T), tableName, batchSize); + await using var insertCommand = GetInsertCommand(context, typeof(T), tableName, options, batchSize); foreach (var chunk in entities.Chunk(batchSize)) { @@ -149,7 +150,7 @@ CancellationToken ctk // Last chunk else { - var partialInsertCommand = GetInsertCommand(context, typeof(T), tableName, chunk.Length); + var partialInsertCommand = GetInsertCommand(context, typeof(T), tableName, options, chunk.Length); FillValues(chunk, partialInsertCommand.Parameters, properties); await ExecuteCommand(sync, partialInsertCommand, ctk); diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs index af081b6..119e004 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs @@ -31,13 +31,14 @@ protected BulkInsertProviderBase(ILogger>? logg protected async Task CreateTableCopyAsync( bool sync, DbContext context, + BulkInsertOptions options, CancellationToken cancellationToken = default) where T : class { var tableInfo = GetTableInfo(context, typeof(T)); var tableName = QuoteTableName(tableInfo.SchemaName, tableInfo.TableName); var tempTableName = QuoteTableName(null, GetTempTableName(tableInfo.TableName)); - var keptColumns = string.Join(", ", GetQuotedColumns(context, typeof(T), false)); + var keptColumns = string.Join(", ", GetQuotedColumns(context, typeof(T), options.CopyGeneratedColumns)); var query = string.Format(CreateTableCopySql, tempTableName, tableName, keptColumns); await ExecuteAsync(sync, context, query, cancellationToken); @@ -108,13 +109,21 @@ private async Task> CopyFromTempTableWithoutKeysAsync( { var (schemaName, tableName, _) = GetTableInfo(context, typeof(T)); var quotedTableName = QuoteTableName(schemaName, tableName); - - var movedProperties = context.GetProperties(typeof(T), false); + var movedProperties = context.GetProperties(typeof(T), options.CopyGeneratedColumns); var returnedProperties = returnData ? context.GetProperties(typeof(T)) : []; var query = SqlDialect.BuildMoveDataSql(context, tempTableName, quotedTableName, movedProperties, returnedProperties, options, onConflict); if (returnData) + { + return await QueryAsync(sync, context, query, cancellationToken); + } + + // If not returning data, just execute the command + await ExecuteAsync(sync, context, query, cancellationToken); + return []; + + static async Task> QueryAsync(bool sync, DbContext context, string query, CancellationToken cancellationToken) { // Use EF to execute the query and return the results IQueryable queryable = context @@ -128,13 +137,9 @@ private async Task> CopyFromTempTableWithoutKeysAsync( return await queryable.ToListAsync(cancellationToken: cancellationToken); } - - // If not returning data, just execute the command - await ExecuteAsync(sync, context, query, cancellationToken); - return []; } - public async Task> BulkInsertReturnEntities( + public virtual async Task> BulkInsertReturnEntities( bool sync, DbContext context, IEnumerable entities, @@ -163,10 +168,12 @@ private static async Task Finish(bool sync, DbConnection connection, bool wasClo { // ReSharper disable once MethodHasAsyncOverloadWithCancellation transaction.Commit(); + transaction.Dispose(); } else { await transaction.CommitAsync(ctk); + await transaction.DisposeAsync(); } } @@ -184,7 +191,7 @@ private static async Task Finish(bool sync, DbConnection connection, bool wasClo } } - public async Task BulkInsert( + public virtual async Task BulkInsert( bool sync, DbContext context, IEnumerable entities, @@ -225,11 +232,11 @@ public async Task BulkInsert( var (connection, wasClosed, transaction, wasBegan) = await context.GetConnection(sync, ctk); var tableName = tempTableRequired - ? await CreateTableCopyAsync(sync, context, ctk) + ? await CreateTableCopyAsync(sync, context, options, ctk) : GetQuotedTableName(context, typeof(T)); var properties = context - .GetProperties(typeof(T), false) + .GetProperties(typeof(T), options.CopyGeneratedColumns) .Select(p => new PropertyAccessor(p)) .ToArray(); diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs index ebeed36..9da4b66 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs @@ -1,8 +1,9 @@ -using System.Linq.Expressions; +using System.Linq.Expressions; using System.Text; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Internal; using PhenX.EntityFrameworkCore.BulkInsert.Options; @@ -87,28 +88,32 @@ WHERE TRUE if (onConflict is OnConflictOptions onConflictTyped) { - q.AppendLine("ON CONFLICT"); + AppendOnConflictStatement(q); if (onConflictTyped.Update != null) { if (onConflictTyped.Match != null) { - q.AppendLine($"({string.Join(", ", GetColumns(context, onConflictTyped.Match))})"); + q.Append(' '); + AppendConflictMatch(q, GetColumns(context, onConflictTyped.Match)); } if (onConflictTyped.Update != null) { - q.AppendLine($"DO UPDATE SET {string.Join(", ", GetUpdates(context, onConflictTyped.Update))}"); + q.Append(' '); + AppendOnConflictUpdate(q, GetUpdates(context, insertedProperties, onConflictTyped.Update)); } if (onConflictTyped.Condition != null) { - q.AppendLine($"WHERE {onConflictTyped.Condition}"); + q.Append(' '); + AppendConflictCondition(q, onConflictTyped); } } else { - q.AppendLine("DO NOTHING"); + q.Append(' '); + AppendDoNothing(q, insertedProperties); } } @@ -122,12 +127,63 @@ WHERE TRUE return q.ToString(); } + protected virtual void AppendDoNothing(StringBuilder sql, IProperty[] insertedProperties) + { + sql.AppendLine("DO NOTHING"); + } + + protected virtual void AppendOnConflictUpdate(StringBuilder sql, IEnumerable updates) + { + sql.AppendLine("DO UPDATE SET"); + + var i = 0; + foreach (var update in updates) + { + if (i > 0) + { + sql.Append(", "); + } + + sql.Append(update); + i++; + }; + } + + protected virtual void AppendConflictMatch(StringBuilder sql, IEnumerable columns) + { + sql.AppendLine("("); + + var i = 0; + foreach (var column in columns) + { + if (i > 0) + { + sql.Append(", "); + } + + sql.Append(column); + i++; + } + + sql.AppendLine(")"); + } + + protected virtual void AppendOnConflictStatement(StringBuilder sql) + { + sql.AppendLine("ON CONFLICT"); + } + + protected virtual void AppendConflictCondition(StringBuilder sql, OnConflictOptions onConflictTyped) + { + sql.AppendLine($"WHERE {onConflictTyped.Condition}"); + } + /// /// Get the name of the excluded column for the ON CONFLICT clause. /// - protected virtual string GetExcludedColumnName(DbContext context, MemberExpression member) + protected virtual string GetExcludedColumnName(string columnName) { - return $"EXCLUDED.{GetColumnName(context, member.Member.Name)}"; + return $"EXCLUDED.{columnName}"; } /// @@ -173,7 +229,7 @@ protected string[] GetColumns(DbContext context, Expression> /// var updates = GetUpdates(context, e => e.Prop1); /// /// - protected IEnumerable GetUpdates(DbContext context, Expression> update) + protected IEnumerable GetUpdates(DbContext context, IProperty[] properties, Expression> update) { switch (update.Body) { @@ -198,8 +254,18 @@ protected IEnumerable GetUpdates(DbContext context, Expression(context, memberExpr.Member.Name)} = {ToSqlExpression(context, memberExpr)}"; break; + case ParameterExpression parameterExpr when (parameterExpr.Type == typeof(T)): + foreach (var property in properties) + { + var columName = property.GetColumnName(); + + yield return $"{Quote(columName)} = {GetExcludedColumnName(columName)}"; + } + + break; + default: - throw new NotSupportedException("Unsupported expression type for update"); + throw new NotSupportedException($"Unsupported expression type {update.Body.GetType()} for update"); } } @@ -216,7 +282,7 @@ private string ToSqlExpression(DbContext context, Expression expr) switch (expr) { case MemberExpression m: - return GetExcludedColumnName(context, m); + return GetExcludedColumnName(GetColumnName(context, m.Member.Name)); case BinaryExpression b: var left = ToSqlExpression(context, b.Left); diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/EnumerableDataReader.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/EnumerableDataReader.cs index cba5db5..00f8241 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/EnumerableDataReader.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/EnumerableDataReader.cs @@ -1,4 +1,4 @@ -using System.Data; +using System.Data; namespace PhenX.EntityFrameworkCore.BulkInsert; @@ -26,12 +26,29 @@ public EnumerableDataReader(IEnumerable rows, PropertyAccessor[] properties) public virtual object GetValue(int i) { - if (_enumerator.Current != null) + var current = _enumerator.Current; + if (current == null) { - return _properties[i].GetValue(_enumerator.Current); + return DBNull.Value; } - return DBNull.Value; + return _properties[i].GetValue(current); + } + + public int GetValues(object[] values) + { + var current = _enumerator.Current; + if (current == null) + { + return 0; + } + + for (var i = 0; i < _properties.Length; i++) + { + values[i] = _properties[i].GetValue(current); + } + + return _properties.Length; } public bool Read() => _enumerator.MoveNext(); @@ -58,8 +75,6 @@ public void Dispose() public bool NextResult() => throw new NotImplementedException(); - public int GetValues(object[] values) => throw new NotImplementedException(); - public bool IsDBNull(int i) => GetValue(i) is DBNull; public object this[int i] => throw new NotImplementedException(); diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbContextExtensions.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbContextExtensions.cs index 2000afb..666b120 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbContextExtensions.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbContextExtensions.cs @@ -1,4 +1,4 @@ -using System.Data; +using System.Data; using System.Data.Common; using Microsoft.EntityFrameworkCore; diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Options/BulkInsertOptions.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Options/BulkInsertOptions.cs index e42cd1a..c72f919 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Options/BulkInsertOptions.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Options/BulkInsertOptions.cs @@ -33,6 +33,11 @@ public class BulkInsertOptions /// public int? BatchSize { get; set; } + /// + /// Indicates if also generated columns should be copied. This is useful for upsert operations. + /// + public bool CopyGeneratedColumns { get; set; } + /// /// The timeout to copy records. /// diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/PhenX.EntityFrameworkCore.BulkInsert.csproj b/src/PhenX.EntityFrameworkCore.BulkInsert/PhenX.EntityFrameworkCore.BulkInsert.csproj index be87c15..b59ec6a 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/PhenX.EntityFrameworkCore.BulkInsert.csproj +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/PhenX.EntityFrameworkCore.BulkInsert.csproj @@ -6,6 +6,7 @@ + diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/LibComparator.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/LibComparator.cs index c868c4e..92fa230 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/LibComparator.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/LibComparator.cs @@ -10,6 +10,8 @@ using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; +using MySqlConnector; + using Npgsql; using PhenX.EntityFrameworkCore.BulkInsert.Extensions; @@ -87,6 +89,11 @@ public void RawInsert() // Use BeginBinaryImport for PostgreSQL RawInsertPostgreSql(); } + else if (DbContext.Database.ProviderName!.Contains("MySql", StringComparison.InvariantCultureIgnoreCase)) + { + // Use MySqlBulkCopy for PostgreSQL + RawInsertMySql(); + } } [Benchmark] @@ -264,4 +271,59 @@ private void RawInsertSqlServer() bulkCopy.WriteToServer(dataTable); } } + + private void RawInsertMySql() + { + var connection = (MySqlConnection)DbContext.Database.GetDbConnection(); + if (connection.State != ConnectionState.Open) + { + connection.Open(); + } + + var bulkCopy = new MySqlBulkCopy(connection); + + bulkCopy.DestinationTableName = nameof(TestEntity); + bulkCopy.BulkCopyTimeout = 60; + + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(0, "Name")); + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(1, "Price")); + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(2, "Identifier")); + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(3, "CreatedAt")); + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(4, "UpdatedAt")); + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(5, "StringEnumValue")); + bulkCopy.ColumnMappings.Add(new MySqlBulkCopyColumnMapping(6, "NumericEnumValue")); + + var dataTable = new DataTable(); + dataTable.Columns.Add("Name", typeof(string)); + dataTable.Columns.Add("Price", typeof(decimal)); + dataTable.Columns.Add("Identifier", typeof(Guid)); + dataTable.Columns.Add("CreatedAt", typeof(DateTime)); + dataTable.Columns.Add("UpdatedAt", typeof(DateTimeOffset)); + dataTable.Columns.Add("StringEnumValue", typeof(string)); + dataTable.Columns.Add("NumericEnumValue", typeof(int)); + + foreach (var entity in data) + { + var row = dataTable.NewRow(); + row["Name"] = entity.Name; + row["Price"] = entity.Price; + row["Identifier"] = entity.Identifier; + row["CreatedAt"] = entity.CreatedAt; + row["UpdatedAt"] = entity.UpdatedAt; + row["StringEnumValue"] = entity.StringEnumValue.ToString(); + row["NumericEnumValue"] = (int)entity.NumericEnumValue; + dataTable.Rows.Add(row); + + if (dataTable.Rows.Count >= 50_000) + { + bulkCopy.WriteToServer(dataTable); + dataTable.Clear(); + } + } + + if (dataTable.Rows.Count > 0) + { + bulkCopy.WriteToServer(dataTable); + } + } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/LibComparatorMySql.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/LibComparatorMySql.cs new file mode 100644 index 0000000..e39fa78 --- /dev/null +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/LibComparatorMySql.cs @@ -0,0 +1,35 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Engines; + +using DotNet.Testcontainers.Containers; + +using Microsoft.EntityFrameworkCore; + +using PhenX.EntityFrameworkCore.BulkInsert.MySql; + +using Testcontainers.MySql; + +namespace PhenX.EntityFrameworkCore.BulkInsert.Benchmark; + +[MinColumn, MaxColumn, BaselineColumn] +[MemoryDiagnoser] +[SimpleJob(RunStrategy.Throughput, launchCount: 1, warmupCount: 0, iterationCount: 5)] +public class LibComparatorMySql : LibComparator +{ + protected override void ConfigureDbContext() + { + var connectionString = GetConnectionString() + ";AllowLoadLocalInfile=true;"; + + DbContext = new TestDbContext(p => p + .UseMySql(connectionString, ServerVersion.AutoDetect(connectionString)) + .UseBulkInsertMySql() + ); + } + + protected override IDatabaseContainer? GetDbContainer() + { + return new MySqlBuilder() + .WithCommand("--log-bin-trust-function-creators=1", "--local-infile=1") + .Build(); + } +} diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/PhenX.EntityFrameworkCore.BulkInsert.Benchmark.csproj b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/PhenX.EntityFrameworkCore.BulkInsert.Benchmark.csproj index 0396901..1b2326d 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/PhenX.EntityFrameworkCore.BulkInsert.Benchmark.csproj +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/PhenX.EntityFrameworkCore.BulkInsert.Benchmark.csproj @@ -7,25 +7,27 @@ - - - + + + + - - + + - - + + - - - + + + + diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/Program.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/Program.cs index 34ca9df..00eeb73 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/Program.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/Program.cs @@ -1,4 +1,4 @@ -using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.Running; namespace PhenX.EntityFrameworkCore.BulkInsert.Benchmark; @@ -11,6 +11,7 @@ public static void Main(string[] args) .Create(DefaultConfig.Instance) .WithOptions(ConfigOptions.DisableOptimizationsValidator); + BenchmarkRunner.Run(config); BenchmarkRunner.Run(config); BenchmarkRunner.Run(config); BenchmarkRunner.Run(config); diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainer.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainer.cs index a230e6c..88d9417 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainer.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainer.cs @@ -1,4 +1,4 @@ -using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Containers; using Microsoft.EntityFrameworkCore; @@ -11,17 +11,14 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContainer; public abstract class TestDbContainer : IAsyncLifetime where TDbContext : TestDbContextBase, new() { + private static readonly TimeSpan WaitTime = TimeSpan.FromSeconds(30); protected readonly IDatabaseContainer? DbContainer; - public TDbContext DbContext { get; private set; } = null!; - protected TestDbContainer() { DbContainer = GetDbContainer(); } - protected string GetRandomContainerName() => "phenx-bulk-insert-test-" + Guid.NewGuid(); - protected abstract IDatabaseContainer? GetDbContainer(); protected virtual string GetConnectionString() @@ -39,29 +36,42 @@ public async Task InitializeAsync() } } - public async Task DisposeAsync() + public async Task CreateContextAsync() { - if (DbContainer != null) + var dbContext = new TDbContext { - await DbContainer.DisposeAsync(); + ConfigureOptions = Configure + }; + + dbContext.Database.SetConnectionString(GetConnectionString()); + + await EnsureConnectedAsync(dbContext); + try + { + await dbContext.Database.EnsureCreatedAsync(); + } + catch + { + // Often fails with SQL server. } + + return dbContext; } - public async Task InitializeDbContextAsync() + protected virtual async Task EnsureConnectedAsync(TDbContext context) { - DbContext = new TDbContext + using var cts = new CancellationTokenSource(WaitTime); + while (!await context.Database.CanConnectAsync(cts.Token)) { - ConfigureOptions = Configure - }; - - DbContext.Database.SetConnectionString(GetConnectionString()); - await DbContext.Database.EnsureCreatedAsync(); + await Task.Delay(100, cts.Token); + } } - public async Task DisposeDbContextAsync() + public async Task DisposeAsync() { - await DbContext.Database.EnsureDeletedAsync(); - await DbContext.DisposeAsync(); - DbContext = null!; + if (DbContainer != null) + { + await DbContainer.DisposeAsync(); + } } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerMySql.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerMySql.cs new file mode 100644 index 0000000..8111c1d --- /dev/null +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerMySql.cs @@ -0,0 +1,34 @@ +using DotNet.Testcontainers.Containers; + +using Microsoft.EntityFrameworkCore; + +using PhenX.EntityFrameworkCore.BulkInsert.MySql; +using PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; + +using Testcontainers.MySql; + +namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContainer; + +public class TestDbContainerMySql : TestDbContainer + where TDbContext : TestDbContextBase, new() +{ + protected override IDatabaseContainer? GetDbContainer() + { + return new MySqlBuilder() + .WithCommand("--log-bin-trust-function-creators=1", "--local-infile=1", "--innodb-print-all-deadlocks=ON") + .WithReuse(true) + .Build(); + } + + protected override string GetConnectionString() + { + return $"{base.GetConnectionString()};AllowLoadLocalInfile=true;"; + } + + protected override void Configure(DbContextOptionsBuilder optionsBuilder) + { + optionsBuilder + .UseMySql(ServerVersion.AutoDetect(GetConnectionString())) + .UseBulkInsertMySql(); + } +} diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerPostgreSql.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerPostgreSql.cs index fd24f5b..22f2ab7 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerPostgreSql.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerPostgreSql.cs @@ -1,4 +1,4 @@ -using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Containers; using Microsoft.EntityFrameworkCore; @@ -15,10 +15,10 @@ public class TestDbContainerPostgreSql : TestDbContainer protected override IDatabaseContainer? GetDbContainer() { return new PostgreSqlBuilder() + .WithReuse(true) .WithDatabase("testdb") .WithUsername("testuser") .WithPassword("testpassword") - .WithName(GetRandomContainerName()) .Build(); } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlServer.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlServer.cs index 3cf3eea..cddcf06 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlServer.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlServer.cs @@ -1,6 +1,5 @@ -using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Containers; -using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using PhenX.EntityFrameworkCore.BulkInsert.SqlServer; @@ -16,20 +15,10 @@ public class TestDbContainerSqlServer : TestDbContainer protected override IDatabaseContainer? GetDbContainer() { return new MsSqlBuilder() - .WithName(GetRandomContainerName()) + .WithReuse(true) .Build(); } - protected override string GetConnectionString() - { - var connectionString = new SqlConnectionStringBuilder(base.GetConnectionString()) - { - InitialCatalog = Guid.NewGuid().ToString("D") - }; - - return connectionString.ToString(); - } - protected override void Configure(DbContextOptionsBuilder optionsBuilder) { optionsBuilder diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlite.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlite.cs index 4138c02..b6e12bf 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlite.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContainer/TestDbContainerSqlite.cs @@ -1,4 +1,4 @@ -using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Containers; using Microsoft.EntityFrameworkCore; @@ -24,4 +24,9 @@ protected override void Configure(DbContextOptionsBuilder optionsBuilder) .UseSqlite() .UseBulkInsertSqlite(); } + + protected override Task EnsureConnectedAsync(TDbContext context) + { + return Task.CompletedTask; + } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs index 9602c48..f8cb721 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs @@ -1,4 +1,4 @@ -using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; using Microsoft.EntityFrameworkCore; @@ -20,6 +20,9 @@ public class TestEntity [Column("some_price")] public decimal Price { get; set; } + [Column("test_run")] + public Guid TestRun { get; set; } + [Column("the_identifier")] public Guid Identifier { get; set; } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithConverters.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithConverters.cs index 72e0af7..ecfc945 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithConverters.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithConverters.cs @@ -14,5 +14,8 @@ public class TestEntityWithConverters [Column("created_at")] public DateTime CreatedAt { get; set; } + + [Column("test_run")] + public Guid TestRun { get; set; } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/PhenX.EntityFrameworkCore.BulkInsert.Tests.csproj b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/PhenX.EntityFrameworkCore.BulkInsert.Tests.csproj index 5ecb9e6..009db83 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/PhenX.EntityFrameworkCore.BulkInsert.Tests.csproj +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/PhenX.EntityFrameworkCore.BulkInsert.Tests.csproj @@ -23,9 +23,11 @@ + + diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs index 0a95eaa..62f4bc2 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs @@ -1,4 +1,4 @@ -using PhenX.EntityFrameworkCore.BulkInsert.Extensions; +using PhenX.EntityFrameworkCore.BulkInsert.Extensions; using PhenX.EntityFrameworkCore.BulkInsert.Options; using PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContainer; using PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; @@ -10,12 +10,26 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.Tests.Basic; public abstract class BasicTestsBase : IClassFixture, IAsyncLifetime where TFixture : TestDbContainer { - protected BasicTestsBase(TFixture dbContainer) + private readonly Guid _run = Guid.NewGuid(); + private TestDbContext _context = null!; + + protected BasicTestsBase(TestDbContainer dbContainer) { DbContainer = dbContainer; } - protected TFixture DbContainer { get; } + public async Task InitializeAsync() + { + _context = await DbContainer.CreateContextAsync(); + } + + public Task DisposeAsync() + { + _context.Dispose(); + return Task.CompletedTask; + } + + protected TestDbContainer DbContainer { get; } [Fact] public async Task InsertsEntitiesSuccessfully() @@ -23,18 +37,18 @@ public async Task InsertsEntitiesSuccessfully() // Arrange var entities = new List { - new TestEntity { Id = 1, Name = "Entity1" }, - new TestEntity { Id = 2, Name = "Entity2" } + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" } }; // Act - await DbContainer.DbContext.ExecuteBulkInsertReturnEntitiesAsync(entities); + await _context.ExecuteBulkInsertAsync(entities); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Equal(2, insertedEntities.Count); - Assert.Contains(insertedEntities, e => e.Name == "Entity1"); - Assert.Contains(insertedEntities, e => e.Name == "Entity2"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); } [Fact] @@ -43,18 +57,130 @@ public void InsertsEntitiesSuccessfully_Sync() // Arrange var entities = new List { - new TestEntity { Id = 1, Name = "Entity1" }, - new TestEntity { Id = 2, Name = "Entity2" } + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" } + }; + + // Act + _context.ExecuteBulkInsert(entities); + + // Assert + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + Assert.Equal(2, insertedEntities.Count); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); + } + + [SkippableFact] + public async Task InsertsEntitiesAndReturn() + { + Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); + + // Arrange + var entities = new List + { + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" } + }; + + // Act + var insertedEntities = await _context.ExecuteBulkInsertReturnEntitiesAsync(entities); + + // Assert + Assert.Equal(2, insertedEntities.Count); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); + } + + [SkippableFact] + public void InsertsEntitiesAndReturn_Sync() + { + Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); + + // Arrange + var entities = new List + { + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" } }; // Act - DbContainer.DbContext.ExecuteBulkInsertReturnEntities(entities); + var insertedEntities = _context.ExecuteBulkInsertReturnEntities(entities); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); Assert.Equal(2, insertedEntities.Count); - Assert.Contains(insertedEntities, e => e.Name == "Entity1"); - Assert.Contains(insertedEntities, e => e.Name == "Entity2"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); + } + + [SkippableFact] + public async Task InsertsEntities_MultipleTimes() + { + Skip.If(_context.Database.ProviderName!.Contains("Postgres", StringComparison.InvariantCultureIgnoreCase)); + Skip.If(_context.Database.ProviderName!.Contains("SqlServer", StringComparison.InvariantCultureIgnoreCase)); + + // Arrange + var entities = new List + { + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" } + }; + + // Act + await _context.ExecuteBulkInsertAsync(entities); + + foreach (var entity in entities) + { + entity.NumericEnumValue = NumericEnum.Second; + } + + await _context.ExecuteBulkInsertAsync(entities, + onConflict: new OnConflictOptions + { + Update = e => e, + }); + + // Assert + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + Assert.Equal(2, insertedEntities.Count); + Assert.Contains(insertedEntities, e => e.NumericEnumValue == NumericEnum.Second); + Assert.Contains(insertedEntities, e => e.NumericEnumValue == NumericEnum.Second); + } + + [SkippableFact] + public async Task InsertsEntities_MultipleTimes_With_Conflict_On_Id() + { + Skip.If(_context.Database.ProviderName!.Contains("Postgres", StringComparison.InvariantCultureIgnoreCase)); + Skip.If(_context.Database.ProviderName!.Contains("SqlServer", StringComparison.InvariantCultureIgnoreCase)); + + // Arrange + var entities = new List + { + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" } + }; + + // Act + await _context.ExecuteBulkInsertAsync(entities); + + var insertedEntities0 = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + foreach (var entity in insertedEntities0) + { + entity.Name = $"Updated_{entity.Name}"; + } + + await _context.ExecuteBulkInsertAsync(insertedEntities0, + o => o.CopyGeneratedColumns = true, + onConflict: new OnConflictOptions + { + Update = e => e, + }); + + // Assert + var insertedEntities1 = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + Assert.Equal(2, insertedEntities1.Count); + Assert.Contains(insertedEntities1, e => e.Name == $"Updated_{_run}_Entity1"); + Assert.Contains(insertedEntities1, e => e.Name == $"Updated_{_run}_Entity2"); } [Fact] @@ -63,39 +189,41 @@ public async Task InsertsEntitiesMoveRowsSuccessfully() // Arrange var entities = new List { - new TestEntity { Id = 1, Name = "Entity1" }, - new TestEntity { Id = 2, Name = "Entity2" } + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" } }; // Act - await DbContainer.DbContext.ExecuteBulkInsertReturnEntitiesAsync(entities, o => + await _context.ExecuteBulkInsertAsync(entities, o => { o.MoveRows = true; }); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Equal(2, insertedEntities.Count); - Assert.Contains(insertedEntities, e => e.Name == "Entity1"); - Assert.Contains(insertedEntities, e => e.Name == "Entity2"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); } - [Fact] + [SkippableFact] public async Task InsertsEntitiesWithConflict_SingleColumn() { - DbContainer.DbContext.TestEntities.Add(new TestEntity { Name = "Entity1" }); - await DbContainer.DbContext.SaveChangesAsync(); - DbContainer.DbContext.ChangeTracker.Clear(); + Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); + + _context.TestEntities.Add(new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }); + await _context.SaveChangesAsync(); + _context.ChangeTracker.Clear(); // Arrange var entities = new List { - new TestEntity { Name = "Entity1" }, - new TestEntity { Name = "Entity2" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" }, }; // Act - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => + await _context.ExecuteBulkInsertAsync(entities, o => { o.MoveRows = true; }, new OnConflictOptions @@ -111,26 +239,28 @@ await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => }); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Equal(2, insertedEntities.Count); - Assert.Contains(insertedEntities, e => e.Name == "Entity1 - Conflict"); - Assert.Contains(insertedEntities, e => e.Name == "Entity2"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1 - Conflict"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); } - [Fact] + [SkippableFact] public async Task InsertsEntitiesWithConflict_DoNothing() { - DbContainer.DbContext.TestEntities.Add(new TestEntity { Name = "Entity1" }); - await DbContainer.DbContext.SaveChangesAsync(); - DbContainer.DbContext.ChangeTracker.Clear(); + Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); + + _context.TestEntities.Add(new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }); + await _context.SaveChangesAsync(); + _context.ChangeTracker.Clear(); var entities = new List { - new TestEntity { Name = "Entity1" }, - new TestEntity { Name = "Entity2" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2" }, }; - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => + await _context.ExecuteBulkInsertAsync(entities, o => { o.MoveRows = true; }, new OnConflictOptions @@ -139,28 +269,28 @@ await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => // Pas de Update => DO NOTHING }); - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Equal(2, insertedEntities.Count); - Assert.Contains(insertedEntities, e => e.Name == "Entity1"); - Assert.Contains(insertedEntities, e => e.Name == "Entity2"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); } [SkippableFact] public async Task InsertsEntitiesWithConflict_Condition() { - // Skip.If(DbContainer.DbContext.Database.ProviderName!.Contains("Npgsql", StringComparison.InvariantCultureIgnoreCase)); + Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); - DbContainer.DbContext.TestEntities.Add(new TestEntity { Name = "Entity1", Price = 10 }); - await DbContainer.DbContext.SaveChangesAsync(); - DbContainer.DbContext.ChangeTracker.Clear(); + _context.TestEntities.Add(new TestEntity { TestRun = _run, Name = $"{_run}_Entity1", Price = 10 }); + await _context.SaveChangesAsync(); + _context.ChangeTracker.Clear(); var entities = new List { - new TestEntity { Name = "Entity1", Price = 20 }, - new TestEntity { Name = "Entity2", Price = 30 }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1", Price = 20 }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2", Price = 30 }, }; - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => + await _context.ExecuteBulkInsertAsync(entities, o => { o.MoveRows = true; }, new OnConflictOptions @@ -170,26 +300,28 @@ await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => Condition = "EXCLUDED.some_price > test_entity.some_price" }); - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Equal(2, insertedEntities.Count); - Assert.Contains(insertedEntities, e => e.Name == "Entity1" && e.Price == 20); - Assert.Contains(insertedEntities, e => e.Name == "Entity2" && e.Price == 30); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1" && e.Price == 20); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2" && e.Price == 30); } - [Fact] + [SkippableFact] public async Task InsertsEntitiesWithConflict_MultipleColumns() { - DbContainer.DbContext.TestEntities.Add(new TestEntity { Name = "Entity1", Price = 10 }); - await DbContainer.DbContext.SaveChangesAsync(); - DbContainer.DbContext.ChangeTracker.Clear(); + Skip.If(_context.Database.ProviderName!.Contains("Mysql", StringComparison.InvariantCultureIgnoreCase)); + + _context.TestEntities.Add(new TestEntity { TestRun = _run, Name = $"{_run}_Entity1", Price = 10 }); + await _context.SaveChangesAsync(); + _context.ChangeTracker.Clear(); var entities = new List { - new TestEntity { Name = "Entity1", Price = 20, Identifier = Guid.NewGuid() }, - new TestEntity { Name = "Entity2", Price = 30, Identifier = Guid.NewGuid() }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity1", Price = 20, Identifier = Guid.NewGuid() }, + new TestEntity { TestRun = _run, Name = $"{_run}_Entity2", Price = 30, Identifier = Guid.NewGuid() }, }; - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => + await _context.ExecuteBulkInsertAsync(entities, o => { o.MoveRows = true; }, new OnConflictOptions @@ -201,12 +333,12 @@ await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => } }); - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Equal(2, insertedEntities.Count); - Assert.Equal(1, insertedEntities.Count(e => e.Name == "Entity1 - Conflict")); - Assert.Contains(insertedEntities, e => e.Name == "Entity2"); + Assert.Equal(1, insertedEntities.Count(e => e.Name == $"{_run}_Entity1 - Conflict")); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity2"); - var entity1 = insertedEntities.First(e => e.Name == "Entity1 - Conflict"); + var entity1 = insertedEntities.First(e => e.Name == $"{_run}_Entity1 - Conflict"); Assert.Equal(0, entity1.Price); } @@ -217,10 +349,10 @@ public async Task DoesNothingWhenEntitiesAreEmpty() var entities = new List(); // Act - await Assert.ThrowsAsync(async () => await DbContainer.DbContext.ExecuteBulkInsertAsync(entities)); + await Assert.ThrowsAsync(async () => await _context.ExecuteBulkInsertAsync(entities)); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Empty(insertedEntities); } @@ -231,25 +363,25 @@ public async Task InsertsEntities_Many() const int count = 156055; var entities = Enumerable.Range(1, count).Select(i => new TestEntity { - Id = i, - Name = $"Entity{i}", + Name = $"{_run}_Entity{i}", Price = (decimal)(i * 0.1), Identifier = Guid.NewGuid(), StringEnumValue = (StringEnum)(i % 2), NumericEnumValue = (NumericEnum)(i % 2), + TestRun = _run, }).ToList(); // Act - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities, o => + await _context.ExecuteBulkInsertAsync(entities, o => { o.MoveRows = false; }); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); Assert.Equal(count, insertedEntities.Count); - Assert.Contains(insertedEntities, e => e.Name == "Entity1"); - Assert.Contains(insertedEntities, e => e.Name == "Entity" + count); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_Entity" + count); } [Fact] @@ -259,18 +391,18 @@ public async Task InsertAndRead_EntityWithValueConverters() var now = DateTime.UtcNow; var entities = new List { - new() { Name = "Entity1", CreatedAt = now }, - new() { Name = "Entity2", CreatedAt = now.AddDays(-1) } + new() { TestRun = _run, Name = $"{_run}_Entity1", CreatedAt = now }, + new() { TestRun = _run, Name = $"{_run}_Entity2", CreatedAt = now.AddDays(-1) } }; // Act - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities); - var inserted = DbContainer.DbContext.TestEntitiesWithConverters.ToList(); + await _context.ExecuteBulkInsertAsync(entities); + var inserted = _context.TestEntitiesWithConverters.Where(x => x.TestRun == _run).ToList(); // Assert Assert.Equal(2, inserted.Count); - Assert.Contains(inserted, e => e.Name == "Entity1" && e.CreatedAt == now); - Assert.Contains(inserted, e => e.Name == "Entity2" && e.CreatedAt == now.AddDays(-1)); + Assert.Contains(inserted, e => e.Name == $"{_run}_Entity1" && e.CreatedAt == now); + Assert.Contains(inserted, e => e.Name == $"{_run}_Entity2" && e.CreatedAt == now.AddDays(-1)); } [Fact] @@ -279,20 +411,20 @@ public async Task BulkInsert_WithOpenTransaction_CommitsSuccessfully() // Arrange var entities = new List { - new TestEntity { Name = "EntityWithTx1" }, - new TestEntity { Name = "EntityWithTx2" } + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTx1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTx2" } }; - await using var transaction = await DbContainer.DbContext.Database.BeginTransactionAsync(); + await using var transaction = await _context.Database.BeginTransactionAsync(); - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities); + await _context.ExecuteBulkInsertAsync(entities); await transaction.CommitAsync(); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); - Assert.Contains(insertedEntities, e => e.Name == "EntityWithTx1"); - Assert.Contains(insertedEntities, e => e.Name == "EntityWithTx2"); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_EntityWithTx1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_EntityWithTx2"); } [Fact] @@ -301,20 +433,20 @@ public void BulkInsert_WithOpenTransaction_CommitsSuccessfully_Sync() // Arrange var entities = new List { - new TestEntity { Name = "EntityWithTx1" }, - new TestEntity { Name = "EntityWithTx2" } + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTx1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTx2" } }; - var transaction = DbContainer.DbContext.Database.BeginTransaction(); + var transaction = _context.Database.BeginTransaction(); - DbContainer.DbContext.ExecuteBulkInsert(entities); + _context.ExecuteBulkInsert(entities); transaction.Commit(); // Assert - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); - Assert.Contains(insertedEntities, e => e.Name == "EntityWithTx1"); - Assert.Contains(insertedEntities, e => e.Name == "EntityWithTx2"); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_EntityWithTx1"); + Assert.Contains(insertedEntities, e => e.Name == $"{_run}_EntityWithTx2"); } [Fact] @@ -323,21 +455,21 @@ public async Task BulkInsert_WithOpenTransaction_RollsBackOnFailure() // Arrange var entities = new List { - new TestEntity { Name = "EntityWithTxFail1" }, - new TestEntity { Name = "EntityWithTxFail2" } + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTxFail1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTxFail2" } }; - await using var transaction = await DbContainer.DbContext.Database.BeginTransactionAsync(); + await using var transaction = await _context.Database.BeginTransactionAsync(); - await DbContainer.DbContext.ExecuteBulkInsertAsync(entities); + await _context.ExecuteBulkInsertAsync(entities); await transaction.RollbackAsync(); // Assert - DbContainer.DbContext.ChangeTracker.Clear(); - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); - Assert.DoesNotContain(insertedEntities, e => e.Name == "EntityWithTxFail1"); - Assert.DoesNotContain(insertedEntities, e => e.Name == "EntityWithTxFail2"); + _context.ChangeTracker.Clear(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + Assert.DoesNotContain(insertedEntities, e => e.Name == $"{_run}_EntityWithTxFail1"); + Assert.DoesNotContain(insertedEntities, e => e.Name == $"{_run}_EntityWithTxFail2"); } [Fact] @@ -346,30 +478,20 @@ public void BulkInsert_WithOpenTransaction_RollsBackOnFailure_Sync() // Arrange var entities = new List { - new TestEntity { Name = "EntityWithTxFail1" }, - new TestEntity { Name = "EntityWithTxFail2" } + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTxFail1" }, + new TestEntity { TestRun = _run, Name = $"{_run}_EntityWithTxFail2" } }; - using var transaction = DbContainer.DbContext.Database.BeginTransaction(); + using var transaction = _context.Database.BeginTransaction(); - DbContainer.DbContext.ExecuteBulkInsert(entities); + _context.ExecuteBulkInsert(entities); transaction.Rollback(); // Assert - DbContainer.DbContext.ChangeTracker.Clear(); - var insertedEntities = DbContainer.DbContext.TestEntities.ToList(); - Assert.DoesNotContain(insertedEntities, e => e.Name == "EntityWithTxFail1"); - Assert.DoesNotContain(insertedEntities, e => e.Name == "EntityWithTxFail2"); - } - - public Task InitializeAsync() - { - return DbContainer.InitializeDbContextAsync(); - } - - public Task DisposeAsync() - { - return DbContainer.DisposeDbContextAsync(); + _context.ChangeTracker.Clear(); + var insertedEntities = _context.TestEntities.Where(x => x.TestRun == _run).ToList(); + Assert.DoesNotContain(insertedEntities, e => e.Name == $"{_run}_EntityWithTxFail1"); + Assert.DoesNotContain(insertedEntities, e => e.Name == $"{_run}_EntityWithTxFail2"); } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsMySql.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsMySql.cs new file mode 100644 index 0000000..9181409 --- /dev/null +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsMySql.cs @@ -0,0 +1,14 @@ +using PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContainer; +using PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; + +using Xunit; + +namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.Tests.Basic; + +[Trait("Category", "MySql")] +public class BasicTestsMySql : BasicTestsBase> +{ + public BasicTestsMySql(TestDbContainerMySql dbContainer) : base(dbContainer) + { + } +}