From b7b70e9934f8cd9991afb86ad772e1731e4a1009 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 11:08:04 +0200 Subject: [PATCH 01/13] Cleanup. --- .../MySqlBulkInsertProvider.cs | 9 +- .../PostgreSqlBulkInsertProvider.cs | 16 ++- .../SqlServerBulkInsertProvider.cs | 16 ++- .../SqlServerDialectBuilder.cs | 88 +++++++++----- .../SqliteBulkInsertProvider.cs | 109 ++++++++++++------ .../BulkInsertProviderBase.cs | 20 ++-- .../Dialect/SqlDialectBuilder.cs | 70 ++++------- .../Metadata/TableMetadata.cs | 17 ++- 8 files changed, 205 insertions(+), 140 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs index bc72001..8ac6832 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs @@ -15,10 +15,6 @@ public MySqlBulkInsertProvider(ILogger? logger = null) { } - //language=sql - /// - protected override string CreateTableCopySql => "CREATE TEMPORARY TABLE {0} SELECT * FROM {1} WHERE 1 = 0;"; - //language=sql /// protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD {BulkInsertId} INT AUTO_INCREMENT PRIMARY KEY;"; @@ -26,6 +22,11 @@ public MySqlBulkInsertProvider(ILogger? logger = null) /// protected override string GetTempTableName(string tableName) => $"#_temp_bulk_insert_{tableName}"; + protected override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + return $"CREATE TEMPORARY TABLE {tempNameName} SELECT * FROM {tableInfo.QuotedTableName} WHERE 1 = 0;"; + } + /// public override Task> BulkInsertReturnEntities( bool sync, diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs index 239b967..be61cb7 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs @@ -1,3 +1,5 @@ +using System.Text; + using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; @@ -19,17 +21,23 @@ public PostgreSqlBulkInsertProvider(ILogger? logge //language=sql /// - protected override string CreateTableCopySql => "CREATE TEMPORARY TABLE {0} AS TABLE {1} WITH NO DATA;"; + protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD COLUMN {BulkInsertId} SERIAL PRIMARY KEY;"; - //language=sql /// - protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD COLUMN {BulkInsertId} SERIAL PRIMARY KEY;"; + protected override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + return $"CREATE TEMPORARY TABLE {tempNameName} AS TABLE {tableInfo.QuotedTableName} WITH NO DATA;"; + } private static string GetBinaryImportCommand(TableMetadata tableInfo, string tableName) { var columns = tableInfo.GetProperties(false).Select(X => X.QuotedColumName); - return $"COPY {tableName} ({string.Join(", ", columns)}) FROM STDIN (FORMAT BINARY)"; + var sql = new StringBuilder(); + sql.Append($"COPY {tableName} ("); + sql.AppendColumns(tableInfo.GetProperties(false)); + sql.Append(") FROM STDIN (FORMAT BINARY)"); + return sql.ToString(); } /// diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs index 42276d3..84af4a8 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs @@ -1,3 +1,5 @@ +using System.Text; + using JetBrains.Annotations; using Microsoft.Data.SqlClient; @@ -17,10 +19,6 @@ public SqlServerBulkInsertProvider(ILogger? logger { } - //language=sql - /// - protected override string CreateTableCopySql => "SELECT {2} INTO {0} FROM {1} WHERE 1 = 0;"; - //language=sql /// protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD {BulkInsertId} INT IDENTITY PRIMARY KEY;"; @@ -28,6 +26,16 @@ public SqlServerBulkInsertProvider(ILogger? logger /// protected override string GetTempTableName(string tableName) => $"#_temp_bulk_insert_{tableName}"; + protected override string CreateTableCopySql(string templNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + var sb = new StringBuilder(); + sb.Append("SELECT"); + sb.AppendJoin(", ", columns.Select(x => x.QuotedColumName)); + sb.Append($"INTO {templNameName} FROM {tableInfo.QuotedTableName} WHERE 1 = 0;"); + + return sb.ToString(); + } + /// protected override async Task BulkInsert( bool sync, diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs index 90569b2..640901b 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs @@ -1,5 +1,7 @@ using System.Text; +using Microsoft.Extensions.Primitives; + using PhenX.EntityFrameworkCore.BulkInsert.Dialect; using PhenX.EntityFrameworkCore.BulkInsert.Metadata; using PhenX.EntityFrameworkCore.BulkInsert.Options; @@ -18,16 +20,10 @@ public override string BuildMoveDataSql( TableMetadata target, string source, IReadOnlyList insertedProperties, - IReadOnlyList properties, + IReadOnlyList returnedProperties, BulkInsertOptions options, OnConflictOptions? onConflict = null) { - var insertedColumns = insertedProperties.Select(x => x.QuotedColumName); - var insertedColumnList = string.Join(", ", insertedColumns); - - var returnedColumns = properties.Select(p => $"INSERTED.{p.ColumnName} AS {p.ColumnName}"); - var columnList = string.Join(", ", returnedColumns); - var q = new StringBuilder(); if (options.CopyGeneratedColumns) @@ -36,49 +32,76 @@ public override string BuildMoveDataSql( } // Merge handling - if (onConflict is OnConflictOptions onConflictTyped && onConflictTyped.Match != null) + if (onConflict is OnConflictOptions onConflictTyped) { - var matchColumns = GetColumns(target, onConflictTyped.Match); - var matchOn = string.Join(" AND ", - matchColumns.Select(col => $"TARGET.{col} = SOURCE.{col}")); - - var updateSet = onConflictTyped.Update != null - ? string.Join(", ", GetUpdates(target, insertedProperties, onConflictTyped.Update)) - : null; + IEnumerable matchColumns; + if (onConflictTyped.Match != null) + { + matchColumns = GetColumns(target, onConflictTyped.Match); + } + else if (target.PrimaryKey.Count > 0) + { + matchColumns = target.PrimaryKey.Select(x => x.QuotedColumName); + } + else + { + throw new InvalidOperationException("Table has no primary key that can be used for conflict detection."); + } q.AppendLine($"MERGE INTO {target.QuotedTableName} AS TARGET"); - q.AppendLine( - $"USING (SELECT {string.Join(", ", insertedColumns)} FROM {source}) AS SOURCE ({insertedColumnList})"); - q.AppendLine($"ON {matchOn}"); - if (updateSet != null) + q.Append("USING (SELECT "); + q.AppendColumns(insertedProperties); + q.Append($" FROM {source}) AS SOURCE ("); + q.AppendColumns(insertedProperties); + q.AppendLine(")"); + + q.Append("ON "); + q.AppendJoin($" AND ", matchColumns, (b, col) => b.Append($"TARGET.{col} = SOURCE.{col}")); + q.AppendLine(); + + if (onConflictTyped.Update != null) { - q.AppendLine($"WHEN MATCHED THEN UPDATE SET {updateSet}"); + q.AppendLine($"WHEN MATCHED THEN UPDATE SET "); + q.AppendJoin(", ", GetUpdates(target, insertedProperties, onConflictTyped.Update)); + q.AppendLine(); } - q.AppendLine( - $"WHEN NOT MATCHED THEN INSERT ({insertedColumnList}) VALUES ({string.Join(", ", insertedColumns.Select(c => $"SOURCE.{c}"))})"); + q.Append($"WHEN NOT MATCHED THEN INSERT ("); + q.AppendColumns(insertedProperties); + q.AppendLine(")"); + + q.Append("VALUES ("); + q.AppendJoin(", ", insertedProperties, (b, col) => b.Append($"SOURCE.{col.QuotedColumName}")); + q.AppendLine(")"); - if (columnList.Length != 0) + if (returnedProperties.Count != 0) { - q.AppendLine($"OUTPUT {columnList}"); + q.Append("OUTPUT "); + q.AppendJoin($", ", returnedProperties, (b, col) => b.Append($"INSERTED.{col.QuotedColumName} AS {col.QuotedColumName}")); + q.AppendLine(); } } // No conflict handling else { - q.AppendLine($"INSERT INTO {target.QuotedTableName} ({insertedColumnList})"); + q.Append($"INSERT INTO {target.QuotedTableName} ("); + q.AppendColumns(insertedProperties); + q.AppendLine(")"); - if (columnList.Length != 0) + if (returnedProperties.Count != 0) { - q.AppendLine($"OUTPUT {columnList}"); + q.Append("OUTPUT "); + q.AppendJoin($", ", returnedProperties, (b, col) => b.Append($"INSERTED.{col.QuotedColumName} AS {col.QuotedColumName}")); + q.AppendLine(); } - q.AppendLine($""" - SELECT {insertedColumnList} - FROM {source} - """); + q.Append("SELECT "); + q.AppendColumns(insertedProperties); + q.AppendLine(); + q.Append($"FROM {source}"); + q.AppendLine(); } q.AppendLine(";"); @@ -88,7 +111,8 @@ public override string BuildMoveDataSql( q.AppendLine($"SET IDENTITY_INSERT {target.QuotedTableName} OFF;"); } - return q.ToString(); + var x = q.ToString(); + return x; } protected override string GetExcludedColumnName(string columnName) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs index 808a8d8..bf2ca7e 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs @@ -1,10 +1,12 @@ using System.Data.Common; +using System.Text; using JetBrains.Annotations; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; using PhenX.EntityFrameworkCore.BulkInsert.Metadata; using PhenX.EntityFrameworkCore.BulkInsert.Options; @@ -23,11 +25,13 @@ public SqliteBulkInsertProvider(ILogger? logger = null //language=sql /// - protected override string CreateTableCopySql => "CREATE TEMP TABLE {0} AS SELECT * FROM {1} WHERE 0;"; + protected override string AddTableCopyBulkInsertId => "--"; // No need to add an ID column in SQLite - //language=sql /// - protected override string AddTableCopyBulkInsertId => "--"; // No need to add an ID column in SQLite + protected override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + return $"CREATE TEMP TABLE {tempNameName} AS SELECT * FROM {tableInfo.QuotedTableName} WHERE 0;"; + } /// protected override Task AddBulkInsertIdColumn( @@ -78,48 +82,58 @@ private static SqliteType GetSqliteType(Type clrType) return sqliteType; } - throw new InvalidOperationException("Unknown Sqlite type for " + clrType); + throw new InvalidOperationException($"Unknown Sqlite type for {clrType}"); } - private DbCommand GetInsertCommand(DbContext context, TableMetadata tableInfo, string tableName, - BulkInsertOptions options, + private static DbCommand GetInsertCommand( + DbContext context, + string tableName, + IReadOnlyList columns, + SqliteType[] columnTypes, + StringBuilder sb, int batchSize) { - var columns = tableInfo.GetProperties(options.CopyGeneratedColumns); - var cmd = context.Database.GetDbConnection().CreateCommand(); + var command = context.Database.GetDbConnection().CreateCommand(); - var sqliteColumns = columns - .Select(c => new - { - Name = c.ColumnName, - Type = GetSqliteType(c.ProviderClrType ?? c.ClrType) - }) - .ToArray(); - - var i = 0; - var batches = Enumerable - .Repeat(0, batchSize) - .Select(_ => + sb.Clear(); + sb.AppendLine($"INSERT INTO {tableName} ("); + sb.AppendColumns(columns); + sb.AppendLine(")"); + sb.AppendLine("VALUES"); + + var p = 0; + for (var i = 0; i < batchSize; i++) + { + if (i > 0) { - var cols = sqliteColumns.Select(column => - { - var paramName = $"@p{i++}"; + sb.Append(','); + } - cmd.Parameters.Add(new SqliteParameter(paramName, column.Type)); + sb.Append('('); - return paramName; - }); + var columnIndex = 0; + foreach (var column in columns) + { + var parameterName = $"@p{p++}"; + command.Parameters.Add(new SqliteParameter(parameterName, columnTypes[columnIndex])); - return $"({string.Join(",", cols)})"; - }); + if (columnIndex > 0) + { + sb.Append(", "); + } - var sql = $"INSERT INTO {tableName} ({string.Join(",", sqliteColumns.Select(c => Quote(c.Name)))}) VALUES {string.Join(",", batches)}"; + sb.Append(parameterName); + columnIndex++; + } - cmd.CommandText = sql; + sb.Append(')'); + sb.AppendLine(); + } - cmd.Prepare(); + command.CommandText = sb.ToString(); + command.Prepare(); - return cmd; + return command; } /// @@ -138,7 +152,20 @@ CancellationToken ctk var batchSize = options.BatchSize ?? 5; batchSize = Math.Min(batchSize, maxParams / properties.Count); - await using var insertCommand = GetInsertCommand(context, tableInfo, tableName, options, batchSize); + // The StringBuilder can be resuse between the batches. + var sb = new StringBuilder(); + + var columnList = tableInfo.GetProperties(options.CopyGeneratedColumns); + var columnTypes = columnList.Select(c => GetSqliteType(c.ProviderClrType ?? c.ClrType)).ToArray(); + + await using var insertCommand = + GetInsertCommand( + context, + tableName, + columnList, + columnTypes, + sb, + batchSize); foreach (var chunk in entities.Chunk(batchSize)) { @@ -151,7 +178,14 @@ CancellationToken ctk // Last chunk else { - var partialInsertCommand = GetInsertCommand(context, tableInfo, tableName, options, chunk.Length); + await using var partialInsertCommand = + GetInsertCommand( + context, + tableName, + columnList, + columnTypes, + sb, + chunk.Length); FillValues(chunk, partialInsertCommand.Parameters, properties); await ExecuteCommand(sync, partialInsertCommand, ctk); @@ -174,15 +208,14 @@ private static async Task ExecuteCommand(bool sync, DbCommand insertCommand, Can private static void FillValues(T[] chunk, DbParameterCollection parameters, IReadOnlyList properties) where T : class { - var index = 0; + var p = 0; foreach (var entity in chunk) { foreach (var property in properties) { var value = property.GetValue(entity); - parameters[index].Value = value; - - index++; + parameters[p].Value = value; + p++; } } } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs index 780d4e2..65fce87 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs @@ -1,4 +1,5 @@ using System.Data.Common; +using System.Reflection; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage; @@ -21,7 +22,6 @@ internal abstract class BulkInsertProviderBase(ILogger "_bulk_insert_id"; - protected abstract string CreateTableCopySql { get; } protected abstract string AddTableCopyBulkInsertId { get; } SqlDialectBuilder IBulkInsertProvider.SqlDialect => SqlDialect; @@ -34,9 +34,9 @@ protected async Task CreateTableCopyAsync( CancellationToken cancellationToken = default) where T : class { var tempTableName = SqlDialect.QuoteTableName(null, GetTempTableName(tableInfo.TableName)); - var tempColumns = string.Join(", ", tableInfo.GetProperties(options.CopyGeneratedColumns).Select(x => x.QuotedColumName)); + var tempColumns = tableInfo.GetProperties(options.CopyGeneratedColumns); - var query = string.Format(CreateTableCopySql, tempTableName, tableInfo.QuotedTableName, tempColumns); + var query = CreateTableCopySql(tempTableName, tableInfo, tempColumns); await ExecuteAsync(sync, context, query, cancellationToken); await AddBulkInsertIdColumn(sync, context, tempTableName, cancellationToken); @@ -44,6 +44,8 @@ protected async Task CreateTableCopyAsync( return tempTableName; } + protected abstract string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns); + protected virtual async Task AddBulkInsertIdColumn(bool sync, DbContext context, string tempTableName, CancellationToken cancellationToken) where T : class { @@ -106,10 +108,14 @@ private async Task> CopyFromTempTableWithoutKeysAsync( where T : class where TResult : class { - var movedProperties = tableInfo.GetProperties(options.CopyGeneratedColumns); - var returnedProperties = returnData ? tableInfo.GetProperties() : []; - - var query = SqlDialect.BuildMoveDataSql(tableInfo, tempTableName, movedProperties, returnedProperties, options, onConflict); + var query = + SqlDialect.BuildMoveDataSql( + tableInfo, + tempTableName, + tableInfo.GetProperties(options.CopyGeneratedColumns), + returnData ? tableInfo.GetProperties() : [], + options, + onConflict); if (returnData) { diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs index 72ef576..9e5aa69 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs @@ -19,8 +19,8 @@ internal abstract class SqlDialectBuilder /// /// Source table /// Target table name - /// Properties to be copied - /// Properties to be returned + /// Properties to be inserted + /// Properties to be returned /// Bulk insert options /// On conflict options /// Entity type @@ -29,34 +29,29 @@ public virtual string BuildMoveDataSql( TableMetadata target, string source, IReadOnlyList insertedProperties, - IReadOnlyList properties, + IReadOnlyList returnedProperties, BulkInsertOptions options, OnConflictOptions? onConflict = null) { - var insertedColumns = insertedProperties.Select(p => p.QuotedColumName); - var insertedColumnList = string.Join(", ", insertedColumns); - - var returnedColumns = properties.Select(p => p.QuotedColumName); - var columnList = string.Join(", ", returnedColumns); - var q = new StringBuilder(); if (SupportsMoveRows && options.MoveRows) { - q.AppendLine($""" - WITH moved_rows AS ( - DELETE FROM {source} - RETURNING {insertedColumnList} - ) - """); + // WITH moved_rows AS (DELETE FROM {source) RETURNING {insertedProperties}) + q.Append($"WITH moved_rows AS (DELETE FROM {source} RETURNING "); + q.AppendColumns(insertedProperties); + q.AppendLine(")"); + source = "moved_rows"; } - q.AppendLine($""" - INSERT INTO {target.QuotedTableName} ({insertedColumnList}) - SELECT {insertedColumnList} - FROM {source} - WHERE TRUE - """); + // INSERT INTO {target} ({columns}) SELECT {columns} FROM {source} WHERE TRUE + q.Append($"INSERT INTO {target.QuotedTableName} ("); + q.AppendColumns(insertedProperties); + q.AppendLine(")"); + q.Append("SELECT "); + q.AppendColumns(insertedProperties); + q.AppendLine(); + q.AppendLine($"FROM {source} WHERE TRUE"); if (onConflict is OnConflictOptions onConflictTyped) { @@ -89,9 +84,11 @@ WHERE TRUE } } - if (columnList.Length != 0) + if (returnedProperties.Count != 0) { - q.AppendLine($"RETURNING {columnList}"); + q.Append("RETURNING "); + q.AppendJoin(", ", returnedProperties.Select(p => p.QuotedColumName)); + q.AppendLine(); } q.AppendLine(";"); @@ -107,36 +104,13 @@ protected virtual void AppendDoNothing(StringBuilder sql, IEnumerable updates) { sql.AppendLine("DO UPDATE SET"); - - var i = 0; - foreach (var update in updates) - { - if (i > 0) - { - sql.Append(", "); - } - - sql.Append(update); - i++; - }; + sql.AppendJoin(", ", updates); } protected virtual void AppendConflictMatch(StringBuilder sql, IEnumerable columns) { sql.AppendLine("("); - - var i = 0; - foreach (var column in columns) - { - if (i > 0) - { - sql.Append(", "); - } - - sql.Append(column); - i++; - } - + sql.AppendJoin(", ", columns); sql.AppendLine(")"); } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/TableMetadata.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/TableMetadata.cs index b733345..e6d9574 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/TableMetadata.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/TableMetadata.cs @@ -8,16 +8,20 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Metadata; internal sealed class TableMetadata(IEntityType entityType, SqlDialectBuilder dialect) { private IReadOnlyList? _notGeneratedProperties; - - public string TableName { get; } = - entityType.GetTableName() ?? throw new InvalidOperationException("Canot determine table name."); + private IReadOnlyList? _primaryKeys; public string QuotedTableName { get; } = dialect.QuoteTableName(entityType.GetSchema(), entityType.GetTableName()!); + public string TableName { get; } = + entityType.GetTableName() ?? throw new InvalidOperationException("Canot determine table name."); + public IReadOnlyList Properties { get; } = entityType.GetProperties().Where(p => !p.IsShadowProperty()).Select(x => new PropertyMetadata(x, dialect)).ToList(); + public IReadOnlyList PrimaryKey => + _primaryKeys ??= GetPrimaryKey(); + public IReadOnlyList GetProperties(bool includeGenerated = true) { if (includeGenerated) @@ -43,4 +47,11 @@ public string GetColumnName(string propertyName) return property.ColumnName; } + + private List GetPrimaryKey() + { + var primaryKey = entityType.FindPrimaryKey()?.Properties ?? []; + + return Properties.Where(x => primaryKey.Any(y => x.Name == y.Name)).ToList(); + } } From e2e8f34603a766936dbc6e9d5fa3890538923778 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 11:08:19 +0200 Subject: [PATCH 02/13] Add missing files. --- .../Helpers.cs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/PhenX.EntityFrameworkCore.BulkInsert/Helpers.cs diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Helpers.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Helpers.cs new file mode 100644 index 0000000..71fd898 --- /dev/null +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Helpers.cs @@ -0,0 +1,30 @@ +using System.Text; + +using PhenX.EntityFrameworkCore.BulkInsert.Metadata; + +namespace PhenX.EntityFrameworkCore.BulkInsert; + +internal static class Helpers +{ + public static StringBuilder AppendJoin(this StringBuilder sb, string separator, IEnumerable items, Action formatter) + { + var first = true; + foreach (var item in items) + { + if (!first) + { + sb.Append(separator); + } + + formatter(sb, item); + first = false; + } + + return sb; + } + + public static StringBuilder AppendColumns(this StringBuilder sb, IReadOnlyList columns) + { + return sb.AppendJoin(", ", columns.Select(c => c.QuotedColumName)); + } +} From 0366de20b5d1eeeb8a39f6d8d921464f592927ff Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 11:53:09 +0200 Subject: [PATCH 03/13] Fixes --- .../PostgreSqlBulkInsertProvider.cs | 8 ++--- .../PostgreSqlDialectBuilder.cs | 25 ++++++++++++++- .../SqlServerBulkInsertProvider.cs | 17 ++++++++-- .../SqlServerDialectBuilder.cs | 2 -- .../Dialect/SqlDialectBuilder.cs | 21 ++++++------ .../Metadata/PropertyMetadata.cs | 11 +++++++ .../DbContext/TestDbContext.cs | 9 +++++- .../DbContext/TestEntity.cs | 1 - .../Tests/Basic/BasicTestsBase.cs | 32 ++++++++++++++++++- 9 files changed, 102 insertions(+), 24 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs index be61cb7..cdc450a 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs @@ -29,13 +29,11 @@ protected override string CreateTableCopySql(string tempNameName, TableMetadata return $"CREATE TEMPORARY TABLE {tempNameName} AS TABLE {tableInfo.QuotedTableName} WITH NO DATA;"; } - private static string GetBinaryImportCommand(TableMetadata tableInfo, string tableName) + private static string GetBinaryImportCommand(IReadOnlyList properties, string tableName) { - var columns = tableInfo.GetProperties(false).Select(X => X.QuotedColumName); - var sql = new StringBuilder(); sql.Append($"COPY {tableName} ("); - sql.AppendColumns(tableInfo.GetProperties(false)); + sql.AppendColumns(properties); sql.Append(") FROM STDIN (FORMAT BINARY)"); return sql.ToString(); } @@ -53,7 +51,7 @@ protected override async Task BulkInsert( { var connection = (NpgsqlConnection)context.Database.GetDbConnection(); - var importCommand = GetBinaryImportCommand(tableInfo, tableName); + var importCommand = GetBinaryImportCommand(properties, tableName); var writer = sync // ReSharper disable once MethodHasAsyncOverloadWithCancellation diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs index 7f9177d..34b016e 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs @@ -1,4 +1,8 @@ -using PhenX.EntityFrameworkCore.BulkInsert.Dialect; +using System.Text; + +using PhenX.EntityFrameworkCore.BulkInsert.Dialect; +using PhenX.EntityFrameworkCore.BulkInsert.Metadata; +using PhenX.EntityFrameworkCore.BulkInsert.Options; namespace PhenX.EntityFrameworkCore.BulkInsert.PostgreSql; @@ -6,4 +10,23 @@ internal class PostgreSqlDialectBuilder : SqlDialectBuilder { protected override string OpenDelimiter => "\""; protected override string CloseDelimiter => "\""; + + protected override void AppendConflictMatch(StringBuilder sql, TableMetadata target, OnConflictOptions conflict) + { + if (conflict.Match != null) + { + base.AppendConflictMatch(sql, target, conflict); + } + else if (target.PrimaryKey.Count > 0) + { + sql.Append(' '); + sql.AppendLine("("); + sql.AppendColumns(target.PrimaryKey); + sql.AppendLine(")"); + } + else + { + throw new InvalidOperationException("Table has no primary key that can be used for conflict detection."); + } + } } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs index 84af4a8..34ecf87 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs @@ -29,9 +29,20 @@ public SqlServerBulkInsertProvider(ILogger? logger protected override string CreateTableCopySql(string templNameName, TableMetadata tableInfo, IReadOnlyList columns) { var sb = new StringBuilder(); - sb.Append("SELECT"); - sb.AppendJoin(", ", columns.Select(x => x.QuotedColumName)); - sb.Append($"INTO {templNameName} FROM {tableInfo.QuotedTableName} WHERE 1 = 0;"); + sb.Append($"CREATE TABLE {templNameName}"); + sb.AppendLine("("); + + foreach (var column in columns) + { + sb.Append($" {column.QuotedColumName} {column.StoreDefinition}"); + if (column != columns[^1]) + { + sb.Append(','); + } + sb.AppendLine(); + } + + sb.AppendLine(")"); return sb.ToString(); } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs index 640901b..caf729f 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs @@ -1,7 +1,5 @@ using System.Text; -using Microsoft.Extensions.Primitives; - using PhenX.EntityFrameworkCore.BulkInsert.Dialect; using PhenX.EntityFrameworkCore.BulkInsert.Metadata; using PhenX.EntityFrameworkCore.BulkInsert.Options; diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs index 9e5aa69..baba7d4 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs @@ -59,11 +59,7 @@ public virtual string BuildMoveDataSql( if (onConflictTyped.Update != null) { - if (onConflictTyped.Match != null) - { - q.Append(' '); - AppendConflictMatch(q, GetColumns(target, onConflictTyped.Match)); - } + AppendConflictMatch(q, target, onConflictTyped); if (onConflictTyped.Update != null) { @@ -93,7 +89,8 @@ public virtual string BuildMoveDataSql( q.AppendLine(";"); - return q.ToString(); + var result = q.ToString(); + return result; } protected virtual void AppendDoNothing(StringBuilder sql, IEnumerable insertedProperties) @@ -107,11 +104,15 @@ protected virtual void AppendOnConflictUpdate(StringBuilder sql, IEnumerable columns) + protected virtual void AppendConflictMatch(StringBuilder sql, TableMetadata target, OnConflictOptions conflict) { - sql.AppendLine("("); - sql.AppendJoin(", ", columns); - sql.AppendLine(")"); + if (conflict.Match != null) + { + sql.Append(' '); + sql.AppendLine("("); + sql.AppendJoin(", ", GetColumns(target, conflict.Match)); + sql.AppendLine(")"); + } } protected virtual void AppendOnConflictStatement(StringBuilder sql) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/PropertyMetadata.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/PropertyMetadata.cs index 14d9786..33e593b 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/PropertyMetadata.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/PropertyMetadata.cs @@ -15,6 +15,8 @@ internal sealed class PropertyMetadata(IProperty property, SqlDialectBuilder di public string QuotedColumName { get; } = dialect.Quote(property.GetColumnName()); + public string StoreDefinition { get; } = GetStoreDefinition(property); + public Type ClrType { get; } = property.ClrType; public Type? ProviderClrType { get; } = property.GetProviderClrType(); @@ -54,6 +56,15 @@ internal sealed class PropertyMetadata(IProperty property, SqlDialectBuilder di return result; } + private static string GetStoreDefinition(IProperty property) + { + var typeMapping = property.GetRelationalTypeMapping(); + + var nullability = property.IsNullable ? "NULL" : "NOT NULL"; + + return $"{typeMapping.StoreType} {nullability}"; + } + public override string ToString() { return $"Name: {Name}, Column: {ColumnName}"; diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs index fdb6093..530d42e 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; @@ -6,6 +6,7 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; public class TestDbContext : TestDbContextBase { public DbSet TestEntities { get; set; } = null!; + public DbSet TestEntitiesWithGuidIds { get; set; } = null!; public DbSet TestEntitiesWithConverters { get; set; } = null!; protected override void OnModelCreating(ModelBuilder modelBuilder) @@ -16,5 +17,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) builder.Property(e => e.CreatedAt) .HasConversion(new DateTimeToBinaryConverter()); }); + + modelBuilder.Entity(builder => + { + builder.Property(e => e.Id) + .ValueGeneratedNever(); + }); } } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs index f8cb721..361e3d2 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntity.cs @@ -10,7 +10,6 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; [Table("test_entity")] public class TestEntity { - public int Id { get; set; } [Column("name")] diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs index 62f4bc2..62f45de 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs @@ -147,10 +147,40 @@ await _context.ExecuteBulkInsertAsync(entities, Assert.Contains(insertedEntities, e => e.NumericEnumValue == NumericEnum.Second); } + [SkippableFact] + public async Task InsertsEntities_MultipleTimes_WithGuidId() + { + // Arrange + var entities = new List + { + new TestEntityWithGuidId { Id = Guid.NewGuid(), TestRun = _run, Name = $"{_run}_Entity1" }, + new TestEntityWithGuidId { Id = Guid.NewGuid(), TestRun = _run, Name = $"{_run}_Entity2" } + }; + + // Act + await _context.ExecuteBulkInsertAsync(entities); + + foreach (var entity in entities) + { + entity.Name = $"Updated_{entity.Name}"; + } + + await _context.ExecuteBulkInsertAsync(entities, + onConflict: new OnConflictOptions + { + Update = e => e, + }); + + // Assert + var insertedEntities = _context.TestEntitiesWithGuidIds.Where(x => x.TestRun == _run).ToList(); + Assert.Equal(2, insertedEntities.Count); + Assert.Contains(insertedEntities, e => e.Name == $"Updated_{_run}_Entity1"); + Assert.Contains(insertedEntities, e => e.Name == $"Updated_{_run}_Entity2"); + } + [SkippableFact] public async Task InsertsEntities_MultipleTimes_With_Conflict_On_Id() { - Skip.If(_context.Database.ProviderName!.Contains("Postgres", StringComparison.InvariantCultureIgnoreCase)); Skip.If(_context.Database.ProviderName!.Contains("SqlServer", StringComparison.InvariantCultureIgnoreCase)); // Arrange From ff9eb1a0c19f1b824fd8e265651e654a65bb7d19 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 11:54:55 +0200 Subject: [PATCH 04/13] Add missing file. --- .../DbContext/TestEntityWithGuidId.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs new file mode 100644 index 0000000..0bcd218 --- /dev/null +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestEntityWithGuidId.cs @@ -0,0 +1,20 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +using Microsoft.EntityFrameworkCore; + +namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext; + +[PrimaryKey(nameof(Id))] +[Table("test_entity_guids")] +public class TestEntityWithGuidId +{ + public Guid Id { get; set; } + + [Column("name")] + [MaxLength(100)] + public string Name { get; set; } = string.Empty; + + [Column("test_run")] + public Guid TestRun { get; set; } +} From 93738de922a0686c6f48caf93d771e3fb1501894 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 11:58:32 +0200 Subject: [PATCH 05/13] Make OnConflictOptions type safe. --- .../Extensions/DbSetExtensions.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbSetExtensions.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbSetExtensions.cs index 570c085..6c17bf2 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbSetExtensions.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/DbSetExtensions.cs @@ -18,7 +18,7 @@ public static async Task> ExecuteBulkInsertReturnEntitiesAsync( this DbSet dbSet, IEnumerable entities, Action? configure = null, - OnConflictOptions? onConflict = null, + OnConflictOptions? onConflict = null, CancellationToken ctk = default ) where T : class { @@ -31,7 +31,7 @@ public static async Task> ExecuteBulkInsertReturnEntitiesAsync( /// /// Executes a bulk insert operation returning the inserted/updated entities, from the DbContext. /// - public static async Task> ExecuteBulkInsertReturnEntitiesAsync(this DbContext dbContext, IEnumerable entities, Action? configure = null, OnConflictOptions? onConflict = null, CancellationToken cancellationToken = default) where T : class + public static async Task> ExecuteBulkInsertReturnEntitiesAsync(this DbContext dbContext, IEnumerable entities, Action? configure = null, OnConflictOptions? onConflict = null, CancellationToken cancellationToken = default) where T : class { var dbSet = dbContext.Set(); if (dbSet == null) @@ -49,7 +49,7 @@ public static async Task ExecuteBulkInsertAsync( this DbSet dbSet, IEnumerable entities, Action? configure = null, - OnConflictOptions? onConflict = null, + OnConflictOptions? onConflict = null, CancellationToken ctk = default ) where T : class { @@ -62,7 +62,7 @@ public static async Task ExecuteBulkInsertAsync( /// /// Executes a bulk insert operation without returning the inserted/updated entities, from the DbContext. /// - public static async Task ExecuteBulkInsertAsync(this DbContext dbContext, IEnumerable entities, Action? configure = null, OnConflictOptions? onConflict = null, CancellationToken cancellationToken = default) where T : class + public static async Task ExecuteBulkInsertAsync(this DbContext dbContext, IEnumerable entities, Action? configure = null, OnConflictOptions? onConflict = null, CancellationToken cancellationToken = default) where T : class { var dbSet = dbContext.Set(); if (dbSet == null) @@ -80,7 +80,7 @@ public static List ExecuteBulkInsertReturnEntities( this DbSet dbSet, IEnumerable entities, Action? configure = null, - OnConflictOptions? onConflict = null + OnConflictOptions? onConflict = null ) where T : class { var provider = InitProvider(dbSet, configure, out var context, out var options); @@ -96,7 +96,7 @@ public static List ExecuteBulkInsertReturnEntities( this DbContext dbContext, IEnumerable entities, Action? configure = null, - OnConflictOptions? onConflict = null + OnConflictOptions? onConflict = null ) where T : class { var dbSet = dbContext.Set(); @@ -115,7 +115,7 @@ public static void ExecuteBulkInsert( this DbSet dbSet, IEnumerable entities, Action? configure = null, - OnConflictOptions? onConflict = null + OnConflictOptions? onConflict = null ) where T : class { var provider = InitProvider(dbSet, configure, out var context, out var options); @@ -131,7 +131,7 @@ public static void ExecuteBulkInsert( this DbContext dbContext, IEnumerable entities, Action? configure = null, - OnConflictOptions? onConflict = null + OnConflictOptions? onConflict = null ) where T : class { var dbSet = dbContext.Set(); From 031395de31bdce67cf29313572e9d12c3a73d18b Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 16:09:28 +0200 Subject: [PATCH 06/13] Move stuff to dialect. --- .../MySqlBulkInsertProvider.cs | 5 ----- .../MySqlDialectBuilder.cs | 5 +++++ .../PostgreSqlBulkInsertProvider.cs | 6 ------ .../PostgreSqlDialectBuilder.cs | 5 +++++ .../SqlServerBulkInsertProvider.cs | 21 ------------------- .../SqlServerDialectBuilder.cs | 10 +++++++++ .../SqliteBulkInsertProvider.cs | 6 ------ .../SqliteDialectBuilder.cs | 9 +++++++- .../BulkInsertProviderBase.cs | 4 +--- .../Dialect/SqlDialectBuilder.cs | 2 ++ 10 files changed, 31 insertions(+), 42 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs index 8ac6832..f4b3190 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs @@ -22,11 +22,6 @@ public MySqlBulkInsertProvider(ILogger? logger = null) /// protected override string GetTempTableName(string tableName) => $"#_temp_bulk_insert_{tableName}"; - protected override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) - { - return $"CREATE TEMPORARY TABLE {tempNameName} SELECT * FROM {tableInfo.QuotedTableName} WHERE 1 = 0;"; - } - /// public override Task> BulkInsertReturnEntities( bool sync, diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDialectBuilder.cs index 9ada13b..f25ee09 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlDialectBuilder.cs @@ -14,6 +14,11 @@ internal class MySqlServerDialectBuilder : SqlDialectBuilder protected override bool SupportsMoveRows => false; + public override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + return $"CREATE TEMPORARY TABLE {tempNameName} SELECT * FROM {tableInfo.QuotedTableName} WHERE 1 = 0;"; + } + protected override void AppendConflictCondition(StringBuilder sql, OnConflictOptions onConflictTyped) { throw new NotSupportedException("Conflict conditions are not supported in MYSQL"); diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs index cdc450a..bd07d14 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs @@ -23,12 +23,6 @@ public PostgreSqlBulkInsertProvider(ILogger? logge /// protected override string AddTableCopyBulkInsertId => $"ALTER TABLE {{0}} ADD COLUMN {BulkInsertId} SERIAL PRIMARY KEY;"; - /// - protected override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) - { - return $"CREATE TEMPORARY TABLE {tempNameName} AS TABLE {tableInfo.QuotedTableName} WITH NO DATA;"; - } - private static string GetBinaryImportCommand(IReadOnlyList properties, string tableName) { var sql = new StringBuilder(); diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs index 34b016e..e4d69c0 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlDialectBuilder.cs @@ -11,6 +11,11 @@ internal class PostgreSqlDialectBuilder : SqlDialectBuilder protected override string OpenDelimiter => "\""; protected override string CloseDelimiter => "\""; + public override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + return $"CREATE TEMPORARY TABLE {tempNameName} AS TABLE {tableInfo.QuotedTableName} WITH NO DATA;"; + } + protected override void AppendConflictMatch(StringBuilder sql, TableMetadata target, OnConflictOptions conflict) { if (conflict.Match != null) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs index 34ecf87..0550b9d 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs @@ -26,27 +26,6 @@ public SqlServerBulkInsertProvider(ILogger? logger /// protected override string GetTempTableName(string tableName) => $"#_temp_bulk_insert_{tableName}"; - protected override string CreateTableCopySql(string templNameName, TableMetadata tableInfo, IReadOnlyList columns) - { - var sb = new StringBuilder(); - sb.Append($"CREATE TABLE {templNameName}"); - sb.AppendLine("("); - - foreach (var column in columns) - { - sb.Append($" {column.QuotedColumName} {column.StoreDefinition}"); - if (column != columns[^1]) - { - sb.Append(','); - } - sb.AppendLine(); - } - - sb.AppendLine(")"); - - return sb.ToString(); - } - /// protected override async Task BulkInsert( bool sync, diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs index caf729f..5fc99b5 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs @@ -14,6 +14,16 @@ internal class SqlServerDialectBuilder : SqlDialectBuilder protected override bool SupportsMoveRows => false; + public override string CreateTableCopySql(string templNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + var q = new StringBuilder(); + q.Append("SELECT"); + q.AppendColumns(columns); + q.Append($"INTO {templNameName} FROM {tableInfo.QuotedTableName} WHERE 1 = 0;"); + + return q.ToString(); + } + public override string BuildMoveDataSql( TableMetadata target, string source, diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs index bf2ca7e..2de65fe 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs @@ -27,12 +27,6 @@ public SqliteBulkInsertProvider(ILogger? logger = null /// protected override string AddTableCopyBulkInsertId => "--"; // No need to add an ID column in SQLite - /// - protected override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) - { - return $"CREATE TEMP TABLE {tempNameName} AS SELECT * FROM {tableInfo.QuotedTableName} WHERE 0;"; - } - /// protected override Task AddBulkInsertIdColumn( bool sync, diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteDialectBuilder.cs index 232ee0a..fa74ead 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteDialectBuilder.cs @@ -1,4 +1,5 @@ -using PhenX.EntityFrameworkCore.BulkInsert.Dialect; +using PhenX.EntityFrameworkCore.BulkInsert.Dialect; +using PhenX.EntityFrameworkCore.BulkInsert.Metadata; namespace PhenX.EntityFrameworkCore.BulkInsert.Sqlite; @@ -8,4 +9,10 @@ internal class SqliteDialectBuilder : SqlDialectBuilder protected override string CloseDelimiter => "\""; protected override bool SupportsMoveRows => false; + + /// + public override string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns) + { + return $"CREATE TEMP TABLE {tempNameName} AS SELECT * FROM {tableInfo.QuotedTableName} WHERE 0;"; + } } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs index 65fce87..3cb7d81 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs @@ -36,7 +36,7 @@ protected async Task CreateTableCopyAsync( var tempTableName = SqlDialect.QuoteTableName(null, GetTempTableName(tableInfo.TableName)); var tempColumns = tableInfo.GetProperties(options.CopyGeneratedColumns); - var query = CreateTableCopySql(tempTableName, tableInfo, tempColumns); + var query = SqlDialect.CreateTableCopySql(tempTableName, tableInfo, tempColumns); await ExecuteAsync(sync, context, query, cancellationToken); await AddBulkInsertIdColumn(sync, context, tempTableName, cancellationToken); @@ -44,8 +44,6 @@ protected async Task CreateTableCopyAsync( return tempTableName; } - protected abstract string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns); - protected virtual async Task AddBulkInsertIdColumn(bool sync, DbContext context, string tempTableName, CancellationToken cancellationToken) where T : class { diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs index baba7d4..58c4428 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs @@ -14,6 +14,8 @@ internal abstract class SqlDialectBuilder protected virtual string ConcatOperator => "||"; protected virtual bool SupportsMoveRows => true; + public abstract string CreateTableCopySql(string tempNameName, TableMetadata tableInfo, IReadOnlyList columns); + /// /// Builds the SQL for moving data from one table to another. /// From c86d2407d88c42c6e69cc673c31d6f6ee07ad934 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 16:14:28 +0200 Subject: [PATCH 07/13] Cleanup --- .../BulkInsertProviderBase.cs | 123 ++++++------------ .../Extensions/ConnectionInfo.cs | 48 ++++++- 2 files changed, 84 insertions(+), 87 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs index 3cb7d81..b670418 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs @@ -24,6 +24,8 @@ internal abstract class BulkInsertProviderBase(ILogger $"_temp_bulk_insert_{tableName}"; + SqlDialectBuilder IBulkInsertProvider.SqlDialect => SqlDialect; protected async Task CreateTableCopyAsync( @@ -31,32 +33,35 @@ protected async Task CreateTableCopyAsync( DbContext context, BulkInsertOptions options, TableMetadata tableInfo, - CancellationToken cancellationToken = default) where T : class + CancellationToken ctk) where T : class { var tempTableName = SqlDialect.QuoteTableName(null, GetTempTableName(tableInfo.TableName)); var tempColumns = tableInfo.GetProperties(options.CopyGeneratedColumns); var query = SqlDialect.CreateTableCopySql(tempTableName, tableInfo, tempColumns); - await ExecuteAsync(sync, context, query, cancellationToken); - await AddBulkInsertIdColumn(sync, context, tempTableName, cancellationToken); + await ExecuteAsync(sync, context, query, ctk); + await AddBulkInsertIdColumn(sync, context, tempTableName, ctk); return tempTableName; } - protected virtual async Task AddBulkInsertIdColumn(bool sync, DbContext context, - string tempTableName, CancellationToken cancellationToken) where T : class + protected virtual async Task AddBulkInsertIdColumn( + bool sync, + DbContext context, + string tempTableName, + CancellationToken ctk) where T : class { var alterQuery = string.Format(AddTableCopyBulkInsertId, tempTableName); - await ExecuteAsync(sync, context, alterQuery, cancellationToken); + await ExecuteAsync(sync, context, alterQuery, ctk); } - protected virtual string GetTempTableName(string tableName) => $"_temp_bulk_insert_{tableName}"; - - protected string Quote(string name) => SqlDialect.Quote(name); - - protected static async Task ExecuteAsync(bool sync, DbContext context, string query, CancellationToken cancellationToken = default) + protected static async Task ExecuteAsync( + bool sync, + DbContext context, + string query, + CancellationToken ctk) { var command = context.Database.GetDbConnection().CreateCommand(); command.Transaction = context.Database.CurrentTransaction!.GetDbTransaction(); @@ -69,7 +74,7 @@ protected static async Task ExecuteAsync(bool sync, DbContext context, string qu } else { - await command.ExecuteNonQueryAsync(cancellationToken); + await command.ExecuteNonQueryAsync(ctk); } } @@ -80,8 +85,8 @@ public async Task> CopyFromTempTableAsync( string tempTableName, bool returnData, BulkInsertOptions options, - OnConflictOptions? onConflict = null, - CancellationToken cancellationToken = default) where T : class + OnConflictOptions? onConflict, + CancellationToken ctk) where T : class { return await CopyFromTempTableWithoutKeysAsync( sync, @@ -91,7 +96,7 @@ public async Task> CopyFromTempTableAsync( returnData, options, onConflict, - cancellationToken: cancellationToken); + ctk); } private async Task> CopyFromTempTableWithoutKeysAsync( @@ -101,10 +106,8 @@ private async Task> CopyFromTempTableWithoutKeysAsync( string tempTableName, bool returnData, BulkInsertOptions options, - OnConflictOptions? onConflict = null, - CancellationToken cancellationToken = default) - where T : class - where TResult : class + OnConflictOptions? onConflict, + CancellationToken ctk) where T : class where TResult : class { var query = SqlDialect.BuildMoveDataSql( @@ -117,11 +120,11 @@ private async Task> CopyFromTempTableWithoutKeysAsync( if (returnData) { - return await QueryAsync(sync, context, query, cancellationToken); + return await QueryAsync(sync, context, query, ctk); } // If not returning data, just execute the command - await ExecuteAsync(sync, context, query, cancellationToken); + await ExecuteAsync(sync, context, query, ctk); return []; static async Task> QueryAsync(bool sync, DbContext context, string query, CancellationToken cancellationToken) @@ -146,9 +149,8 @@ public virtual async Task> BulkInsertReturnEntities( TableMetadata tableInfo, IEnumerable entities, BulkInsertOptions options, - OnConflictOptions? onConflict = null, - CancellationToken ctk = default - ) where T : class + OnConflictOptions? onConflict, + CancellationToken ctk) where T : class { List result; @@ -157,77 +159,27 @@ public virtual async Task> BulkInsertReturnEntities( { var (tableName, _) = await PerformBulkInsertAsync(sync, context, tableInfo, entities, options, tempTableRequired: true, ctk: ctk); - result = await CopyFromTempTableAsync(sync, context, tableInfo, tableName, true, options, onConflict, cancellationToken: ctk); + result = await CopyFromTempTableAsync(sync, context, tableInfo, tableName, true, options, onConflict, ctk: ctk); // Commit the transaction if we own them. - await Commit(sync, connectionInfo, ctk); + await connectionInfo.Commit(sync, ctk); } finally { - await Finish(sync, connectionInfo, ctk); + await connectionInfo.Close(sync, ctk); } return result; } - private static async Task Commit(bool sync, ConnectionInfo connectionInfo, CancellationToken ctk) - { - var (_, _, transaction, wasBegan) = connectionInfo; - - if (!wasBegan) - { - if (sync) - { - // ReSharper disable once MethodHasAsyncOverloadWithCancellation - transaction.Commit(); - } - else - { - await transaction.CommitAsync(ctk); - } - } - } - - private static async Task Finish(bool sync, ConnectionInfo connectionInfo, CancellationToken ctk) - { - var (connection, wasClosed, transaction, wasBegan) = connectionInfo; - - if (!wasBegan) - { - if (sync) - { - // ReSharper disable once MethodHasAsyncOverloadWithCancellation - transaction.Dispose(); - } - else - { - await transaction.DisposeAsync(); - } - } - - if (wasClosed) - { - if (sync) - { - // ReSharper disable once MethodHasAsyncOverload - connection.Close(); - } - else - { - await connection.CloseAsync(); - } - } - } - public virtual async Task BulkInsert( bool sync, DbContext context, TableMetadata tableInfo, IEnumerable entities, BulkInsertOptions options, - OnConflictOptions? onConflict = null, - CancellationToken ctk = default - ) where T : class + OnConflictOptions? onConflict, + CancellationToken ctk) where T : class { if (onConflict != null) { @@ -239,11 +191,11 @@ public virtual async Task BulkInsert( await CopyFromTempTableAsync(sync, context, tableInfo, tableName, false, options, onConflict, ctk); // Commit the transaction if we own them. - await Commit(sync, connectionInfo, ctk); + await connectionInfo.Commit(sync, ctk); } finally { - await Finish(sync, connectionInfo, ctk); + await connectionInfo.Close(sync, ctk); } } else @@ -259,7 +211,7 @@ public virtual async Task BulkInsert( IEnumerable entities, BulkInsertOptions options, bool tempTableRequired, - CancellationToken ctk = default) where T : class + CancellationToken ctk) where T : class { if (entities.TryGetNonEnumeratedCount(out var count) && count == 0) { @@ -279,11 +231,11 @@ public virtual async Task BulkInsert( await BulkInsert(false, context, tableInfo, entities, tableName, properties, options, ctk); // Commit the transaction if we own them. - await Commit(sync, connectionInfo, ctk); + await connectionInfo.Commit(sync, ctk); } finally { - await Finish(sync, connectionInfo, ctk); + await connectionInfo.Close(sync, ctk); } return (tableName, connectionInfo.Connection); @@ -300,6 +252,5 @@ protected abstract Task BulkInsert( string tableName, IReadOnlyList properties, BulkInsertOptions options, - CancellationToken ctk - ) where T : class; + CancellationToken ctk) where T : class; } diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/ConnectionInfo.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/ConnectionInfo.cs index 2d8df8b..33ecc9b 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/ConnectionInfo.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Extensions/ConnectionInfo.cs @@ -4,4 +4,50 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Extensions; -internal readonly record struct ConnectionInfo(DbConnection Connection, bool WasClosed, IDbContextTransaction Transaction, bool WasBegan); +internal readonly record struct ConnectionInfo(DbConnection Connection, bool WasClosed, IDbContextTransaction Transaction, bool WasBegan) +{ + public async Task Commit(bool sync, CancellationToken ctk) + { + if (!WasBegan) + { + if (sync) + { + // ReSharper disable once MethodHasAsyncOverloadWithCancellation + Transaction.Commit(); + } + else + { + await Transaction.CommitAsync(ctk); + } + } + } + + public async Task Close(bool sync, CancellationToken ctk) + { + if (!WasBegan) + { + if (sync) + { + // ReSharper disable once MethodHasAsyncOverloadWithCancellation + Transaction.Dispose(); + } + else + { + await Transaction.DisposeAsync(); + } + } + + if (WasClosed) + { + if (sync) + { + // ReSharper disable once MethodHasAsyncOverload + Connection.Close(); + } + else + { + await Connection.CloseAsync(); + } + } + } +} From 934bec1fb2dac0e39f3334f53f4bd66ee970b95a Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 16:15:48 +0200 Subject: [PATCH 08/13] Cleanup. --- .../SqlServerBulkInsertProvider.cs | 2 -- .../SqliteBulkInsertProvider.cs | 1 - .../BulkInsertProviderBase.cs | 1 - 3 files changed, 4 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs index 0550b9d..8d93176 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs @@ -1,5 +1,3 @@ -using System.Text; - using JetBrains.Annotations; using Microsoft.Data.SqlClient; diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs index 2de65fe..447f23b 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs @@ -6,7 +6,6 @@ using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Primitives; using PhenX.EntityFrameworkCore.BulkInsert.Metadata; using PhenX.EntityFrameworkCore.BulkInsert.Options; diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs index b670418..3ae3c89 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs @@ -1,5 +1,4 @@ using System.Data.Common; -using System.Reflection; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage; From e8797c5cf4babb8ecfd898cb98da421e11bb5bb8 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 16:16:33 +0200 Subject: [PATCH 09/13] Rename connectionInfo. --- .../BulkInsertProviderBase.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs index 3ae3c89..d731913 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertProviderBase.cs @@ -153,7 +153,7 @@ public virtual async Task> BulkInsertReturnEntities( { List result; - var connectionInfo = await context.GetConnection(sync, ctk); + var connection = await context.GetConnection(sync, ctk); try { var (tableName, _) = await PerformBulkInsertAsync(sync, context, tableInfo, entities, options, tempTableRequired: true, ctk: ctk); @@ -161,11 +161,11 @@ public virtual async Task> BulkInsertReturnEntities( result = await CopyFromTempTableAsync(sync, context, tableInfo, tableName, true, options, onConflict, ctk: ctk); // Commit the transaction if we own them. - await connectionInfo.Commit(sync, ctk); + await connection.Commit(sync, ctk); } finally { - await connectionInfo.Close(sync, ctk); + await connection.Close(sync, ctk); } return result; @@ -182,7 +182,7 @@ public virtual async Task BulkInsert( { if (onConflict != null) { - var connectionInfo = await context.GetConnection(sync, ctk); + var connection = await context.GetConnection(sync, ctk); try { var (tableName, _) = await PerformBulkInsertAsync(sync, context, tableInfo, entities, options, tempTableRequired: true, ctk: ctk); @@ -190,11 +190,11 @@ public virtual async Task BulkInsert( await CopyFromTempTableAsync(sync, context, tableInfo, tableName, false, options, onConflict, ctk); // Commit the transaction if we own them. - await connectionInfo.Commit(sync, ctk); + await connection.Commit(sync, ctk); } finally { - await connectionInfo.Close(sync, ctk); + await connection.Close(sync, ctk); } } else @@ -217,7 +217,7 @@ public virtual async Task BulkInsert( throw new InvalidOperationException("No entities to insert."); } - var connectionInfo = await context.GetConnection(sync, ctk); + var connection = await context.GetConnection(sync, ctk); var tableName = tempTableRequired ? await CreateTableCopyAsync(sync, context, options, tableInfo, ctk) @@ -230,14 +230,14 @@ public virtual async Task BulkInsert( await BulkInsert(false, context, tableInfo, entities, tableName, properties, options, ctk); // Commit the transaction if we own them. - await connectionInfo.Commit(sync, ctk); + await connection.Commit(sync, ctk); } finally { - await connectionInfo.Close(sync, ctk); + await connection.Close(sync, ctk); } - return (tableName, connectionInfo.Connection); + return (tableName, connection.Connection); } /// From dd0a6c5b9c3f055787626a844d67da5c4f85a480 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 18:35:03 +0200 Subject: [PATCH 10/13] Rename property. --- .../SqlServerDialectBuilder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs index 5fc99b5..2b07ab7 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs @@ -119,8 +119,8 @@ public override string BuildMoveDataSql( q.AppendLine($"SET IDENTITY_INSERT {target.QuotedTableName} OFF;"); } - var x = q.ToString(); - return x; + var result = q.ToString(); + return result; } protected override string GetExcludedColumnName(string columnName) From d16e579731a7c8ac88d7b89de866e0b51c628a04 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 18:45:43 +0200 Subject: [PATCH 11/13] Fix temp table. --- .../SqlServerDialectBuilder.cs | 22 ++++++++++++++----- .../Tests/Basic/BasicTestsBase.cs | 2 -- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs index 2b07ab7..ef5c4be 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerDialectBuilder.cs @@ -14,12 +14,22 @@ internal class SqlServerDialectBuilder : SqlDialectBuilder protected override bool SupportsMoveRows => false; - public override string CreateTableCopySql(string templNameName, TableMetadata tableInfo, IReadOnlyList columns) + public override string CreateTableCopySql(string tempTableName, TableMetadata tableInfo, IReadOnlyList columns) { var q = new StringBuilder(); - q.Append("SELECT"); - q.AppendColumns(columns); - q.Append($"INTO {templNameName} FROM {tableInfo.QuotedTableName} WHERE 1 = 0;"); + q.Append($"CREATE TABLE {tempTableName} ("); + + foreach (var column in columns) + { + q.Append($"{column.QuotedColumName} {column.StoreDefinition}"); + if (column != columns[^1]) + { + q.Append(','); + } + q.AppendLine(); + } + + q.AppendLine(")"); return q.ToString(); } @@ -70,8 +80,10 @@ public override string BuildMoveDataSql( if (onConflictTyped.Update != null) { + var properties = target.GetProperties(false); + q.AppendLine($"WHEN MATCHED THEN UPDATE SET "); - q.AppendJoin(", ", GetUpdates(target, insertedProperties, onConflictTyped.Update)); + q.AppendJoin(", ", GetUpdates(target, properties, onConflictTyped.Update)); q.AppendLine(); } diff --git a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs index 62f45de..5db1690 100644 --- a/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs +++ b/tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/Tests/Basic/BasicTestsBase.cs @@ -181,8 +181,6 @@ await _context.ExecuteBulkInsertAsync(entities, [SkippableFact] public async Task InsertsEntities_MultipleTimes_With_Conflict_On_Id() { - Skip.If(_context.Database.ProviderName!.Contains("SqlServer", StringComparison.InvariantCultureIgnoreCase)); - // Arrange var entities = new List { From 846cb56045b0aa6a2bd09b84c14e5b7793e7b82b Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 20:39:17 +0200 Subject: [PATCH 12/13] Just some minor performance improvements. --- .../Dialect/SqlDialectBuilder.cs | 148 ++++++++++-------- 1 file changed, 79 insertions(+), 69 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs index 58c4428..50f0ae1 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs @@ -138,16 +138,17 @@ protected virtual string GetExcludedColumnName(string columnName) /// /// Quotes a column name using database-specific delimiters. /// - public string Quote(string entity) => $"{OpenDelimiter}{entity}{CloseDelimiter}"; + public string Quote(string entity) + { + return $"{OpenDelimiter}{entity}{CloseDelimiter}"; + } /// /// Quotes a schema and table name using database-specific delimiters. /// public string QuoteTableName(string? schema, string tableName) { - return schema != null - ? $"{Quote(schema)}.{Quote(tableName)}" - : Quote(tableName); + return schema != null ? $"{Quote(schema)}.{Quote(tableName)}" : Quote(tableName); } /// @@ -157,12 +158,11 @@ protected string[] GetColumns(TableMetadata table, Expression { return columns.Body switch { - NewExpression newExpression => newExpression.Arguments.OfType() - .Select(m => table.GetQuotedColumnName(m.Member.Name)) - .ToArray(), - MemberExpression memberExpression => [ - table.GetQuotedColumnName(memberExpression.Member.Name) - ], + NewExpression newExpression => + newExpression.Arguments.OfType() + .Select(m => table.GetQuotedColumnName(m.Member.Name)).ToArray(), + MemberExpression memberExpression => + [table.GetQuotedColumnName(memberExpression.Member.Name)], _ => throw new NotSupportedException("Unsupported expression type") }; } @@ -228,85 +228,95 @@ private string ToSqlExpression(TableMetadata table, Expression expr) { switch (expr) { - case MemberExpression m: - return GetExcludedColumnName(table.GetColumnName(m.Member.Name)); + case MemberExpression memberExpr: + return GetExcludedColumnName(table.GetColumnName(memberExpr.Member.Name)); - case BinaryExpression b: - var left = ToSqlExpression(table, b.Left); - var right = ToSqlExpression(table, b.Right); - var op = b.NodeType switch + case BinaryExpression binaryExpr: { - ExpressionType.Add => b.Type == typeof(string) ? ConcatOperator : "+", - ExpressionType.Subtract => "-", - ExpressionType.Multiply => "*", - ExpressionType.Divide => "/", - ExpressionType.Modulo => "%", - ExpressionType.AndAlso => "AND", - ExpressionType.OrElse => "OR", - ExpressionType.Equal => "=", - ExpressionType.NotEqual => "<>", - ExpressionType.LessThan => "<", - ExpressionType.LessThanOrEqual => "<=", - ExpressionType.GreaterThan => ">", - ExpressionType.GreaterThanOrEqual => ">=", - _ => throw new NotSupportedException($"Unsupported operator: {b.NodeType}") - }; - return $"({left} {op} {right})"; - - case ConstantExpression c: - if (c.Type == typeof(RawSqlValue) && c.Value != null) + var op = binaryExpr.NodeType switch + { + ExpressionType.Add => binaryExpr.Type == typeof(string) ? ConcatOperator : "+", + ExpressionType.Subtract => "-", + ExpressionType.Multiply => "*", + ExpressionType.Divide => "/", + ExpressionType.Modulo => "%", + ExpressionType.AndAlso => "AND", + ExpressionType.OrElse => "OR", + ExpressionType.Equal => "=", + ExpressionType.NotEqual => "<>", + ExpressionType.LessThan => "<", + ExpressionType.LessThanOrEqual => "<=", + ExpressionType.GreaterThan => ">", + ExpressionType.GreaterThanOrEqual => ">=", + _ => throw new NotSupportedException($"Unsupported operator: {binaryExpr.NodeType}") + }; + + var lhs = ToSqlExpression(table, binaryExpr.Left); + var rhs = ToSqlExpression(table, binaryExpr.Right); + + return $"({lhs} {op} {rhs})"; + } + + case ConstantExpression contantExpr: + if (contantExpr.Type == typeof(RawSqlValue) && contantExpr.Value != null) { - return ((RawSqlValue)c.Value!).Sql; + return ((RawSqlValue)contantExpr.Value!).Sql; } - if (c.Type == typeof(string) || - c.Type == typeof(Guid)) + if (contantExpr.Type == typeof(string) || + contantExpr.Type == typeof(Guid)) { - return $"'{c.Value}'"; + return $"'{contantExpr.Value}'"; } - if (c.Type == typeof(bool)) + if (contantExpr.Type == typeof(bool)) { - return (bool)c.Value! ? "TRUE" : "FALSE"; + return (bool)contantExpr.Value! ? "TRUE" : "FALSE"; } - return c.Value?.ToString() ?? "NULL"; + return contantExpr.Value?.ToString() ?? "NULL"; - case UnaryExpression u: - if (u.NodeType == ExpressionType.Convert) + case UnaryExpression unaryExpr: + if (unaryExpr.NodeType == ExpressionType.Convert) { - return ToSqlExpression(table, u.Operand); + return ToSqlExpression(table, unaryExpr.Operand); } - if (u.NodeType == ExpressionType.Not) + if (unaryExpr.NodeType == ExpressionType.Not) { - return $"NOT ({ToSqlExpression(table, u.Operand)})"; + return $"NOT ({ToSqlExpression(table, unaryExpr.Operand)})"; } - throw new NotSupportedException($"Unary operator not supported: {u.NodeType}"); + throw new NotSupportedException($"Unary operator not supported: {unaryExpr.NodeType}"); - case MethodCallExpression mce: - // Supporte quelques méthodes courantes (ToLower, ToUpper, Trim, etc.) - var objSql = mce.Object != null ? ToSqlExpression(table, mce.Object) : null; - var argsSql = mce.Arguments.Select(expr1 => ToSqlExpression(table, expr1)).ToArray(); - switch (mce.Method.Name) + case MethodCallExpression methodExpr: { - case "ToLower": - return $"LOWER({objSql})"; - case "ToUpper": - return $"UPPER({objSql})"; - case "Trim": - return $"BTRIM({objSql})"; - case "Contains" when mce is { Object: not null, Arguments.Count: 1 }: - return $"{objSql} LIKE '%' || {argsSql[0]} || '%'"; - case "StartsWith" when mce is { Object: not null, Arguments.Count: 1 }: - return $"{objSql} LIKE {argsSql[0]} || '%'"; - case "EndsWith" when mce is { Object: not null, Arguments.Count: 1 }: - return $"{objSql} LIKE '%' || {argsSql[0]}"; - default: - throw new NotSupportedException($"Method not supported: {mce.Method.Name}"); + var lhs = methodExpr.Object != null ? ToSqlExpression(table, methodExpr.Object) : null; + + string Rhs() + { + return ToSqlExpression(table, methodExpr.Arguments[0]); + } + + switch (methodExpr.Method.Name) + { + case "ToLower": + return $"LOWER({lhs})"; + case "ToUpper": + return $"UPPER({lhs})"; + case "Trim": + return $"BTRIM({lhs})"; + case "Contains" when methodExpr is { Object: not null, Arguments.Count: 1 }: + return $"{lhs} LIKE '%' || {Rhs()} || '%'"; + case "EndsWith" when methodExpr is { Object: not null, Arguments.Count: 1 }: + return $"{lhs} LIKE '%' || {Rhs()}"; + case "StartsWith" when methodExpr is { Object: not null, Arguments.Count: 1 }: + return $"{lhs} LIKE {Rhs()} || '%'"; + default: + throw new NotSupportedException($"Method not supported: {methodExpr.Method.Name}"); + } } - case ParameterExpression p: - return Quote(p.Name ?? "param"); + case ParameterExpression parameterExpr: + return Quote(parameterExpr.Name ?? "param"); default: throw new NotSupportedException($"Expression not supported: {expr.NodeType}"); From aa6fa0a6ff8e8f2d8b4e1ee612233d2f6b260c6e Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 22 May 2025 20:51:14 +0200 Subject: [PATCH 13/13] Simplify --- .../Dialect/SqlDialectBuilder.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs index 50f0ae1..d271cf8 100644 --- a/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs +++ b/src/PhenX.EntityFrameworkCore.BulkInsert/Dialect/SqlDialectBuilder.cs @@ -291,11 +291,6 @@ private string ToSqlExpression(TableMetadata table, Expression expr) { var lhs = methodExpr.Object != null ? ToSqlExpression(table, methodExpr.Object) : null; - string Rhs() - { - return ToSqlExpression(table, methodExpr.Arguments[0]); - } - switch (methodExpr.Method.Name) { case "ToLower": @@ -305,11 +300,11 @@ string Rhs() case "Trim": return $"BTRIM({lhs})"; case "Contains" when methodExpr is { Object: not null, Arguments.Count: 1 }: - return $"{lhs} LIKE '%' || {Rhs()} || '%'"; + return $"{lhs} LIKE '%' || {ToSqlExpression(table, methodExpr.Arguments[0])} || '%'"; case "EndsWith" when methodExpr is { Object: not null, Arguments.Count: 1 }: - return $"{lhs} LIKE '%' || {Rhs()}"; + return $"{lhs} LIKE '%' || {ToSqlExpression(table, methodExpr.Arguments[0])}"; case "StartsWith" when methodExpr is { Object: not null, Arguments.Count: 1 }: - return $"{lhs} LIKE {Rhs()} || '%'"; + return $"{lhs} LIKE {ToSqlExpression(table, methodExpr.Arguments[0])} || '%'"; default: throw new NotSupportedException($"Method not supported: {methodExpr.Method.Name}"); }