Skip to content

Commit fbd395f

Browse files
author
fabien.menager
committed
Refactor bulk insert provider to support SQL Server and PostgreSQL with improved container management and testing
1 parent c1ed463 commit fbd395f

29 files changed

Lines changed: 718 additions & 351 deletions

‎EntityFrameworkCore.ExecuteInsert.Benchmark/BulkInsertVsExecuteInsert.cs‎

Lines changed: 11 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,20 @@
11
using BenchmarkDotNet.Attributes;
2-
using BenchmarkDotNet.Engines;
2+
3+
using DotNet.Testcontainers.Containers;
34

45
using EFCore.BulkExtensions;
56

67
using EntityFrameworkCore.ExecuteInsert.Abstractions;
78

8-
using Microsoft.EntityFrameworkCore;
9-
10-
using Testcontainers.PostgreSql;
11-
129
namespace EntityFrameworkCore.ExecuteInsert.Benchmark;
1310

14-
[MinColumn, MaxColumn, BaselineColumn]
15-
[MemoryDiagnoser]
16-
[SimpleJob(RunStrategy.Throughput, launchCount: 1, warmupCount: 0, iterationCount: 5)]
17-
public class BulkInsertVsExecuteInsert
11+
public abstract class BulkInsertVsExecuteInsert
1812
{
1913
[Params(100_000/*, 1_000_000/*, 10_000_000*/)]
2014
public int N;
2115

2216
private IList<TestEntity> data;
23-
private TestDbContext DbContext;
17+
protected TestDbContext DbContext;
2418

2519
[GlobalSetup]
2620
public void GlobalSetup()
@@ -37,27 +31,19 @@ public void GlobalSetup()
3731

3832
public BulkInsertVsExecuteInsert()
3933
{
40-
PostgresContainer = GetPostgresContainer();
41-
PostgresContainer.StartAsync().GetAwaiter().GetResult();
34+
DbContainer = GetDbContainer();
35+
DbContainer.StartAsync().GetAwaiter().GetResult();
4236

43-
var connectionString = PostgresContainer.GetConnectionString() + ";Include Error Detail=true";
37+
ConfigureDbContext();
4438

45-
DbContext = new TestDbContext();
46-
DbContext.Database.SetConnectionString(connectionString);
47-
DbContext.Database.EnsureDeleted();
4839
DbContext.Database.EnsureCreated();
4940
}
5041

51-
public PostgreSqlContainer PostgresContainer { get; }
42+
protected abstract void ConfigureDbContext();
5243

53-
private static PostgreSqlContainer GetPostgresContainer()
54-
{
55-
return new PostgreSqlBuilder()
56-
.WithDatabase("testdb")
57-
.WithUsername("testuser")
58-
.WithPassword("testpassword")
59-
.Build();
60-
}
44+
public IDatabaseContainer DbContainer { get; }
45+
46+
protected abstract IDatabaseContainer GetDbContainer();
6147

6248
[Benchmark(Baseline = true)]
6349
public async Task ExecuteInsert()

‎EntityFrameworkCore.ExecuteInsert.Benchmark/BulkInsertVsExecuteInsertPostgreSql.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Engines;
3+
4+
using DotNet.Testcontainers.Containers;
5+
6+
using EFCore.BulkExtensions;
7+
8+
using EntityFrameworkCore.ExecuteInsert.Abstractions;
9+
using EntityFrameworkCore.ExecuteInsert.PostgreSql;
10+
11+
using Microsoft.EntityFrameworkCore;
12+
13+
using Testcontainers.PostgreSql;
14+
15+
namespace EntityFrameworkCore.ExecuteInsert.Benchmark;
16+
17+
[MinColumn, MaxColumn, BaselineColumn]
18+
[MemoryDiagnoser]
19+
[SimpleJob(RunStrategy.Throughput, launchCount: 1, warmupCount: 0, iterationCount: 5)]
20+
public class BulkInsertVsExecuteInsertPostgreSql : BulkInsertVsExecuteInsert
21+
{
22+
protected override void ConfigureDbContext()
23+
{
24+
var connectionString = DbContainer.GetConnectionString() + ";Include Error Detail=true";
25+
26+
DbContext = new TestDbContext(p => p
27+
.UseNpgsql(connectionString)
28+
.UseExecuteInsertPostgres()
29+
);
30+
}
31+
32+
protected override IDatabaseContainer GetDbContainer()
33+
{
34+
return new PostgreSqlBuilder()
35+
.WithDatabase("testdb")
36+
.WithUsername("testuser")
37+
.WithPassword("testpassword")
38+
.Build();
39+
}
40+
}

‎EntityFrameworkCore.ExecuteInsert.Benchmark/BulkInsertVsExecuteInsertSqlServer.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Engines;
3+
4+
using DotNet.Testcontainers.Containers;
5+
6+
using EntityFrameworkCore.ExecuteInsert.SqlServer;
7+
8+
using Microsoft.EntityFrameworkCore;
9+
10+
using Testcontainers.MsSql;
11+
12+
namespace EntityFrameworkCore.ExecuteInsert.Benchmark;
13+
14+
[MinColumn, MaxColumn, BaselineColumn]
15+
[MemoryDiagnoser]
16+
[SimpleJob(RunStrategy.Throughput, launchCount: 1, warmupCount: 0, iterationCount: 5)]
17+
public class BulkInsertVsExecuteInsertSqlServer : BulkInsertVsExecuteInsert
18+
{
19+
protected override void ConfigureDbContext()
20+
{
21+
var connectionString = DbContainer.GetConnectionString();
22+
23+
DbContext = new TestDbContext(p => p
24+
.UseSqlServer(connectionString)
25+
.UseExecuteInsertSqlServer()
26+
);
27+
}
28+
29+
protected override IDatabaseContainer GetDbContainer()
30+
{
31+
return new MsSqlBuilder().Build();
32+
}
33+
}

‎EntityFrameworkCore.ExecuteInsert.Benchmark/EntityFrameworkCore.ExecuteInsert.Benchmark.csproj‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@
1010
<ItemGroup>
1111
<PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
1212
<PackageReference Include="EFCore.BulkExtensions.MIT.PostgreSql" Version="8.19.0" />
13-
<PackageReference Include="Testcontainers.PostgreSql" Version="4.4.0" />
1413
<PackageReference Include="Z.EntityFramework.Extensions.EFCore" Version="8.103.8" />
14+
<PackageReference Include="Testcontainers.PostgreSql" Version="4.4.0" />
15+
<PackageReference Include="Testcontainers.MsSql" Version="4.4.0" />
1516
</ItemGroup>
1617

1718
<ItemGroup>
1819
<ProjectReference Include="..\src\EntityFrameworkCore.ExecuteInsert.PostgreSql\EntityFrameworkCore.ExecuteInsert.PostgreSql.csproj" />
20+
<ProjectReference Include="..\src\EntityFrameworkCore.ExecuteInsert.SqlServer\EntityFrameworkCore.ExecuteInsert.SqlServer.csproj" />
1921
</ItemGroup>
2022

2123
</Project>

‎EntityFrameworkCore.ExecuteInsert.Benchmark/Program.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ public class Program
66
{
77
public static void Main(string[] args)
88
{
9-
var summary = BenchmarkRunner.Run<BulkInsertVsExecuteInsert>();
9+
BenchmarkRunner.Run<BulkInsertVsExecuteInsert>();
10+
BenchmarkRunner.Run<BulkInsertVsExecuteInsertSqlServer>();
1011
}
1112
}

‎EntityFrameworkCore.ExecuteInsert.Benchmark/TestDbContext.cs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
1-
using EntityFrameworkCore.ExecuteInsert.PostgreSql;
2-
31
using Microsoft.EntityFrameworkCore;
42

53
namespace EntityFrameworkCore.ExecuteInsert.Benchmark;
64

75
public class TestDbContext : DbContext
86
{
7+
public Action<DbContextOptionsBuilder> Configure { get; }
8+
99
public DbSet<TestEntity> TestEntities { get; set; } = null!;
1010

11+
public TestDbContext(Action<DbContextOptionsBuilder> configure)
12+
{
13+
Configure = configure;
14+
}
15+
1116
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
1217
{
13-
optionsBuilder
14-
.UseNpgsql()
15-
.UseExecuteInsertPostgres();
18+
Configure(optionsBuilder);
1619
}
1720
}

‎src/EntityFrameworkCore.ExecuteInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using System.Data.Common;
2+
3+
using Microsoft.EntityFrameworkCore;
4+
5+
using Npgsql;
6+
7+
namespace EntityFrameworkCore.ExecuteInsert.PostgreSql;
8+
9+
public class PostgreSqlBulkInsertProvider : BulkInsertProviderBase
10+
{
11+
public override string OpenDelimiter => "\"";
12+
public override string CloseDelimiter => "\"";
13+
14+
//language=sql
15+
protected override string CreateTableCopySql => "CREATE TEMPORARY TABLE {0} AS TABLE {1} WITH NO DATA;";
16+
17+
//language=sql
18+
protected override string AddTableCopyBulkInsertId => "ALTER TABLE {0} ADD COLUMN _bulk_insert_id SERIAL PRIMARY KEY;";
19+
20+
private string GetBinaryImportCommand(DbContext context, Type entityType, string tableName)
21+
{
22+
var columns = GetEscapedColumns(context, entityType, false);
23+
24+
return $"COPY {tableName} ({string.Join(", ", columns)}) FROM STDIN (FORMAT BINARY)";
25+
}
26+
27+
protected override async Task BulkImport<T>(DbContext context, DbConnection connection, IEnumerable<T> entities,
28+
string tableName, PropertyAccessor[] properties, CancellationToken ctk) where T : class
29+
{
30+
var importCommand = GetBinaryImportCommand(context, typeof(T), tableName);
31+
32+
await using (var writer = await ((NpgsqlConnection)connection).BeginBinaryImportAsync(importCommand, ctk))
33+
{
34+
foreach (var entity in entities)
35+
{
36+
await writer.StartRowAsync(ctk);
37+
38+
foreach (var property in properties)
39+
{
40+
var value = property.GetValue(entity);
41+
42+
await writer.WriteAsync(value, ctk);
43+
}
44+
}
45+
46+
await writer.CompleteAsync(ctk);
47+
}
48+
}
49+
}

‎src/EntityFrameworkCore.ExecuteInsert.PostgreSql/PostgresBulkInsertExtensions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ public static class PostgresBulkInsertExtensions
77
{
88
public static DbContextOptionsBuilder UseExecuteInsertPostgres(this DbContextOptionsBuilder optionsBuilder)
99
{
10-
var extension = optionsBuilder.Options.FindExtension<ExecuteInsertOptionsExtension<BulkInsertProvider>>() ?? new ExecuteInsertOptionsExtension<BulkInsertProvider>();
10+
var extension = optionsBuilder.Options.FindExtension<ExecuteInsertOptionsExtension<PostgreSqlBulkInsertProvider>>() ?? new ExecuteInsertOptionsExtension<PostgreSqlBulkInsertProvider>();
1111

1212
((IDbContextOptionsBuilderInfrastructure)optionsBuilder).AddOrUpdateExtension(extension);
1313

‎src/EntityFrameworkCore.ExecuteInsert.SqlServer/EntityFrameworkCore.ExecuteInsert.SqlServer.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
</ItemGroup>
66

77
<ItemGroup>
8-
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.1" />
8+
<PackageReference Include="JetBrains.Annotations" Version="2024.3.0" />
9+
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.15" />
910
</ItemGroup>
1011

1112
</Project>

‎src/EntityFrameworkCore.ExecuteInsert.SqlServer/EnumerableDataReader.cs‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
using System.Data;
2+
3+
namespace EntityFrameworkCore.ExecuteInsert.SqlServer;
4+
5+
public class EnumerableDataReader<T> : IDataReader
6+
{
7+
private readonly IEnumerator<T> _enumerator;
8+
private readonly PropertyAccessor[] _properties;
9+
private readonly Dictionary<string, int> _ordinalMap;
10+
11+
public EnumerableDataReader(IEnumerable<T> rows, PropertyAccessor[] properties)
12+
{
13+
_enumerator = rows.GetEnumerator();
14+
_properties = properties;
15+
_ordinalMap = properties
16+
.Select((p, i) => new
17+
{
18+
Property = p,
19+
Index = i,
20+
})
21+
.ToDictionary(
22+
p => p.Property.Name,
23+
p => p.Index
24+
);
25+
}
26+
27+
public virtual object GetValue(int i)
28+
{
29+
if (_enumerator.Current != null)
30+
{
31+
return _properties[i].GetValue(_enumerator.Current);
32+
}
33+
34+
return DBNull.Value;
35+
}
36+
37+
public bool Read() => _enumerator.MoveNext();
38+
39+
public int FieldCount => _properties.Length;
40+
public Type GetFieldType(int i) => _properties[i].ProviderClrType;
41+
42+
public int GetOrdinal(string name) => _ordinalMap.GetValueOrDefault(name, -1);
43+
44+
public int Depth => 0;
45+
public bool IsClosed => false;
46+
public int RecordsAffected => 0;
47+
48+
public void Close()
49+
{
50+
}
51+
52+
public void Dispose()
53+
{
54+
_enumerator.Dispose();
55+
}
56+
57+
public DataTable GetSchemaTable() => throw new NotImplementedException();
58+
59+
public bool NextResult() => throw new NotImplementedException();
60+
61+
public int GetValues(object[] values) => throw new NotImplementedException();
62+
63+
public bool IsDBNull(int i) => GetValue(i) is DBNull;
64+
65+
public object this[int i] => throw new NotImplementedException();
66+
67+
public object this[string name] => throw new NotImplementedException();
68+
69+
public string GetString(int i) => throw new NotImplementedException();
70+
71+
public bool GetBoolean(int i) => throw new NotImplementedException();
72+
73+
public byte GetByte(int i) => throw new NotImplementedException();
74+
75+
public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length) => throw new NotImplementedException();
76+
77+
public char GetChar(int i) => throw new NotImplementedException();
78+
79+
public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length) => throw new NotImplementedException();
80+
81+
public IDataReader GetData(int i) => throw new NotImplementedException();
82+
83+
public string GetDataTypeName(int i) => throw new NotImplementedException();
84+
85+
public DateTime GetDateTime(int i) => throw new NotImplementedException();
86+
87+
public decimal GetDecimal(int i) => throw new NotImplementedException();
88+
89+
public double GetDouble(int i) => throw new NotImplementedException();
90+
91+
public float GetFloat(int i) => throw new NotImplementedException();
92+
93+
public Guid GetGuid(int i) => throw new NotImplementedException();
94+
95+
public short GetInt16(int i) => throw new NotImplementedException();
96+
97+
public int GetInt32(int i) => throw new NotImplementedException();
98+
99+
public long GetInt64(int i) => throw new NotImplementedException();
100+
101+
public string GetName(int i) => throw new NotImplementedException();
102+
}

0 commit comments

Comments
 (0)