Skip to content

Commit 138e2b3

Browse files
SebastianStehleSebastianStehle
committed
Jsonb
1 parent 535d570 commit 138e2b3

18 files changed

Lines changed: 219 additions & 120 deletions

File tree

‎src/PhenX.EntityFrameworkCore.BulkInsert.MySql/MySqlBulkInsertProvider.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,8 @@
99

1010
namespace PhenX.EntityFrameworkCore.BulkInsert.MySql;
1111

12-
internal class MySqlBulkInsertProvider : BulkInsertProviderBase<MySqlServerDialectBuilder>
12+
internal class MySqlBulkInsertProvider(ILogger<MySqlBulkInsertProvider>? logger = null) : BulkInsertProviderBase<MySqlServerDialectBuilder>(logger)
1313
{
14-
public MySqlBulkInsertProvider(ILogger<MySqlBulkInsertProvider>? logger = null) : base(logger)
15-
{
16-
}
1714

1815
//language=sql
1916
/// <inheritdoc />

‎src/PhenX.EntityFrameworkCore.BulkInsert.PostgreSql/PostgreSqlBulkInsertProvider.cs‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,18 @@
66
using Microsoft.Extensions.Logging;
77

88
using Npgsql;
9+
using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping;
10+
11+
using NpgsqlTypes;
912

1013
using PhenX.EntityFrameworkCore.BulkInsert.Metadata;
1114
using PhenX.EntityFrameworkCore.BulkInsert.Options;
1215

1316
namespace PhenX.EntityFrameworkCore.BulkInsert.PostgreSql;
1417

1518
[UsedImplicitly]
16-
internal class PostgreSqlBulkInsertProvider : BulkInsertProviderBase<PostgreSqlDialectBuilder>
19+
internal class PostgreSqlBulkInsertProvider(ILogger<PostgreSqlBulkInsertProvider>? logger = null) : BulkInsertProviderBase<PostgreSqlDialectBuilder>(logger)
1720
{
18-
public PostgreSqlBulkInsertProvider(ILogger<PostgreSqlBulkInsertProvider>? logger = null) : base(logger)
19-
{
20-
}
2121

2222
//language=sql
2323
/// <inheritdoc />
@@ -51,6 +51,9 @@ protected override async Task BulkInsert<T>(
5151
? connection.BeginBinaryImport(command)
5252
: await connection.BeginBinaryImportAsync(command, ctk);
5353

54+
// The type mapping can be null for obvious types like string.
55+
var columnTypes = columns.Select(GetPostgreSqlType).ToArray();
56+
5457
foreach (var entity in entities)
5558
{
5659
if (sync)
@@ -63,19 +66,40 @@ protected override async Task BulkInsert<T>(
6366
await writer.StartRowAsync(ctk);
6467
}
6568

69+
var columnIndex = 0;
6670
foreach (var column in columns)
6771
{
6872
var value = column.GetValue(entity);
6973

74+
// Get the actual type, so that the writer can do the conversation to the target type automatically.
75+
var type = columnTypes[columnIndex];
76+
7077
if (sync)
7178
{
72-
// ReSharper disable once MethodHasAsyncOverloadWithCancellation
73-
writer.Write(value);
79+
if (type != null)
80+
{
81+
// ReSharper disable once MethodHasAsyncOverloadWithCancellation
82+
writer.Write(value, type.Value);
83+
}
84+
else
85+
{
86+
// ReSharper disable once MethodHasAsyncOverloadWithCancellation
87+
writer.Write(value);
88+
}
7489
}
7590
else
7691
{
77-
await writer.WriteAsync(value, ctk);
92+
if (type != null)
93+
{
94+
await writer.WriteAsync(value, type.Value, ctk);
95+
}
96+
else
97+
{
98+
await writer.WriteAsync(value, ctk);
99+
}
78100
}
101+
102+
columnIndex++;
79103
}
80104
}
81105

@@ -91,6 +115,12 @@ protected override async Task BulkInsert<T>(
91115
await writer.CompleteAsync(ctk);
92116
await writer.DisposeAsync();
93117
}
118+
}
119+
120+
private static NpgsqlDbType? GetPostgreSqlType(ColumnMetadata column)
121+
{
122+
var mapping = column.Property.GetRelationalTypeMapping() as NpgsqlTypeMapping;
94123

124+
return mapping?.NpgsqlDbType;
95125
}
96126
}

‎src/PhenX.EntityFrameworkCore.BulkInsert.SqlServer/SqlServerBulkInsertProvider.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,8 @@
1111
namespace PhenX.EntityFrameworkCore.BulkInsert.SqlServer;
1212

1313
[UsedImplicitly]
14-
internal class SqlServerBulkInsertProvider : BulkInsertProviderBase<SqlServerDialectBuilder>
14+
internal class SqlServerBulkInsertProvider(ILogger<SqlServerBulkInsertProvider>? logger = null) : BulkInsertProviderBase<SqlServerDialectBuilder>(logger)
1515
{
16-
public SqlServerBulkInsertProvider(ILogger<SqlServerBulkInsertProvider>? logger = null) : base(logger)
17-
{
18-
}
1916

2017
//language=sql
2118
/// <inheritdoc />

‎src/PhenX.EntityFrameworkCore.BulkInsert.Sqlite/SqliteBulkInsertProvider.cs‎

Lines changed: 13 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,9 @@
1313
namespace PhenX.EntityFrameworkCore.BulkInsert.Sqlite;
1414

1515
[UsedImplicitly]
16-
internal class SqliteBulkInsertProvider : BulkInsertProviderBase<SqliteDialectBuilder>
16+
internal class SqliteBulkInsertProvider(ILogger<SqliteBulkInsertProvider>? logger = null) : BulkInsertProviderBase<SqliteDialectBuilder>(logger)
1717
{
18-
public SqliteBulkInsertProvider(ILogger<SqliteBulkInsertProvider>? logger = null) : base(logger)
19-
{
20-
}
18+
private const int MaxParams = 1000;
2119

2220
/// <inheritdoc />
2321
protected override string BulkInsertId => "rowid";
@@ -34,48 +32,17 @@ protected override Task AddBulkInsertIdColumn<T>(
3432
CancellationToken cancellationToken
3533
) where T : class => Task.CompletedTask;
3634

37-
/// <summary>
38-
/// Taken from https://github.com/dotnet/efcore/blob/667c569c49a1ab7e142621395d3f14f2af0508b4/src/Microsoft.Data.Sqlite.Core/SqliteValueBinder.cs#L231
39-
/// As the method is not exposed in the public API, we need to copy it here.
40-
/// </summary>
41-
private static readonly Dictionary<Type, SqliteType> SqliteTypeMapping =
42-
new()
43-
{
44-
{ typeof(bool), SqliteType.Integer },
45-
{ typeof(byte), SqliteType.Integer },
46-
{ typeof(byte[]), SqliteType.Blob },
47-
{ typeof(char), SqliteType.Text },
48-
{ typeof(DateTime), SqliteType.Text },
49-
{ typeof(DateTimeOffset), SqliteType.Text },
50-
{ typeof(DateOnly), SqliteType.Text },
51-
{ typeof(TimeOnly), SqliteType.Text },
52-
{ typeof(DBNull), SqliteType.Text },
53-
{ typeof(decimal), SqliteType.Text },
54-
{ typeof(double), SqliteType.Real },
55-
{ typeof(float), SqliteType.Real },
56-
{ typeof(Guid), SqliteType.Text },
57-
{ typeof(int), SqliteType.Integer },
58-
{ typeof(long), SqliteType.Integer },
59-
{ typeof(sbyte), SqliteType.Integer },
60-
{ typeof(short), SqliteType.Integer },
61-
{ typeof(string), SqliteType.Text },
62-
{ typeof(TimeSpan), SqliteType.Text },
63-
{ typeof(uint), SqliteType.Integer },
64-
{ typeof(ulong), SqliteType.Integer },
65-
{ typeof(ushort), SqliteType.Integer }
66-
};
67-
68-
private static SqliteType GetSqliteType(Type clrType)
35+
private static SqliteType GetSqliteType(ColumnMetadata column)
6936
{
70-
var type = Nullable.GetUnderlyingType(clrType) ?? clrType;
71-
type = type.IsEnum ? Enum.GetUnderlyingType(type) : type;
72-
73-
if (SqliteTypeMapping.TryGetValue(type, out var sqliteType))
37+
var storeType = column.Property.GetRelationalTypeMapping().StoreType;
38+
return storeType switch
7439
{
75-
return sqliteType;
76-
}
77-
78-
throw new InvalidOperationException($"Unknown Sqlite type for {clrType}");
40+
"INTEGER" => SqliteType.Integer,
41+
"FLOAT" => SqliteType.Real,
42+
"TEXT" => SqliteType.Text,
43+
"BLOB" => SqliteType.Blob,
44+
_ => throw new NotSupportedException($"Invalid store type '{storeType}' for property '{column.PropertyName}'"),
45+
};
7946
}
8047

8148
private static DbCommand GetInsertCommand(
@@ -141,15 +108,13 @@ protected override async Task BulkInsert<T>(
141108
CancellationToken ctk
142109
) where T : class
143110
{
144-
const int maxParams = 1000;
145-
var batchSize = options.BatchSize ?? 5;
146-
batchSize = Math.Min(batchSize, maxParams / columns.Count);
111+
var batchSize = Math.Min(options.BatchSize ?? 5, MaxParams / columns.Count);
147112

148113
// The StringBuilder can be resuse between the batches.
149114
var sb = new StringBuilder();
150115

151116
var columnList = tableInfo.GetColumns(options.CopyGeneratedColumns);
152-
var columnTypes = columnList.Select(c => GetSqliteType(c.ProviderClrType ?? c.ClrType)).ToArray();
117+
var columnTypes = columnList.Select(GetSqliteType).ToArray();
153118

154119
await using var insertCommand =
155120
GetInsertCommand(

‎src/PhenX.EntityFrameworkCore.BulkInsert/BulkInsertOptionsExtension.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,8 @@ public void Validate(IDbContextOptions options)
2020
{
2121
}
2222

23-
private class BulkInsertOptionsExtensionInfo : DbContextOptionsExtensionInfo
23+
private class BulkInsertOptionsExtensionInfo(IDbContextOptionsExtension extension) : DbContextOptionsExtensionInfo(extension)
2424
{
25-
public BulkInsertOptionsExtensionInfo(IDbContextOptionsExtension extension)
26-
: base(extension) { }
2725

2826
/// <inheritdoc />
2927
public override int GetServiceProviderHashCode() => 0;

‎src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/ColumnMetadata.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ internal sealed class ColumnMetadata(IProperty property, SqlDialectBuilder dial
99
{
1010
private readonly PropertyAccessor.Getter<object, object?> _getter = BuildGetter(property);
1111

12+
public IProperty Property { get; } = property;
13+
1214
public string PropertyName { get; } = property.Name;
1315

1416
public string ColumnName { get; } = property.GetColumnName();

‎src/PhenX.EntityFrameworkCore.BulkInsert/Metadata/MetadataProviderExtension.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,8 @@ public void Validate(IDbContextOptions options)
1717
{
1818
}
1919

20-
private class MetadataProviderExtensionInfo : DbContextOptionsExtensionInfo
20+
private class MetadataProviderExtensionInfo(IDbContextOptionsExtension extension) : DbContextOptionsExtensionInfo(extension)
2121
{
22-
public MetadataProviderExtensionInfo(IDbContextOptionsExtension extension)
23-
: base(extension) { }
2422

2523
/// <inheritdoc />
2624
public override int GetServiceProviderHashCode() => 0;

‎tests/PhenX.EntityFrameworkCore.BulkInsert.Benchmark/TestDbContext.cs‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,12 @@
22

33
namespace PhenX.EntityFrameworkCore.BulkInsert.Benchmark;
44

5-
public class TestDbContext : DbContext
5+
public class TestDbContext(Action<DbContextOptionsBuilder> configure) : DbContext
66
{
7-
public Action<DbContextOptionsBuilder> Configure { get; }
7+
public Action<DbContextOptionsBuilder> Configure { get; } = configure;
88

99
public DbSet<TestEntity> TestEntities { get; set; } = null!;
1010

11-
public TestDbContext(Action<DbContextOptionsBuilder> configure)
12-
{
13-
Configure = configure;
14-
}
15-
1611
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
1712
{
1813
Configure(optionsBuilder);

‎tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/Extensions.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using System.Text.Json;
2+
3+
using Microsoft.EntityFrameworkCore;
4+
using Microsoft.EntityFrameworkCore.Metadata.Builders;
5+
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
6+
7+
namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext;
8+
9+
public static class Extensions
10+
{
11+
public static PropertyBuilder<T> AsJsonString<T>(this PropertyBuilder<T> propertyBuilder, string? columnType)
12+
where T : class
13+
{
14+
var converter = new ValueConverter<T, string>(
15+
v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null),
16+
v => JsonSerializer.Deserialize<T>(v, (JsonSerializerOptions?)null)!
17+
);
18+
19+
propertyBuilder.HasConversion(converter).HasColumnType(columnType);
20+
return propertyBuilder;
21+
}
22+
23+
}

‎tests/PhenX.EntityFrameworkCore.BulkInsert.Tests/DbContext/TestDbContext.cs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@ namespace PhenX.EntityFrameworkCore.BulkInsert.Tests.DbContext;
66
public class TestDbContext : TestDbContextBase
77
{
88
public DbSet<TestEntity> TestEntities { get; set; } = null!;
9+
public DbSet<TestEntityWithJson> TestEntitiesWithJson { get; set; } = null!;
910
public DbSet<TestEntityWithGuidId> TestEntitiesWithGuidIds { get; set; } = null!;
1011
public DbSet<TestEntityWithConverters> TestEntitiesWithConverters { get; set; } = null!;
1112

1213
protected override void OnModelCreating(ModelBuilder modelBuilder)
1314
{
1415
base.OnModelCreating(modelBuilder);
16+
1517
modelBuilder.Entity<TestEntityWithConverters>(builder =>
1618
{
1719
builder.Property(e => e.CreatedAt)
@@ -25,3 +27,59 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
2527
});
2628
}
2729
}
30+
31+
public class TestDbContextPostgreSql : TestDbContext
32+
{
33+
protected override void OnModelCreating(ModelBuilder modelBuilder)
34+
{
35+
base.OnModelCreating(modelBuilder);
36+
37+
modelBuilder.Entity<TestEntityWithJson>(b =>
38+
{
39+
b.Property(x => x.Json).AsJsonString("jsonb");
40+
});
41+
}
42+
}
43+
44+
public class TestDbContextMySql : TestDbContext
45+
{
46+
protected override void OnModelCreating(ModelBuilder modelBuilder)
47+
{
48+
base.OnModelCreating(modelBuilder);
49+
50+
modelBuilder.Entity<TestEntityWithJson>(b =>
51+
{
52+
b.Property(x => x.Json).AsJsonString("json");
53+
});
54+
}
55+
}
56+
57+
public class TestDbContextSqlServer : TestDbContext
58+
{
59+
protected override void OnModelCreating(ModelBuilder modelBuilder)
60+
{
61+
base.OnModelCreating(modelBuilder);
62+
63+
modelBuilder.Entity<TestEntityWithJson>(b =>
64+
{
65+
b.Property(x => x.Json).AsJsonString(null);
66+
});
67+
}
68+
}
69+
70+
71+
public class TestDbContextSqlite : TestDbContext
72+
{
73+
protected override void OnModelCreating(ModelBuilder modelBuilder)
74+
{
75+
base.OnModelCreating(modelBuilder);
76+
77+
modelBuilder.Entity<TestEntityWithJson>(b =>
78+
{
79+
b.Property(x => x.Json).AsJsonString(null);
80+
});
81+
}
82+
}
83+
84+
85+

0 commit comments

Comments
 (0)