From 38d5d626a5b76f305fab3a07de272a513d5a1d9d Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Mon, 31 Aug 2026 09:43:23 +0200 Subject: [PATCH 1/5] Copy-DbaDbTableData - Map query columns onto the writable destination columns With -Query and without -ForceExplicitMapping the command added no column mappings and left SqlBulkCopy to map by position. SqlBulkCopy's own list contains every destination column: a computed column makes the server reject the insert (error 271), and a rowversion column is silently dropped together with the source column that lands on it - every column behind it shifts by one, the last one stays empty, and the command reports success. That is the silent data corruption of #10661 (reproduced in the lab with a rowversion column; a computed column fails loudly on this stack). The command now builds the positional mapping itself, source ordinal by source ordinal onto the destination columns that can actually be written - computed and rowversion columns excluded. An identity placeholder keeps working as documented, because a mapping onto an identity column without -KeepIdentity is still ignored by the server. A query that returns more columns than the destination can take is refused with a message naming both counts instead of dropping the surplus. Table mode and -ForceExplicitMapping keep their name-based mapping; the only change there is that rowversion columns are no longer mapped, which SqlBulkCopy tolerated anyway. Help text updated accordingly. Fixes #10661 (do Copy-DbaDbTableData) Co-Authored-By: Claude Fable 5 --- public/Copy-DbaDbTableData.ps1 | 32 ++++++++--- tests/Copy-DbaDbTableData.Tests.ps1 | 87 +++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 8 deletions(-) diff --git a/public/Copy-DbaDbTableData.ps1 b/public/Copy-DbaDbTableData.ps1 index 3ed8b1aba75..ecbe0a0888e 100644 --- a/public/Copy-DbaDbTableData.ps1 +++ b/public/Copy-DbaDbTableData.ps1 @@ -56,13 +56,14 @@ function Copy-DbaDbTableData { Custom SQL SELECT query to use as the data source instead of copying the entire table or view. Supports 3 or 4-part object names. Use this when you need to filter rows, join multiple tables, or transform data during the copy operation. Still requires specifying a Table or View parameter for metadata purposes. - Note: Columns are mapped by ordinal position. If the destination table has an identity column, include a placeholder value (e.g., 0) in your SELECT list at that position. + Note: Columns are mapped by position onto the writable columns of the destination table, so computed and rowversion columns of the destination do not count and must not have a counterpart in the SELECT list. + If the destination table has an identity column, include a placeholder value (e.g., 0) in your SELECT list at that position. The placeholder will be ignored and the identity value auto-generated unless -KeepIdentity is specified. .PARAMETER ForceExplicitMapping When used together with Query parameter, force the use of explicit column mapping (name-based) instead of switching over to ordinal position mapping. Use with care if query contains aliases. Default behaviour when using Query parameter is to use ordinal position mapping, due to the possibility of the query including aliases (SELECT x AS y) which could lead to column mismatching and data not copying. - The downside of it automatically switching over to ordinal mapping is that it also tries to copy over computed columns, which will cause it to fail. + Positional mapping skips the computed and rowversion columns of the destination table, so the SELECT list only has to match its writable columns. .PARAMETER AutoCreateTable Automatically creates the destination table if it doesn't exist, using the same structure as the source table. @@ -672,11 +673,14 @@ function Copy-DbaDbTableData { $bulkCopy.NotifyAfter = $NotifyAfter $bulkCopy.BulkCopyTimeout = $BulkCopyTimeout - # Get list of non-computed columns from destination table to avoid insert failures + # Get list of writable columns from destination table to avoid insert failures. Computed and rowversion + # columns cannot be written, and they are also what breaks the implicit positional mapping of SqlBulkCopy: + # it counts a computed column (the server then rejects the insert) and silently drops the source column + # that lands on a rowversion column, shifting every column behind it by one (see #10661). # Refresh the columns collection to ensure it's populated $desttable.Columns.Refresh() - $destColumns = $desttable.Columns | Where-Object Computed -eq $false | Select-Object -ExpandProperty Name - Write-Message -Level Verbose -Message "Destination table has $($destColumns.Count) non-computed columns" + $destColumns = @($desttable.Columns | Where-Object { -not $PSItem.Computed -and $PSItem.DataType.SqlDataType -ne "Timestamp" } | Select-Object -ExpandProperty Name) + Write-Message -Level Verbose -Message "Destination table has $($destColumns.Count) writable columns" # The legacy bulk copy library uses a 4 byte integer to track the RowsCopied, so the only option is to use # integer wrap so that copy operations of row counts greater than [int32]::MaxValue will report accurate numbers. @@ -705,18 +709,30 @@ function Copy-DbaDbTableData { $reader = $cmd.ExecuteReader() # Only apply explicit column mapping for straight table copies (not custom queries) - # Custom queries may have different column names/aliases, so let SqlBulkCopy use ordinal mapping + # Custom queries may have different column names/aliases, so they are mapped by position # Appending -ForceExplicitMapping will override this behaviour and keep explicit column mapping if (-not (Test-Bound -ParameterName Query) -or $ForceExplicitMapping) { - # Map only columns that exist in both source and destination (excluding computed columns) + # Map only columns that exist in both source and destination (excluding computed and rowversion columns) for ($i = 0; $i -lt $reader.FieldCount; $i++) { $sourceColumn = $reader.GetName($i) if ($destColumns -contains $sourceColumn) { $null = $bulkCopy.ColumnMappings.Add($sourceColumn, $sourceColumn) } else { - Write-Message -Level Verbose -Message "Skipping column '$sourceColumn' (not in destination or is computed)" + Write-Message -Level Verbose -Message "Skipping column '$sourceColumn' (not in destination or not writable)" } } + } else { + # Map the query columns by position onto the writable destination columns. This is what SqlBulkCopy does + # on its own, except that its list also contains the computed and rowversion columns (see above). + if ($reader.FieldCount -gt $destColumns.Count) { + $columnCountMessage = "The query returns $($reader.FieldCount) columns but $fqtndest has only $($destColumns.Count) writable columns. Computed and rowversion columns cannot be written and do not count." + $reader.Close() + throw $columnCountMessage + } + for ($i = 0; $i -lt $reader.FieldCount; $i++) { + Write-Message -Level Verbose -Message "Mapping query column $i ($($reader.GetName($i))) to destination column $($destColumns[$i])" + $null = $bulkCopy.ColumnMappings.Add($i, $destColumns[$i]) + } } $bulkCopy.WriteToServer($reader) diff --git a/tests/Copy-DbaDbTableData.Tests.ps1 b/tests/Copy-DbaDbTableData.Tests.ps1 index cbeb260bdaa..8706cad9f60 100644 --- a/tests/Copy-DbaDbTableData.Tests.ps1 +++ b/tests/Copy-DbaDbTableData.Tests.ps1 @@ -216,6 +216,93 @@ Describe $CommandName -Tag IntegrationTests { } } + Context "When using Query without ForceExplicitMapping and the destination has unwritable columns" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + $null = $sourceDb.Query("CREATE TABLE dbo.dbatoolsci_positional_source (Id INT, A INT, B INT, C INT)") + $null = $sourceDb.Query("INSERT dbo.dbatoolsci_positional_source (Id, A, B, C) VALUES (1, 11, 22, 33), (2, 111, 222, 333)") + # A computed and a rowversion column sit between the writable ones, so a positional mapping that + # counts them shifts every column behind them (#10661). + $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_dest (Id INT, A INT, Computed AS (A * 10), RV ROWVERSION, B INT, C INT)") + $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_identity (Id INT IDENTITY(1, 1), A INT, B INT, C INT)") + $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_rowversion (Id INT, A INT, RV ROWVERSION, B INT, C INT)") + + $splatPositional = @{ + SqlInstance = $TestConfig.InstanceCopy1 + Destination = $TestConfig.InstanceCopy2 + Database = "tempdb" + Table = "dbatoolsci_positional_source" + Query = "SELECT Id, A, B, C FROM dbo.dbatoolsci_positional_source" + DestinationTable = "dbatoolsci_positional_dest" + } + + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + AfterAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + $null = $sourceDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_source") + $null = $destinationDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_dest") + $null = $destinationDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_identity") + $null = $destinationDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_rowversion") + + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + It "Maps the query columns by position onto the writable destination columns only" { + $result = Copy-DbaDbTableData @splatPositional + $WarnVar | Should -BeNullOrEmpty + $result.RowsCopied | Should -Be 2 + + $destData = $destinationDb.Query("SELECT Id, A, Computed, B, C FROM dbo.dbatoolsci_positional_dest ORDER BY Id") + $destData.A | Should -Be @(11, 111) + $destData.Computed | Should -Be @(110, 1110) + $destData.B | Should -Be @(22, 222) + $destData.C | Should -Be @(33, 333) + } + + It "Does not shift the columns behind a rowversion column" { + # This is the silent variant: SqlBulkCopy drops the source column that lands on the rowversion + # column and reports success, so the last column ends up empty and the ones before it are off by one. + $splatRowversion = $splatPositional.Clone() + $splatRowversion.DestinationTable = "dbatoolsci_positional_rowversion" + $result = Copy-DbaDbTableData @splatRowversion + $WarnVar | Should -BeNullOrEmpty + $result.RowsCopied | Should -Be 2 + + $destData = $destinationDb.Query("SELECT Id, A, B, C FROM dbo.dbatoolsci_positional_rowversion ORDER BY Id") + $destData.B | Should -Be @(22, 222) + $destData.C | Should -Be @(33, 333) + } + + It "Still ignores the identity placeholder unless KeepIdentity is used" { + $splatIdentity = $splatPositional.Clone() + $splatIdentity.Query = "SELECT 0, A, B, C FROM dbo.dbatoolsci_positional_source ORDER BY Id" + $splatIdentity.DestinationTable = "dbatoolsci_positional_identity" + $result = Copy-DbaDbTableData @splatIdentity + $WarnVar | Should -BeNullOrEmpty + $result.RowsCopied | Should -Be 2 + + $destData = $destinationDb.Query("SELECT Id, A, B, C FROM dbo.dbatoolsci_positional_identity ORDER BY Id") + $destData.Id | Should -Be @(1, 2) + $destData.A | Should -Be @(11, 111) + $destData.C | Should -Be @(33, 333) + } + + It "Refuses a query with more columns than the destination can take instead of dropping them" { + $splatTooMany = $splatPositional.Clone() + $splatTooMany.Query = "SELECT Id, A, B, C, C AS Extra FROM dbo.dbatoolsci_positional_source" + $splatTooMany.Truncate = $true + $result = Copy-DbaDbTableData @splatTooMany -WarningAction SilentlyContinue + $result | Should -BeNullOrEmpty + $WarnVar | Should -Match "5 columns" + $WarnVar | Should -Match "4 writable columns" + $destinationDb.Query("SELECT COUNT(*) AS RowCnt FROM dbo.dbatoolsci_positional_dest").RowCnt | Should -Be 0 + } + } + Context "Regression tests" { BeforeAll { $PSDefaultParameterValues["*-Dba*:EnableException"] = $true From 58bd85931d22191f20b61a5fcee18dd7b4491e86 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Mon, 31 Aug 2026 10:15:14 +0200 Subject: [PATCH 2/5] Copy-DbaDbTableData - Drop the single quotes from the skipped-column message Style only, no behaviour change: the verbose message the previous commit reworded still wrapped the column name in single quotes, the repository standard is double quotes only. (do Copy-DbaDbTableData) Co-Authored-By: Claude Fable 5 --- public/Copy-DbaDbTableData.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/Copy-DbaDbTableData.ps1 b/public/Copy-DbaDbTableData.ps1 index ecbe0a0888e..e864cd6ac82 100644 --- a/public/Copy-DbaDbTableData.ps1 +++ b/public/Copy-DbaDbTableData.ps1 @@ -718,7 +718,7 @@ function Copy-DbaDbTableData { if ($destColumns -contains $sourceColumn) { $null = $bulkCopy.ColumnMappings.Add($sourceColumn, $sourceColumn) } else { - Write-Message -Level Verbose -Message "Skipping column '$sourceColumn' (not in destination or not writable)" + Write-Message -Level Verbose -Message "Skipping column $sourceColumn (not in destination or not writable)" } } } else { From 3a2bc617c659f75dd868f97dd19f001d0dc32eaa Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Mon, 31 Aug 2026 11:03:39 +0200 Subject: [PATCH 3/5] Testing Copy-DbaDbTableData - Select the id column with the case it was created with The two Query-mode tests select "Id" from dbo.dbatoolsci_example4, whose column is "id". On a case insensitive instance that is the same column; on a case sensitive one the server answers "Invalid column name 'Id'" and the test fails with "Expected 1, but got $null". Found by the first setCS run whose COPY lane sits on the case sensitive SQL05 instances. (do Copy-DbaDbTableData) Co-Authored-By: Claude Fable 5 --- tests/Copy-DbaDbTableData.Tests.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Copy-DbaDbTableData.Tests.ps1 b/tests/Copy-DbaDbTableData.Tests.ps1 index 8706cad9f60..ce73cf2ee82 100644 --- a/tests/Copy-DbaDbTableData.Tests.ps1 +++ b/tests/Copy-DbaDbTableData.Tests.ps1 @@ -108,12 +108,12 @@ Describe $CommandName -Tag IntegrationTests { } It "Copy data using a query that relies on the default source database" { - $result = Copy-DbaDbTableData -SqlInstance $TestConfig.InstanceCopy2 -Database tempdb -Table dbo.dbatoolsci_example4 -Query "SELECT TOP (1) Id FROM dbo.dbatoolsci_example4 ORDER BY Id DESC" -DestinationTable dbatoolsci_example3 -Truncate + $result = Copy-DbaDbTableData -SqlInstance $TestConfig.InstanceCopy2 -Database tempdb -Table dbo.dbatoolsci_example4 -Query "SELECT TOP (1) id FROM dbo.dbatoolsci_example4 ORDER BY id DESC" -DestinationTable dbatoolsci_example3 -Truncate $result.RowsCopied | Should -Be 1 } It "Copy data using a query that uses a 3 part query" { - $result = Copy-DbaDbTableData -SqlInstance $TestConfig.InstanceCopy2 -Database tempdb -Table dbo.dbatoolsci_example4 -Query "SELECT TOP (1) Id FROM tempdb.dbo.dbatoolsci_example4 ORDER BY Id DESC" -DestinationTable dbatoolsci_example3 -Truncate + $result = Copy-DbaDbTableData -SqlInstance $TestConfig.InstanceCopy2 -Database tempdb -Table dbo.dbatoolsci_example4 -Query "SELECT TOP (1) id FROM tempdb.dbo.dbatoolsci_example4 ORDER BY id DESC" -DestinationTable dbatoolsci_example3 -Truncate $result.RowsCopied | Should -Be 1 } } From 863aaa76dd5044608eb39e7ea502ddb768401126 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Thu, 3 Sep 2026 15:19:43 +0200 Subject: [PATCH 4/5] Copy-DbaDbTableData - Exclude generated always columns from the writable destination columns The period columns of a temporal table and the ledger metadata columns are GENERATED ALWAYS: not computed, not rowversion, but just as unwritable, and SMO reports them with Computed = false. The positional mapping counted them, mapped query columns onto them and the server rejected the insert with error 13536. The lookup is guarded by the version because SMO only supports GeneratedAlwaysType on SQL Server 2016 and later. (do Copy-DbaDbTableData) Co-Authored-By: Claude Fable 5 --- public/Copy-DbaDbTableData.ps1 | 22 ++++++++++-------- tests/Copy-DbaDbTableData.Tests.ps1 | 36 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/public/Copy-DbaDbTableData.ps1 b/public/Copy-DbaDbTableData.ps1 index e864cd6ac82..527156028dd 100644 --- a/public/Copy-DbaDbTableData.ps1 +++ b/public/Copy-DbaDbTableData.ps1 @@ -56,14 +56,14 @@ function Copy-DbaDbTableData { Custom SQL SELECT query to use as the data source instead of copying the entire table or view. Supports 3 or 4-part object names. Use this when you need to filter rows, join multiple tables, or transform data during the copy operation. Still requires specifying a Table or View parameter for metadata purposes. - Note: Columns are mapped by position onto the writable columns of the destination table, so computed and rowversion columns of the destination do not count and must not have a counterpart in the SELECT list. + Note: Columns are mapped by position onto the writable columns of the destination table, so computed, rowversion and generated always columns of the destination do not count and must not have a counterpart in the SELECT list. If the destination table has an identity column, include a placeholder value (e.g., 0) in your SELECT list at that position. The placeholder will be ignored and the identity value auto-generated unless -KeepIdentity is specified. .PARAMETER ForceExplicitMapping When used together with Query parameter, force the use of explicit column mapping (name-based) instead of switching over to ordinal position mapping. Use with care if query contains aliases. Default behaviour when using Query parameter is to use ordinal position mapping, due to the possibility of the query including aliases (SELECT x AS y) which could lead to column mismatching and data not copying. - Positional mapping skips the computed and rowversion columns of the destination table, so the SELECT list only has to match its writable columns. + Positional mapping skips the computed, rowversion and generated always columns of the destination table, so the SELECT list only has to match its writable columns. .PARAMETER AutoCreateTable Automatically creates the destination table if it doesn't exist, using the same structure as the source table. @@ -673,13 +673,15 @@ function Copy-DbaDbTableData { $bulkCopy.NotifyAfter = $NotifyAfter $bulkCopy.BulkCopyTimeout = $BulkCopyTimeout - # Get list of writable columns from destination table to avoid insert failures. Computed and rowversion - # columns cannot be written, and they are also what breaks the implicit positional mapping of SqlBulkCopy: - # it counts a computed column (the server then rejects the insert) and silently drops the source column - # that lands on a rowversion column, shifting every column behind it by one (see #10661). + # Get list of writable columns from destination table to avoid insert failures. Computed, rowversion + # and generated always columns (temporal periods, ledger metadata) cannot be written, and they are also + # what breaks the implicit positional mapping of SqlBulkCopy: it counts a computed column (the server + # then rejects the insert) and silently drops the source column that lands on a rowversion column, + # shifting every column behind it by one (see #10661). GeneratedAlwaysType is only supported by SMO on + # SQL Server 2016 and later, so it has to be guarded by the version. # Refresh the columns collection to ensure it's populated $desttable.Columns.Refresh() - $destColumns = @($desttable.Columns | Where-Object { -not $PSItem.Computed -and $PSItem.DataType.SqlDataType -ne "Timestamp" } | Select-Object -ExpandProperty Name) + $destColumns = @($desttable.Columns | Where-Object { -not $PSItem.Computed -and $PSItem.DataType.SqlDataType -ne "Timestamp" -and -not ($destServer.VersionMajor -ge 13 -and $PSItem.GeneratedAlwaysType -ne "None") } | Select-Object -ExpandProperty Name) Write-Message -Level Verbose -Message "Destination table has $($destColumns.Count) writable columns" # The legacy bulk copy library uses a 4 byte integer to track the RowsCopied, so the only option is to use @@ -712,7 +714,7 @@ function Copy-DbaDbTableData { # Custom queries may have different column names/aliases, so they are mapped by position # Appending -ForceExplicitMapping will override this behaviour and keep explicit column mapping if (-not (Test-Bound -ParameterName Query) -or $ForceExplicitMapping) { - # Map only columns that exist in both source and destination (excluding computed and rowversion columns) + # Map only columns that exist in both source and destination (excluding computed, rowversion and generated always columns) for ($i = 0; $i -lt $reader.FieldCount; $i++) { $sourceColumn = $reader.GetName($i) if ($destColumns -contains $sourceColumn) { @@ -723,9 +725,9 @@ function Copy-DbaDbTableData { } } else { # Map the query columns by position onto the writable destination columns. This is what SqlBulkCopy does - # on its own, except that its list also contains the computed and rowversion columns (see above). + # on its own, except that its list also contains the computed, rowversion and generated always columns (see above). if ($reader.FieldCount -gt $destColumns.Count) { - $columnCountMessage = "The query returns $($reader.FieldCount) columns but $fqtndest has only $($destColumns.Count) writable columns. Computed and rowversion columns cannot be written and do not count." + $columnCountMessage = "The query returns $($reader.FieldCount) columns but $fqtndest has only $($destColumns.Count) writable columns. Computed, rowversion and generated always columns cannot be written and do not count." $reader.Close() throw $columnCountMessage } diff --git a/tests/Copy-DbaDbTableData.Tests.ps1 b/tests/Copy-DbaDbTableData.Tests.ps1 index ce73cf2ee82..90ebb6a7450 100644 --- a/tests/Copy-DbaDbTableData.Tests.ps1 +++ b/tests/Copy-DbaDbTableData.Tests.ps1 @@ -217,6 +217,14 @@ Describe $CommandName -Tag IntegrationTests { } Context "When using Query without ForceExplicitMapping and the destination has unwritable columns" { + BeforeDiscovery { + # GENERATED ALWAYS columns arrived with SQL Server 2016, so the temporal scenario below cannot + # be built before that. The value decides a Skip, which Pester needs while it discovers the + # tests, so it cannot be read in BeforeAll. + $discoveryDestServer = Connect-DbaInstance -SqlInstance $TestConfig.InstanceCopy2 + $destSupportsTemporal = $discoveryDestServer.VersionMajor -ge 13 + } + BeforeAll { $PSDefaultParameterValues["*-Dba*:EnableException"] = $true @@ -227,6 +235,11 @@ Describe $CommandName -Tag IntegrationTests { $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_dest (Id INT, A INT, Computed AS (A * 10), RV ROWVERSION, B INT, C INT)") $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_identity (Id INT IDENTITY(1, 1), A INT, B INT, C INT)") $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_rowversion (Id INT, A INT, RV ROWVERSION, B INT, C INT)") + if ($destinationDb.Parent.VersionMajor -ge 13) { + # The period columns of a temporal table are GENERATED ALWAYS: not computed, not rowversion, + # but just as unwritable, and interleaved with the writable columns here on purpose. + $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_temporal (Id INT PRIMARY KEY, A INT, ValidFrom DATETIME2 GENERATED ALWAYS AS ROW START NOT NULL, B INT, ValidTo DATETIME2 GENERATED ALWAYS AS ROW END NOT NULL, C INT, PERIOD FOR SYSTEM_TIME (ValidFrom, ValidTo)) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.dbatoolsci_positional_temporal_history))") + } $splatPositional = @{ SqlInstance = $TestConfig.InstanceCopy1 @@ -247,6 +260,13 @@ Describe $CommandName -Tag IntegrationTests { $null = $destinationDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_dest") $null = $destinationDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_identity") $null = $destinationDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_rowversion") + $destinationDb.Tables.Refresh() + if ($destinationDb.Tables | Where-Object Name -eq "dbatoolsci_positional_temporal") { + # System versioning has to be turned off before the temporal table can be dropped. + $null = $destinationDb.Query("ALTER TABLE dbo.dbatoolsci_positional_temporal SET (SYSTEM_VERSIONING = OFF)") + $null = $destinationDb.Query("DROP TABLE dbo.dbatoolsci_positional_temporal") + $null = $destinationDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_positional_temporal_history") + } $PSDefaultParameterValues.Remove("*-Dba*:EnableException") } @@ -277,6 +297,22 @@ Describe $CommandName -Tag IntegrationTests { $destData.C | Should -Be @(33, 333) } + It "Does not count the generated always columns of a temporal destination" -Skip:(-not $destSupportsTemporal) { + # The period columns are GENERATED ALWAYS, so the server refuses explicit values for them. + # A positional mapping that counts them maps writable source columns onto them and fails. + $splatTemporal = $splatPositional.Clone() + $splatTemporal.DestinationTable = "dbatoolsci_positional_temporal" + $result = Copy-DbaDbTableData @splatTemporal + $WarnVar | Should -BeNullOrEmpty + $result.RowsCopied | Should -Be 2 + + $destData = $destinationDb.Query("SELECT Id, A, ValidFrom, B, ValidTo, C FROM dbo.dbatoolsci_positional_temporal ORDER BY Id") + $destData.A | Should -Be @(11, 111) + $destData.B | Should -Be @(22, 222) + $destData.C | Should -Be @(33, 333) + $destData.ValidFrom | Should -Not -BeNullOrEmpty + } + It "Still ignores the identity placeholder unless KeepIdentity is used" { $splatIdentity = $splatPositional.Clone() $splatIdentity.Query = "SELECT 0, A, B, C FROM dbo.dbatoolsci_positional_source ORDER BY Id" From 5487c457658c1957f4c492cdfcc670667f988338 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 12 Sep 2026 10:11:48 +0200 Subject: [PATCH 5/5] Copy-DbaDbTableData - Treat Azure SQL Database as supporting generated always columns The writable-column filter guarded the GeneratedAlwaysType lookup with the major version alone. Azure SQL Database supports temporal and ledger tables while its product version says 12, so the guard now names the engine type as well, the same way SMO decides whether the property exists. The temporal regression test uses the same rule, and a new context copies into a temporal table on a real Azure SQL Database where the lab configuration provides one. (do Copy-DbaDbTableData) Co-Authored-By: Claude Fable 5.1 --- public/Copy-DbaDbTableData.ps1 | 6 +- tests/Copy-DbaDbTableData.Tests.ps1 | 110 ++++++++++++++++++++++++++-- 2 files changed, 109 insertions(+), 7 deletions(-) diff --git a/public/Copy-DbaDbTableData.ps1 b/public/Copy-DbaDbTableData.ps1 index 527156028dd..aae9f10b2f4 100644 --- a/public/Copy-DbaDbTableData.ps1 +++ b/public/Copy-DbaDbTableData.ps1 @@ -678,10 +678,12 @@ function Copy-DbaDbTableData { # what breaks the implicit positional mapping of SqlBulkCopy: it counts a computed column (the server # then rejects the insert) and silently drops the source column that lands on a rowversion column, # shifting every column behind it by one (see #10661). GeneratedAlwaysType is only supported by SMO on - # SQL Server 2016 and later, so it has to be guarded by the version. + # SQL Server 2016 and later and on Azure SQL Database, so it has to be guarded the same way. Azure SQL + # Database is named on its own because its product version says 12 no matter what it supports. + $destSupportsGeneratedAlways = $destServer.VersionMajor -ge 13 -or $destServer.DatabaseEngineType -eq "SqlAzureDatabase" # Refresh the columns collection to ensure it's populated $desttable.Columns.Refresh() - $destColumns = @($desttable.Columns | Where-Object { -not $PSItem.Computed -and $PSItem.DataType.SqlDataType -ne "Timestamp" -and -not ($destServer.VersionMajor -ge 13 -and $PSItem.GeneratedAlwaysType -ne "None") } | Select-Object -ExpandProperty Name) + $destColumns = @($desttable.Columns | Where-Object { -not $PSItem.Computed -and $PSItem.DataType.SqlDataType -ne "Timestamp" -and -not ($destSupportsGeneratedAlways -and $PSItem.GeneratedAlwaysType -ne "None") } | Select-Object -ExpandProperty Name) Write-Message -Level Verbose -Message "Destination table has $($destColumns.Count) writable columns" # The legacy bulk copy library uses a 4 byte integer to track the RowsCopied, so the only option is to use diff --git a/tests/Copy-DbaDbTableData.Tests.ps1 b/tests/Copy-DbaDbTableData.Tests.ps1 index 90ebb6a7450..6537361fc39 100644 --- a/tests/Copy-DbaDbTableData.Tests.ps1 +++ b/tests/Copy-DbaDbTableData.Tests.ps1 @@ -5,6 +5,14 @@ param( $PSDefaultParameterValues = $TestConfig.Defaults ) +BeforeDiscovery { + # Azure SQL Database supports GENERATED ALWAYS columns while its product version says 12, so it is the + # one destination where a version check alone gets the writable columns wrong. No CI environment has + # one; a lab configuration supplies it through AzureSqlDbServer and everywhere else the test skips + # itself. The value decides a Skip, so it has to exist at discovery time. + $script:hasAzureSqlDb = -not [string]::IsNullOrWhiteSpace($TestConfig.AzureSqlDbServer) +} + Describe $CommandName -Tag UnitTests { Context "Parameter validation" { It "Should have the expected parameters" { @@ -218,11 +226,11 @@ Describe $CommandName -Tag IntegrationTests { Context "When using Query without ForceExplicitMapping and the destination has unwritable columns" { BeforeDiscovery { - # GENERATED ALWAYS columns arrived with SQL Server 2016, so the temporal scenario below cannot - # be built before that. The value decides a Skip, which Pester needs while it discovers the - # tests, so it cannot be read in BeforeAll. + # GENERATED ALWAYS columns arrived with SQL Server 2016 and exist on Azure SQL Database, so the + # temporal scenario below cannot be built anywhere else. The value decides a Skip, which Pester + # needs while it discovers the tests, so it cannot be read in BeforeAll. $discoveryDestServer = Connect-DbaInstance -SqlInstance $TestConfig.InstanceCopy2 - $destSupportsTemporal = $discoveryDestServer.VersionMajor -ge 13 + $destSupportsTemporal = $discoveryDestServer.VersionMajor -ge 13 -or $discoveryDestServer.DatabaseEngineType -eq "SqlAzureDatabase" } BeforeAll { @@ -235,7 +243,7 @@ Describe $CommandName -Tag IntegrationTests { $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_dest (Id INT, A INT, Computed AS (A * 10), RV ROWVERSION, B INT, C INT)") $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_identity (Id INT IDENTITY(1, 1), A INT, B INT, C INT)") $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_rowversion (Id INT, A INT, RV ROWVERSION, B INT, C INT)") - if ($destinationDb.Parent.VersionMajor -ge 13) { + if ($destinationDb.Parent.VersionMajor -ge 13 -or $destinationDb.Parent.DatabaseEngineType -eq "SqlAzureDatabase") { # The period columns of a temporal table are GENERATED ALWAYS: not computed, not rowversion, # but just as unwritable, and interleaved with the writable columns here on purpose. $null = $destinationDb.Query("CREATE TABLE dbo.dbatoolsci_positional_temporal (Id INT PRIMARY KEY, A INT, ValidFrom DATETIME2 GENERATED ALWAYS AS ROW START NOT NULL, B INT, ValidTo DATETIME2 GENERATED ALWAYS AS ROW END NOT NULL, C INT, PERIOD FOR SYSTEM_TIME (ValidFrom, ValidTo)) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.dbatoolsci_positional_temporal_history))") @@ -339,6 +347,98 @@ Describe $CommandName -Tag IntegrationTests { } } + Context "When using Query against a temporal destination on Azure SQL Database" -Skip:(-not $script:hasAzureSqlDb) { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + # Azure SQL Database reports product version 12.0 and still has GENERATED ALWAYS columns, so + # the writable columns of a temporal destination cannot be decided by the version alone there. + # A serverless database that has been idle is paused and takes up to a minute to wake up. A + # generous ConnectTimeout does not cover that: while the database resumes, Azure does not + # keep the attempt waiting, it answers it right away with error 40613 "Database ... is not + # currently available. Please retry the connection later." So the first connection is + # retried until the database is up, and only an error that says something else is thrown + # immediately. + $splatAzureConnect = @{ + SqlInstance = $TestConfig.AzureSqlDbServer + Database = $TestConfig.AzureSqlDbName + SqlCredential = $TestConfig.AzureSqlDbCred + ConnectTimeout = 120 + } + $azureResumeAttempt = 0 + while ($null -eq $serverAzure) { + $azureResumeAttempt++ + try { + $serverAzure = Connect-DbaInstance @splatAzureConnect + } catch { + if ($azureResumeAttempt -ge 10 -or $PSItem.Exception.Message -notmatch "is not currently available") { + throw + } + Start-Sleep -Seconds 15 + } + } + $azureDb = $serverAzure.Databases[$TestConfig.AzureSqlDbName] + + $null = $sourceDb.Query("CREATE TABLE dbo.dbatoolsci_azure_source (Id INT, A INT, B INT, C INT)") + $null = $sourceDb.Query("INSERT dbo.dbatoolsci_azure_source (Id, A, B, C) VALUES (1, 11, 22, 33), (2, 111, 222, 333)") + # The same interleaved layout as the temporal test above, on the engine whose version cannot decide it. + $null = $azureDb.Query("CREATE TABLE dbo.dbatoolsci_azure_temporal (Id INT PRIMARY KEY, A INT, ValidFrom DATETIME2 GENERATED ALWAYS AS ROW START NOT NULL, B INT, ValidTo DATETIME2 GENERATED ALWAYS AS ROW END NOT NULL, C INT, PERIOD FOR SYSTEM_TIME (ValidFrom, ValidTo)) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.dbatoolsci_azure_temporal_history))") + + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + AfterAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + $null = $sourceDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_azure_source") + # The BeforeAll can fail partway through - an Azure SQL Database that stays unavailable is the + # realistic case - and then the database object below was never assigned. + if ($azureDb) { + $azureDb.Tables.Refresh() + if ($azureDb.Tables | Where-Object Name -eq "dbatoolsci_azure_temporal") { + # System versioning has to be turned off before the temporal table can be dropped. + $null = $azureDb.Query("ALTER TABLE dbo.dbatoolsci_azure_temporal SET (SYSTEM_VERSIONING = OFF)") + $null = $azureDb.Query("DROP TABLE dbo.dbatoolsci_azure_temporal") + $null = $azureDb.Query("DROP TABLE IF EXISTS dbo.dbatoolsci_azure_temporal_history") + } + } + if ($serverAzure) { + $null = $serverAzure | Disconnect-DbaInstance + } + + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + It "really is an Azure SQL Database, which is what makes this case different" { + # Guards the test below: against a SQL Server 2016 or later it would pass without proving + # anything, because there the version check alone already excludes the period columns. + $serverAzure.DatabaseEngineEdition | Should -Be "SqlDatabase" + $serverAzure.Version.Major | Should -Be 12 + } + + It "Does not count the generated always columns of a temporal destination on Azure SQL Database" { + $splatAzureCopy = @{ + SqlInstance = $TestConfig.InstanceCopy1 + Database = "tempdb" + Table = "dbatoolsci_azure_source" + Query = "SELECT Id, A, B, C FROM dbo.dbatoolsci_azure_source" + Destination = $TestConfig.AzureSqlDbServer + DestinationSqlCredential = $TestConfig.AzureSqlDbCred + DestinationDatabase = $TestConfig.AzureSqlDbName + DestinationTable = "dbatoolsci_azure_temporal" + } + $result = Copy-DbaDbTableData @splatAzureCopy + $WarnVar | Should -BeNullOrEmpty + $result.RowsCopied | Should -Be 2 + + $destData = $azureDb.Query("SELECT Id, A, ValidFrom, B, ValidTo, C FROM dbo.dbatoolsci_azure_temporal ORDER BY Id") + $destData.A | Should -Be @(11, 111) + $destData.B | Should -Be @(22, 222) + $destData.C | Should -Be @(33, 333) + $destData.ValidFrom | Should -Not -BeNullOrEmpty + } + } + Context "Regression tests" { BeforeAll { $PSDefaultParameterValues["*-Dba*:EnableException"] = $true