diff --git a/private/functions/Get-BulkRowsCopiedCount.ps1 b/private/functions/Get-BulkRowsCopiedCount.ps1 index a5d4c21c2e9..4ffbc78416c 100644 --- a/private/functions/Get-BulkRowsCopiedCount.ps1 +++ b/private/functions/Get-BulkRowsCopiedCount.ps1 @@ -14,6 +14,7 @@ function Get-BulkRowsCopiedCount { - Copy-DbaDbTableData - Copy-DbaDbViewData - Import-DbaCsv + - Import-DbaParquet .EXAMPLE Get-BulkRowsCopied $bulkObject @@ -27,14 +28,17 @@ function Get-BulkRowsCopiedCount { Copyright: (c) 2020 by dbatools, licensed under MIT License: MIT https://opensource.org/licenses/MIT #> - [OutputType([int])] + [OutputType([long])] param ( [Microsoft.Data.SqlClient.SqlBulkCopy] $BulkCopy ) $BindingFlags = [Reflection.BindingFlags] "NonPublic,GetField,Instance" $rowsCopiedField = [Microsoft.Data.SqlClient.SqlBulkCopy].GetField("_rowsCopied", $BindingFlags) try { - return [int]$rowsCopiedField.GetValue($BulkCopy) + # The field is an Int64 in current Microsoft.Data.SqlClient (an Int32 in the legacy library), + # so it must not be narrowed to [int]: above [int32]::MaxValue rows that cast throws, and the + # -1 then taken for the row count inflates the reported total by billions of rows (see #10675). + return [long]$rowsCopiedField.GetValue($BulkCopy) } catch { return -1; } diff --git a/public/Copy-DbaDbTableData.ps1 b/public/Copy-DbaDbTableData.ps1 index 3ed8b1aba75..ae6f220efff 100644 --- a/public/Copy-DbaDbTableData.ps1 +++ b/public/Copy-DbaDbTableData.ps1 @@ -722,7 +722,13 @@ function Copy-DbaDbTableData { $bulkCopy.WriteToServer($reader) $finalRowCountReported = Get-BulkRowsCopiedCount $bulkCopy - $script:totalRowsCopied += (Get-AdjustedTotalRowsCopied -ReportedRowsCopied $finalRowCountReported -PreviousRowsCopied $script:prevRowsCopied).NewRowCountAdded + # -1 signals that the reflection lookup failed, not a wrapped counter, so it must not + # reach Get-AdjustedTotalRowsCopied: fed in as a row count it inflates the total by + # billions of rows (see #10675). The running total from the notifications is then the + # best number available. + if ($finalRowCountReported -ge 0) { + $script:totalRowsCopied += (Get-AdjustedTotalRowsCopied -ReportedRowsCopied $finalRowCountReported -PreviousRowsCopied $script:prevRowsCopied).NewRowCountAdded + } $RowsTotal = $script:totalRowsCopied $TotalTime = [math]::Round($elapsed.Elapsed.TotalSeconds, 1) diff --git a/public/Import-DbaCsv.ps1 b/public/Import-DbaCsv.ps1 index eb5da19bbc3..d1f9ca2b211 100644 --- a/public/Import-DbaCsv.ps1 +++ b/public/Import-DbaCsv.ps1 @@ -1493,7 +1493,13 @@ WHERE c.object_id = OBJECT_ID(@tableName) $finalRowCountReported = Get-BulkRowsCopiedCount $bulkCopy - $script:totalRowsCopied += (Get-AdjustedTotalRowsCopied -ReportedRowsCopied $finalRowCountReported -PreviousRowsCopied $script:prevRowsCopied).NewRowCountAdded + # -1 signals that the reflection lookup failed, not a wrapped counter, so it must not + # reach Get-AdjustedTotalRowsCopied: fed in as a row count it inflates the total by + # billions of rows (see #10675). The running total from the notifications is then the + # best number available. + if ($finalRowCountReported -ge 0) { + $script:totalRowsCopied += (Get-AdjustedTotalRowsCopied -ReportedRowsCopied $finalRowCountReported -PreviousRowsCopied $script:prevRowsCopied).NewRowCountAdded + } if ($completed) { Write-Progress -Id 1 -Activity "Inserting $($script:totalRowsCopied) rows" -Status "Complete" -Completed diff --git a/public/Import-DbaParquet.ps1 b/public/Import-DbaParquet.ps1 index c32fa0cdf74..a2086ed7f20 100644 --- a/public/Import-DbaParquet.ps1 +++ b/public/Import-DbaParquet.ps1 @@ -1031,7 +1031,13 @@ WHERE c.object_id = OBJECT_ID(@tableName) $finalRowCountReported = Get-BulkRowsCopiedCount $bulkCopy - $script:totalRowsCopied += (Get-AdjustedTotalRowsCopied -ReportedRowsCopied $finalRowCountReported -PreviousRowsCopied $script:prevRowsCopied).NewRowCountAdded + # -1 signals that the reflection lookup failed, not a wrapped counter, so it must not + # reach Get-AdjustedTotalRowsCopied: fed in as a row count it inflates the total by + # billions of rows (see #10675). The running total from the notifications is then the + # best number available. + if ($finalRowCountReported -ge 0) { + $script:totalRowsCopied += (Get-AdjustedTotalRowsCopied -ReportedRowsCopied $finalRowCountReported -PreviousRowsCopied $script:prevRowsCopied).NewRowCountAdded + } if ($completed) { Write-Progress -Id 1 -Activity "Inserting $($script:totalRowsCopied) rows" -Status "Complete" -Completed diff --git a/tests/Get-BulkRowsCopiedCount.Tests.ps1 b/tests/Get-BulkRowsCopiedCount.Tests.ps1 new file mode 100644 index 00000000000..140e367eb57 --- /dev/null +++ b/tests/Get-BulkRowsCopiedCount.Tests.ps1 @@ -0,0 +1,52 @@ +#Requires -Module @{ ModuleName="Pester"; ModuleVersion="5.0" } +param( + $ModuleName = "dbatools", + $CommandName = "Get-BulkRowsCopiedCount", + $PSDefaultParameterValues = $TestConfig.Defaults +) + +Describe $CommandName -Tag UnitTests { + Context "Reading the rows copied counter" { + It "Returns the counter value below Int32.MaxValue" { + InModuleScope "dbatools" { + $bulkCopy = New-Object -TypeName Microsoft.Data.SqlClient.SqlBulkCopy -ArgumentList "Server=dummy" + $rowsCopiedField = [Microsoft.Data.SqlClient.SqlBulkCopy].GetField("_rowsCopied", [Reflection.BindingFlags]"NonPublic,GetField,Instance") + $rowsCopiedField.SetValue($bulkCopy, [long]100000) + + Get-BulkRowsCopiedCount $bulkCopy | Should -Be 100000 + } + } + + It "Returns the counter value above Int32.MaxValue" { + # The field is an Int64 in current Microsoft.Data.SqlClient. A cast to [int] throws above + # [int32]::MaxValue, the catch then returned -1, and the callers took that for a wrapped + # counter and inflated the reported total by billions of rows (#10675). + InModuleScope "dbatools" { + $bulkCopy = New-Object -TypeName Microsoft.Data.SqlClient.SqlBulkCopy -ArgumentList "Server=dummy" + $rowsCopiedField = [Microsoft.Data.SqlClient.SqlBulkCopy].GetField("_rowsCopied", [Reflection.BindingFlags]"NonPublic,GetField,Instance") + $rowsCopiedField.SetValue($bulkCopy, [long]3000000000) + + Get-BulkRowsCopiedCount $bulkCopy | Should -Be 3000000000 + } + } + + It "Keeps the total of the callers correct above Int32.MaxValue" { + # The pattern of Copy-DbaDbTableData, Import-DbaCsv and Import-DbaParquet: the running total + # from the copy notifications, plus the adjustment for the rows after the last notification. + InModuleScope "dbatools" { + $bulkCopy = New-Object -TypeName Microsoft.Data.SqlClient.SqlBulkCopy -ArgumentList "Server=dummy" + $rowsCopiedField = [Microsoft.Data.SqlClient.SqlBulkCopy].GetField("_rowsCopied", [Reflection.BindingFlags]"NonPublic,GetField,Instance") + $rowsCopiedField.SetValue($bulkCopy, [long]3000000000) + + $totalRowsCopied = [long]2999995000 + $prevRowsCopied = [long]2999995000 + $finalRowCountReported = Get-BulkRowsCopiedCount $bulkCopy + if ($finalRowCountReported -ge 0) { + $totalRowsCopied += (Get-AdjustedTotalRowsCopied -ReportedRowsCopied $finalRowCountReported -PreviousRowsCopied $prevRowsCopied).NewRowCountAdded + } + + $totalRowsCopied | Should -Be 3000000000 + } + } + } +}