diff --git a/.github/workflows/test-avoidconflicts.yml b/.github/workflows/test-avoidconflicts.yml new file mode 100644 index 00000000..d9eb6621 --- /dev/null +++ b/.github/workflows/test-avoidconflicts.yml @@ -0,0 +1,441 @@ +name: Test AvoidConflicts Parameter + +on: + push: + branches: [main] + paths: + - 'dbatools.library.psm1' + - 'dbatools.library.psd1' + - 'tests/test-avoidconflicts.ps1' + - '.github/workflows/test-avoidconflicts.yml' + pull_request: + branches: [main] + paths: + - 'dbatools.library.psm1' + - 'dbatools.library.psd1' + - 'tests/test-avoidconflicts.ps1' + - '.github/workflows/test-avoidconflicts.yml' + workflow_dispatch: + +defaults: + run: + shell: pwsh + +jobs: + build: + name: Build Library + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 8.0.x + 6.0.x + + - name: Install .NET Framework targeting packs + shell: pwsh + run: | + choco install netfx-4.7.2-devpack -y --no-progress + + - name: Build the library + run: ./build/build.ps1 + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: dbatools-library + path: artifacts/dbatools.library/ + + test-pwsh-core: + name: PowerShell Core - AvoidConflicts Tests + runs-on: windows-latest + needs: build + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: dbatools-library + path: artifacts/dbatools.library/ + + - name: Install SqlServer module + run: | + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module SqlServer -Force -AllowClobber -Scope CurrentUser + Write-Host "SqlServer module installed: $((Get-Module -ListAvailable SqlServer).Version)" + + - name: Test 1 - Default import without SqlServer (baseline) + run: | + Write-Host "=== Test 1: Default import without SqlServer ===" -ForegroundColor Cyan + Write-Host "PowerShell: $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" + + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -Force -ErrorAction Stop + + $sqlClient = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + + if ($sqlClient) { + Write-Host "✅ PASS: SqlClient loaded: $($sqlClient.GetName().Version)" -ForegroundColor Green + } else { + Write-Host "❌ FAIL: SqlClient not loaded" -ForegroundColor Red + exit 1 + } + + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Host "✅ PASS: SMO types available" -ForegroundColor Green + } else { + Write-Host "❌ FAIL: SMO types not available" -ForegroundColor Red + exit 1 + } + + - name: Test 2 - SqlServer first WITHOUT AvoidConflicts (expect failure) + run: | + Write-Host "=== Test 2: SqlServer first, WITHOUT AvoidConflicts ===" -ForegroundColor Cyan + Write-Host "Expected: FAIL due to assembly conflict" + + # Run in isolated process to avoid assembly caching + $result = pwsh -NoProfile -Command { + try { + Import-Module SqlServer -ErrorAction Stop + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -Force -ErrorAction Stop + Write-Output "UNEXPECTED_PASS" + } catch { + if ($_.Exception.Message -match "already loaded|Assembly with same name") { + Write-Output "EXPECTED_FAIL" + } else { + Write-Output "FAIL: $($_.Exception.Message)" + } + } + } + + Write-Host "Result: $result" + + if ($result -eq "EXPECTED_FAIL") { + Write-Host "✅ PASS: Correctly fails with conflict (expected behavior)" -ForegroundColor Green + } elseif ($result -eq "UNEXPECTED_PASS") { + Write-Host "⚠️ INFO: No conflict occurred (assemblies may be compatible)" -ForegroundColor Yellow + } else { + Write-Host "❌ FAIL: Unexpected result: $result" -ForegroundColor Red + exit 1 + } + + - name: Test 3 - SqlServer first WITH AvoidConflicts (must succeed) + run: | + Write-Host "=== Test 3: SqlServer first, WITH AvoidConflicts ===" -ForegroundColor Cyan + Write-Host "Expected: SUCCESS - AvoidConflicts resolves version mismatches" + + # Run in isolated process + $result = pwsh -NoProfile -Command { + try { + # Import SqlServer first + Import-Module SqlServer -ErrorAction Stop + + # Get assemblies loaded by SqlServer + $sqlClientBefore = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + $connectionInfoBefore = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.SqlServer.ConnectionInfo' } + + Write-Host "SqlServer loaded SqlClient: $($sqlClientBefore.GetName().Version)" -ForegroundColor Gray + Write-Host "SqlServer loaded ConnectionInfo: $($connectionInfoBefore.GetName().Version)" -ForegroundColor Gray + + # Import dbatools.library with AvoidConflicts + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -ArgumentList $true -Force -ErrorAction Stop + + # Verify SMO types work + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Output "PASS" + } else { + Write-Output "FAIL: SMO types not available" + } + } catch { + Write-Output "FAIL: $($_.Exception.Message)" + } + } + + Write-Host "Result: $result" + + if ($result -eq "PASS") { + Write-Host "✅ PASS: Module loaded successfully with AvoidConflicts" -ForegroundColor Green + } else { + Write-Host "❌ FAIL: $result" -ForegroundColor Red + exit 1 + } + + - name: Test 4 - Verify AssemblyLoadContext resolver works + run: | + Write-Host "=== Test 4: AssemblyLoadContext resolver for version mismatches ===" -ForegroundColor Cyan + Write-Host "This tests the Core-specific fix for assembly version resolution" + + $result = pwsh -NoProfile -Command { + try { + Import-Module SqlServer -ErrorAction Stop + + # Capture verbose output to verify resolver messages + $verboseOutput = @() + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -ArgumentList $true -Force -Verbose -ErrorAction Stop 4>&1 | + ForEach-Object { + if ($_ -is [System.Management.Automation.VerboseRecord]) { + $verboseOutput += $_.Message + } + } + + # Check that both modules are loaded and functional + $sqlServer = Get-Module SqlServer + $dbatools = Get-Module dbatools.library + + if ($sqlServer -and $dbatools) { + # Test that we can access types from dbatools.library + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Output "PASS" + } else { + Write-Output "FAIL: SMO types not accessible" + } + } else { + Write-Output "FAIL: Not all modules loaded (SqlServer: $($null -ne $sqlServer), dbatools.library: $($null -ne $dbatools))" + } + } catch { + Write-Output "FAIL: $($_.Exception.Message)" + } + } + + Write-Host "Result: $result" + + if ($result -eq "PASS") { + Write-Host "✅ PASS: AssemblyLoadContext resolver working correctly" -ForegroundColor Green + } else { + Write-Host "❌ FAIL: $result" -ForegroundColor Red + exit 1 + } + + - name: Test 5 - Verify correct syntax documentation + run: | + Write-Host "=== Test 5: ArgumentList syntax validation ===" -ForegroundColor Cyan + + # Test that hashtable syntax is rejected with helpful error + $result = pwsh -NoProfile -Command { + try { + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -ArgumentList @{AvoidConflicts=$true} -Force -ErrorAction Stop + Write-Output "UNEXPECTED_PASS" + } catch { + if ($_.Exception.Message -match "Cannot convert.*Hashtable.*SwitchParameter|Boolean parameters") { + Write-Output "EXPECTED_ERROR" + } else { + Write-Output "UNEXPECTED_ERROR: $($_.Exception.Message)" + } + } + } + + Write-Host "Result: $result" + + if ($result -eq "EXPECTED_ERROR") { + Write-Host "✅ PASS: Hashtable syntax correctly rejected (use -ArgumentList `$true instead)" -ForegroundColor Green + } else { + Write-Host "❌ FAIL: $result" -ForegroundColor Red + exit 1 + } + + test-windows-powershell: + name: Windows PowerShell 5.1 - AvoidConflicts Tests + runs-on: windows-latest + needs: build + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: dbatools-library + path: artifacts/dbatools.library/ + + - name: Install SqlServer module + shell: powershell + run: | + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module SqlServer -Force -AllowClobber -Scope CurrentUser + Write-Host "SqlServer module installed: $((Get-Module -ListAvailable SqlServer).Version)" + + - name: Test 1 - Default import without SqlServer (baseline) + shell: powershell + run: | + Write-Host "=== Test 1: Default import without SqlServer ===" -ForegroundColor Cyan + Write-Host "PowerShell: $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" + + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -Force -ErrorAction Stop + + $sqlClient = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + + if ($sqlClient) { + Write-Host "[PASS] SqlClient loaded: $($sqlClient.GetName().Version)" -ForegroundColor Green + } else { + Write-Host "[FAIL] SqlClient not loaded" -ForegroundColor Red + exit 1 + } + + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Host "[PASS] SMO types available" -ForegroundColor Green + } else { + Write-Host "[FAIL] SMO types not available" -ForegroundColor Red + exit 1 + } + + - name: Test 2 - SqlServer first WITH AvoidConflicts (must succeed) + shell: powershell + run: | + Write-Host "=== Test 2: SqlServer first, WITH AvoidConflicts ===" -ForegroundColor Cyan + Write-Host "PowerShell: $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" + + # Import SqlServer first + Import-Module SqlServer -ErrorAction Stop + Write-Host "SqlServer loaded" -ForegroundColor Green + + # Get assemblies loaded by SqlServer + $sqlClientBefore = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + $connectionInfoBefore = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.SqlServer.ConnectionInfo' } + + if ($sqlClientBefore) { + Write-Host "SqlServer loaded SqlClient: $($sqlClientBefore.GetName().Version)" -ForegroundColor Yellow + } + if ($connectionInfoBefore) { + Write-Host "SqlServer loaded ConnectionInfo: $($connectionInfoBefore.GetName().Version)" -ForegroundColor Yellow + } + + # Import dbatools.library with AvoidConflicts + Write-Host "`nImporting dbatools.library with -AvoidConflicts..." -ForegroundColor Cyan + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -ArgumentList $true -Force -Verbose -ErrorAction Stop + + Write-Host "dbatools.library loaded successfully" -ForegroundColor Green + + # Verify SMO types work + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Host "[PASS] SMO types available" -ForegroundColor Green + } else { + Write-Host "[FAIL] SMO types not available" -ForegroundColor Red + exit 1 + } + + # Verify both modules are loaded + $loadedModules = Get-Module | Where-Object { $_.Name -in 'SqlServer', 'dbatools.library' } + Write-Host "`nLoaded modules:" -ForegroundColor Cyan + $loadedModules | ForEach-Object { Write-Host " - $($_.Name) v$($_.Version)" -ForegroundColor Gray } + + if ($loadedModules.Count -eq 2) { + Write-Host "`n[PASS] Both modules loaded successfully!" -ForegroundColor Green + } else { + Write-Host "`n[FAIL] Expected 2 modules, got $($loadedModules.Count)" -ForegroundColor Red + exit 1 + } + + - name: Test 3 - Verify Redirector handles version mismatches + shell: powershell + run: | + Write-Host "=== Test 3: Redirector for version mismatches (Desktop-specific) ===" -ForegroundColor Cyan + Write-Host "This tests the Desktop PowerShell Redirector class" + + # Start fresh process + powershell -NoProfile -Command { + try { + Import-Module SqlServer -ErrorAction Stop + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -ArgumentList $true -Force -ErrorAction Stop + + # Test that types work after potential version mismatch resolution + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Output "PASS" + } else { + Write-Output "FAIL: SMO types not accessible" + } + } catch { + Write-Output "FAIL: $($_.Exception.Message)" + } + } | ForEach-Object { + $result = $_ + Write-Host "Result: $result" + + if ($result -eq "PASS") { + Write-Host "[PASS] Redirector working correctly" -ForegroundColor Green + } else { + Write-Host "[FAIL] $result" -ForegroundColor Red + exit 1 + } + } + + test-comprehensive: + name: Comprehensive Test Suite + runs-on: windows-latest + needs: build + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: dbatools-library + path: artifacts/dbatools.library/ + + - name: Install SqlServer module (PowerShell Core) + run: | + Set-PSRepository -Name PSGallery -InstallationPolicy Trusted + Install-Module SqlServer -Force -AllowClobber -Scope CurrentUser + + - name: Run test-avoidconflicts.ps1 test suite + run: | + Write-Host "Running comprehensive test suite..." -ForegroundColor Cyan + ./tests/test-avoidconflicts.ps1 + if ($LASTEXITCODE -ne 0) { + Write-Host "Test suite failed!" -ForegroundColor Red + exit 1 + } + + summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [test-pwsh-core, test-windows-powershell, test-comprehensive] + if: always() + steps: + - name: Check test results + shell: bash + run: | + echo "## AvoidConflicts Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.test-pwsh-core.result }}" == "success" ]; then + echo "✅ PowerShell Core tests: PASSED" >> $GITHUB_STEP_SUMMARY + else + echo "❌ PowerShell Core tests: FAILED" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ needs.test-windows-powershell.result }}" == "success" ]; then + echo "✅ Windows PowerShell 5.1 tests: PASSED" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Windows PowerShell 5.1 tests: FAILED" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ needs.test-comprehensive.result }}" == "success" ]; then + echo "✅ Comprehensive test suite: PASSED" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Comprehensive test suite: FAILED" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY + echo "- Default module import (no conflicts)" >> $GITHUB_STEP_SUMMARY + echo "- SqlServer + dbatools.library with AvoidConflicts" >> $GITHUB_STEP_SUMMARY + echo "- Assembly version mismatch resolution" >> $GITHUB_STEP_SUMMARY + echo "- ArgumentList syntax validation" >> $GITHUB_STEP_SUMMARY + echo "- Both PowerShell Core and Windows PowerShell 5.1" >> $GITHUB_STEP_SUMMARY + + # Fail if any test failed + if [ "${{ needs.test-pwsh-core.result }}" != "success" ] || \ + [ "${{ needs.test-windows-powershell.result }}" != "success" ] || \ + [ "${{ needs.test-comprehensive.result }}" != "success" ]; then + echo "" + echo "Some tests failed!" + exit 1 + fi diff --git a/README.md b/README.md index 7795d32b..355c7033 100644 --- a/README.md +++ b/README.md @@ -60,19 +60,22 @@ If you need to use both the SqlServer module and dbatools.library in the same se ### Usage -Import the SqlServer module first, then import dbatools.library with `-AvoidConflicts`: +Import the SqlServer module first, then import dbatools.library **directly** with `-AvoidConflicts`: ```powershell # Import SqlServer module first Import-Module SqlServer # Then import dbatools.library with -AvoidConflicts +# IMPORTANT: Use -ArgumentList $true (NOT a hashtable) Import-Module dbatools.library -ArgumentList $true ``` +> **Note:** You must import `dbatools.library` directly, not via the `dbatools` module. The `-ArgumentList` parameter cannot be passed through module dependencies. If you need to use the full `dbatools` module with SqlServer, import `dbatools.library` first with `-ArgumentList $true`, then import `dbatools`. + When `-AvoidConflicts` is enabled, dbatools.library will: - Check if each assembly is already loaded in the current session -- Skip loading any assemblies that are already present +- Skip loading any assemblies that are already present (including Microsoft.Data.SqlClient) - Load only the assemblies that are missing ### Examples @@ -83,6 +86,13 @@ Import-Module SqlServer Import-Module dbatools.library -ArgumentList $true ``` +**Using with the full dbatools module:** +```powershell +Import-Module SqlServer +Import-Module dbatools.library -ArgumentList $true # Load library first with AvoidConflicts +Import-Module dbatools # Then load dbatools (will use already-loaded library) +``` + **See what's being skipped with -Verbose:** ```powershell Import-Module SqlServer @@ -96,6 +106,18 @@ Import-Module dbatools.library -ArgumentList $true -Verbose Import-Module dbatools.library ``` +### Common Mistakes + +❌ **Wrong:** Using a hashtable for ArgumentList +```powershell +Import-Module dbatools.library -ArgumentList @{AvoidConflicts = $true} # This will NOT work +``` + +✅ **Correct:** Using a boolean value +```powershell +Import-Module dbatools.library -ArgumentList $true # This works correctly +``` + ### ⚠️ Important: PowerShell Core + Credentials Issue **If you plan to use SQL Server credentials with PowerShell Core (pwsh), you MUST install to AllUsers scope or grant appropriate permissions.** diff --git a/dbatools.library.psd1 b/dbatools.library.psd1 index ab9b1c7f..962e6829 100644 --- a/dbatools.library.psd1 +++ b/dbatools.library.psd1 @@ -7,7 +7,7 @@ # @{ # Version number of this module. - ModuleVersion = '2025.12.21' + ModuleVersion = '2025.12.26' # ID used to uniquely identify this module GUID = '00b61a37-6c36-40d8-8865-ac0180288c84' diff --git a/dbatools.library.psm1 b/dbatools.library.psm1 index 36fab144..99cdacd7 100644 --- a/dbatools.library.psm1 +++ b/dbatools.library.psm1 @@ -17,55 +17,26 @@ function Get-DbatoolsLibraryPath { $script:libraryroot = Get-DbatoolsLibraryPath if ($PSVersionTable.PSEdition -ne "Core") { - $dir = [System.IO.Path]::Combine($script:libraryroot, "lib") - $dir = ("$dir\").Replace('\', '\\') - if (-not ("Redirector" -as [type])) { $source = @" using System; - using System.Linq; + using System.IO; using System.Reflection; - using System.Text.RegularExpressions; public class Redirector { - public Redirector() + private static string _libPath; + + public Redirector(string libPath) { + _libPath = libPath; this.EventHandler = new ResolveEventHandler(AssemblyResolve); } public readonly ResolveEventHandler EventHandler; - protected Assembly AssemblyResolve(object sender, ResolveEventArgs e) + protected static Assembly AssemblyResolve(object sender, ResolveEventArgs e) { - string[] dlls = { - "System.Memory", - "System.Runtime", - "System.Management.Automation", - "System.Runtime.CompilerServices.Unsafe", - "Microsoft.Bcl.AsyncInterfaces", - "System.Text.Json", - "System.Resources.Extensions", - "Microsoft.SqlServer.ConnectionInfo", - "Microsoft.SqlServer.Smo", - "Microsoft.Identity.Client", - "System.Diagnostics.DiagnosticSource", - "Microsoft.IdentityModel.Abstractions", - "Microsoft.Data.SqlClient", - "Microsoft.SqlServer.Types", - "System.Configuration.ConfigurationManager", - "Microsoft.SqlServer.Management.Sdk.Sfc", - "Microsoft.SqlServer.Management.IntegrationServices", - "Microsoft.SqlServer.Replication", - "Microsoft.SqlServer.Rmo", - "System.Private.CoreLib", - "Azure.Core", - "Azure.Identity", - "Microsoft.Data.Tools.Utilities", - "Microsoft.Data.Tools.Schema.Sql", - "Microsoft.SqlServer.TransactSql.ScriptDom" - }; - var requestedName = new AssemblyName(e.Name); var assemblyName = requestedName.Name; @@ -86,13 +57,17 @@ if ($PSVersionTable.PSEdition -ne "Core") { } } - // Only load from disk if not already loaded and it's in our list - foreach (string dll in dlls) + // Try to load from our lib folder if the file exists + string dllPath = Path.Combine(_libPath, assemblyName + ".dll"); + if (File.Exists(dllPath)) { - if (assemblyName == dll) + try { - string filelocation = "$dir" + dll + ".dll"; - return Assembly.LoadFrom(filelocation); + return Assembly.LoadFrom(dllPath); + } + catch + { + // Failed to load, return null to let default resolution continue } } @@ -105,10 +80,90 @@ if ($PSVersionTable.PSEdition -ne "Core") { } try { - $redirector = New-Object Redirector + $libPath = [System.IO.Path]::Combine($script:libraryroot, "lib") + $redirector = New-Object Redirector($libPath) [System.AppDomain]::CurrentDomain.add_AssemblyResolve($redirector.EventHandler) } catch { - # unsure + Write-Verbose "Could not register Redirector: $_" + } +} else { + # PowerShell Core: Use AssemblyLoadContext.Resolving event for version redirection + # This handles version mismatches when SqlServer module loads different versions of assemblies + # IMPORTANT: Must be implemented in C# because the resolver runs on .NET threads without PowerShell runspaces + $dir = [System.IO.Path]::Combine($script:libraryroot, "lib") + $dir = ("$dir" + [System.IO.Path]::DirectorySeparatorChar).Replace('\', '\\') + + if (-not ("CoreRedirector" -as [type])) { + $coreSource = @" + using System; + using System.IO; + using System.Reflection; + using System.Runtime.Loader; + + public class CoreRedirector + { + private static string _libPath; + private static bool _registered = false; + + public static void Register(string libPath) + { + if (_registered) return; + _libPath = libPath; + AssemblyLoadContext.Default.Resolving += OnResolving; + _registered = true; + } + + private static Assembly OnResolving(AssemblyLoadContext context, AssemblyName assemblyName) + { + string name = assemblyName.Name; + + // First, check if any version of this assembly is already loaded + // This handles version mismatches (e.g., dbatools.dll requesting ConnectionInfo 17.100.0.0 when 17.200.0.0 is loaded) + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + try + { + if (assembly.GetName().Name == name) + { + return assembly; + } + } + catch + { + // Some assemblies may throw when accessing GetName() + } + } + + // Try to load from our lib folder if the file exists + string dllPath = _libPath + name + ".dll"; + if (File.Exists(dllPath)) + { + try + { + return AssemblyLoadContext.Default.LoadFromAssemblyPath(dllPath); + } + catch + { + // Failed to load, return null to let default resolution continue + } + } + + return null; + } + } +"@ + + try { + $null = Add-Type -TypeDefinition $coreSource -ReferencedAssemblies 'System.Runtime.Loader' + } catch { + Write-Verbose "Could not compile CoreRedirector: $_" + } + } + + try { + [CoreRedirector]::Register($dir) + } catch { + Write-Verbose "Could not register CoreRedirector: $_" } } @@ -118,11 +173,44 @@ $sqlclient = [System.IO.Path]::Combine($script:libraryroot, "lib", "Microsoft.Da # Get loaded assemblies once for reuse (used for AvoidConflicts checks and later assembly loading) $script:loadedAssemblies = [System.AppDomain]::CurrentDomain.GetAssemblies() +# Check for incompatible System.ClientModel or SqlClient versions +# Azure.Core 1.44+ requires System.ClientModel 1.1+ with IPersistableModel.Write() method +# SqlServer module's SqlClient 5.x includes older System.ClientModel that's incompatible +$script:hasIncompatibleClientModel = $false +if ($AvoidConflicts) { + # Check if System.ClientModel is already loaded with incompatible version + $existingClientModel = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'System.ClientModel' } + if ($existingClientModel) { + $clientModelVersion = $existingClientModel.GetName().Version + # System.ClientModel 1.1.0+ has the required IPersistableModel interface changes + if ($clientModelVersion -lt [Version]'1.1.0') { + $script:hasIncompatibleClientModel = $true + Write-Verbose "Detected incompatible System.ClientModel version $clientModelVersion - will skip Azure.Core and Azure.Identity to avoid MissingMethodException" + } + } + + # Check if SqlServer's older SqlClient is loaded (which bundles incompatible System.ClientModel) + # SqlClient 5.x from SqlServer module uses System.ClientModel 1.0.x + # Our Azure.Core requires System.ClientModel 1.1+ + if (-not $script:hasIncompatibleClientModel) { + $existingSqlClient = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + if ($existingSqlClient) { + $sqlClientVersion = $existingSqlClient.GetName().Version + # SqlClient 5.x bundles older System.ClientModel; 6.x bundles compatible versions + if ($sqlClientVersion.Major -lt 6) { + $script:hasIncompatibleClientModel = $true + Write-Verbose "Detected SqlClient $sqlClientVersion (pre-6.0) which uses incompatible System.ClientModel - will skip Azure.Core and Azure.Identity to avoid MissingMethodException" + } + } + } +} + # Check if SqlClient is already loaded when AvoidConflicts is set $skipSqlClient = $false if ($AvoidConflicts) { - $skipSqlClient = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } - if ($skipSqlClient) { + $existingAssembly = $script:loadedAssemblies | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + if ($existingAssembly) { + $skipSqlClient = $true Write-Verbose "Skipping Microsoft.Data.SqlClient.dll - already loaded" } } @@ -210,6 +298,13 @@ foreach ($name in $names) { continue } + # Skip Azure.Core and Azure.Identity if System.ClientModel is incompatible + # These assemblies depend on System.ClientModel 1.1+ which has breaking API changes + if ($script:hasIncompatibleClientModel -and $name -in @('Azure.Core', 'Azure.Identity')) { + Write-Verbose "Skipping $name.dll - incompatible System.ClientModel already loaded" + continue + } + # Check if assembly is already loaded (always check to avoid duplicate loads) if ($script:loadedAssemblyNames.Contains("$name,")) { if ($AvoidConflicts) { diff --git a/project/Dataplat.Dbatools.Csv/CHANGELOG.md b/project/Dataplat.Dbatools.Csv/CHANGELOG.md index d621992f..f66c0df4 100644 --- a/project/Dataplat.Dbatools.Csv/CHANGELOG.md +++ b/project/Dataplat.Dbatools.Csv/CHANGELOG.md @@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.10] - 2025-12-26 + +### Added +- **SQL Server schema inference** - New `CsvSchemaInference` class that analyzes CSV data to determine optimal SQL Server column types. Two modes available: + - `InferSchemaFromSample()` - Fast inference from first N rows (default 1000) + - `InferSchema()` - Full file scan with progress callback for zero-risk type detection +- `InferredColumn` class containing column name, SQL data type, max length, nullability, unicode flag, and decimal precision/scale +- Type detection for: `uniqueidentifier`, `bit`, `int`, `bigint`, `decimal(p,s)`, `datetime2`, `varchar(n)`, `nvarchar(n)` +- `GenerateCreateTableStatement()` utility to produce SQL DDL from inferred schema +- `ToColumnTypes()` utility to convert inferred schema to `CsvReaderOptions.ColumnTypes` dictionary +- Early exit optimization: types are eliminated as values fail validation, reducing unnecessary checks +- Progress callback support for full-scan mode (fires every ~1% or 10K rows) +- **MoneyConverter** for SQL Server `money`/`smallmoney` types with support for currency symbols, thousands separators, and accounting format +- **VectorConverter** for SQL Server 2025 `VECTOR` data type with support for JSON array and comma-separated formats + +### Fixed +- **DecimalConverter scientific notation support** - Changed `NumberStyles` from `Number` to `Float | AllowThousands` to properly parse scientific notation (e.g., `1.2345678E5`) +- **SQL identifier escaping** - Escape closing brackets in schema, table, and column names to prevent SQL injection in generated `CREATE TABLE` statements + ## [1.1.1] - 2025-12-04 ### Changed diff --git a/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj b/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj index 39b72bb6..b0ef36cf 100644 --- a/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj +++ b/project/Dataplat.Dbatools.Csv/Dataplat.Dbatools.Csv.csproj @@ -7,20 +7,20 @@ Dataplat.Dbatools.Csv - 1.1.1 + 1.1.10 Chrissy LeMaire Dataplat Dataplat.Dbatools.Csv - High-performance CSV reader and writer for .NET. Features streaming IDataReader for SqlBulkCopy, automatic compression (GZip, Deflate, Brotli, ZLib), multi-character delimiters, parallel processing, string interning, and robust error handling. 20%+ faster than LumenWorks CsvReader. From the trusted dbatools project. + High-performance CSV reader with native IDataReader for SqlBulkCopy - 6x faster than legacy solutions for database imports. Database-first design with culture-aware parsing, intelligent null handling, and robust support for messy real-world data (duplicate headers, field mismatches). Features automatic compression (GZip, Deflate, Brotli, ZLib), progress reporting with rows/second metrics, and cancellation support. From the trusted dbatools project. Copyright (c) 2025 Chrissy LeMaire csv;parser;reader;writer;datareader;idatareader;sqlbulkcopy;compression;gzip;brotli;dbatools;high-performance;parallel MIT - https://github.com/dataplat/dbatools.library + https://dataplat.dbatools.io/csv https://github.com/dataplat/dbatools.library git main README.md - Initial release with high-performance CSV parsing, parallel processing support, and comprehensive edge case handling. + v1.1.10: SQL Server schema inference - auto-detect column types (int, bigint, decimal, datetime2, bit, uniqueidentifier, varchar/nvarchar). v1.1.5: Updated package metadata and URL. v1.1.1: ~25% performance improvement for all-columns reads. v1.1.0: Added CancellationToken and progress reporting support. true true false diff --git a/project/Dataplat.Dbatools.Csv/README.md b/project/Dataplat.Dbatools.Csv/README.md index 89498511..f881a131 100644 --- a/project/Dataplat.Dbatools.Csv/README.md +++ b/project/Dataplat.Dbatools.Csv/README.md @@ -8,6 +8,7 @@ **What makes this library unique:** - **Native IDataReader** - Stream directly to SqlBulkCopy with zero intermediate allocations +- **Schema Inference** - Auto-detect SQL Server column types (int, bigint, decimal, datetime2, bit, uniqueidentifier, varchar/nvarchar) - **Built-in compression** - GZip, Brotli, Deflate, ZLib with decompression bomb protection - **Real-world data handling** - Lenient parsing, smart quotes, duplicate headers, field count mismatches - **Faster than LumenWorks & CsvHelper** - ~1.5x faster with modern .NET (Span, ArrayPool) @@ -28,6 +29,8 @@ Install-Package Dataplat.Dbatools.Csv ## Features - **Streaming IDataReader** - Works seamlessly with SqlBulkCopy and other ADO.NET consumers +- **Schema Inference** - Analyze CSV data to determine optimal SQL Server column types +- **Strongly Typed Columns** - Define column types for automatic conversion with built-in and custom converters - **High Performance** - ~1.5x faster than LumenWorks/CsvHelper with ArrayPool-based memory management - **Parallel Processing** - Optional multi-threaded parsing for large files (25K+ rows/sec) - **String Interning** - Reduce memory for files with repeated values @@ -271,6 +274,130 @@ while (reader.Read()) } ``` +### Schema Inference + +Automatically detect optimal SQL Server column types from CSV data. No more `nvarchar(MAX)` for everything: + +```csharp +using Dataplat.Dbatools.Csv.Reader; + +// Fast: Sample first 1000 rows (tiny risk if data changes after sample) +var columns = CsvSchemaInference.InferSchemaFromSample("data.csv"); + +// Safe: Scan entire file with progress reporting (zero risk of type mismatches) +var columns = CsvSchemaInference.InferSchema("data.csv", null, progress => { + Console.WriteLine($"Progress: {progress:P0}"); +}); + +// Examine inferred types +foreach (var col in columns) +{ + Console.WriteLine($"{col.ColumnName}: {col.SqlDataType} {(col.IsNullable ? "NULL" : "NOT NULL")}"); +} +// Output: +// Id: int NOT NULL +// Name: nvarchar(100) NULL +// Price: decimal(10,2) NOT NULL +// Created: datetime2 NULL + +// Generate CREATE TABLE statement +string sql = CsvSchemaInference.GenerateCreateTableStatement(columns, "Products", "dbo"); +// CREATE TABLE [dbo].[Products] ( +// [Id] int NOT NULL, +// [Name] nvarchar(100) NULL, +// [Price] decimal(10,2) NOT NULL, +// [Created] datetime2 NULL +// ); + +// Use inferred types with CsvDataReader +var typeMap = CsvSchemaInference.ToColumnTypes(columns); +var options = new CsvReaderOptions { ColumnTypes = typeMap }; +using var reader = new CsvDataReader("data.csv", options); +``` + +**Detected types:** `uniqueidentifier`, `bit`, `int`, `bigint`, `decimal(p,s)`, `datetime2`, `varchar(n)`, `nvarchar(n)` (when Unicode is detected) + +**InferredColumn properties:** + +| Property | Type | Description | +|----------|------|-------------| +| `ColumnName` | string | Column header name | +| `SqlDataType` | string | SQL Server data type (e.g., `int`, `decimal(10,2)`, `nvarchar(50)`) | +| `IsNullable` | bool | True if any NULL/empty values were found | +| `IsUnicode` | bool | True if non-ASCII characters detected | +| `MaxLength` | int | Maximum string length observed | +| `Precision` | int | Decimal precision (total digits) | +| `Scale` | int | Decimal scale (digits after decimal point) | +| `Ordinal` | int | Column position (0-based) | +| `TotalCount` | long | Total rows analyzed | +| `NonNullCount` | long | Rows with non-null values | + +### Strongly Typed Columns + +Define column types explicitly for automatic conversion during reading: + +```csharp +var options = new CsvReaderOptions +{ + ColumnTypes = new Dictionary + { + ["Id"] = typeof(int), + ["Price"] = typeof(decimal), + ["IsActive"] = typeof(bool), + ["Created"] = typeof(DateTime), + ["UniqueId"] = typeof(Guid) + } +}; + +using var reader = new CsvDataReader("data.csv", options); +while (reader.Read()) +{ + int id = reader.GetInt32(0); // Already converted from string + decimal price = reader.GetDecimal(1); // Culture-aware parsing + bool active = reader.GetBoolean(2); // Handles true/false/yes/no/1/0 + DateTime created = reader.GetDateTime(3); + Guid guid = reader.GetGuid(4); +} +``` + +**Built-in type converters:** `Guid`, `bool`, `DateTime`, `short`, `int`, `long`, `float`, `double`, `decimal`, `byte`, `string` + +**Combine with schema inference:** + +```csharp +// Infer types from CSV data, then use them for reading +var columns = CsvSchemaInference.InferSchemaFromSample("data.csv"); +var typeMap = CsvSchemaInference.ToColumnTypes(columns); + +var options = new CsvReaderOptions { ColumnTypes = typeMap }; +using var reader = new CsvDataReader("data.csv", options); +``` + +**Custom type converters:** + +```csharp +using Dataplat.Dbatools.Csv.TypeConverters; + +// Create a custom converter for enums or custom types +public class StatusConverter : TypeConverterBase +{ + public override bool TryConvert(string value, out OrderStatus result) + { + return Enum.TryParse(value, true, out result); + } +} + +// Register and use +var registry = TypeConverterRegistry.Default; +registry.Register(new StatusConverter()); + +var options = new CsvReaderOptions +{ + TypeConverterRegistry = registry, + ColumnTypes = new Dictionary { ["Status"] = typeof(OrderStatus) } +}; +``` + ### Null vs Empty String Handling CSV files can represent missing data in two ways: an empty field (`,,`) or an explicitly quoted empty string (`,"",...`). The `DistinguishEmptyFromNull` option controls how these are interpreted. diff --git a/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs b/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs new file mode 100644 index 00000000..85f77e47 --- /dev/null +++ b/project/dbatools.Tests/Csv/CsvSchemaInferenceTest.cs @@ -0,0 +1,705 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Dataplat.Dbatools.Csv.Reader; + +namespace Dataplat.Dbatools.Csv.Tests +{ + [TestClass] + public class CsvSchemaInferenceTest + { + private string _tempDir; + + [TestInitialize] + public void Setup() + { + _tempDir = Path.Combine(Path.GetTempPath(), "CsvSchemaInferenceTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + } + + [TestCleanup] + public void Cleanup() + { + if (Directory.Exists(_tempDir)) + { + try { Directory.Delete(_tempDir, true); } catch { } + } + } + + #region File-Based Tests + + [TestMethod] + public void TestInferSchema_RealFile_MixedTypes() + { + string csvPath = Path.Combine(_tempDir, "mixed_types.csv"); + File.WriteAllText(csvPath, @"Id,Name,Price,Quantity,IsActive,Created,UniqueId +1,Widget A,19.99,100,true,2024-01-15,550e8400-e29b-41d4-a716-446655440000 +2,Widget B,29.50,50,false,2024-02-20,6ba7b810-9dad-11d1-80b4-00c04fd430c8 +3,Gadget C,99.00,25,yes,2024-03-25,f47ac10b-58cc-4372-a567-0e02b2c3d479 +4,Thing D,5.99,1000,no,2024-04-30,7c9e6679-7425-40de-944b-e07fc1f90ae7 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(7, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); // Id + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); // Name + Assert.IsTrue(columns[2].SqlDataType.StartsWith("decimal(")); // Price + Assert.AreEqual("int", columns[3].SqlDataType); // Quantity + Assert.AreEqual("bit", columns[4].SqlDataType); // IsActive + Assert.AreEqual("datetime2", columns[5].SqlDataType); // Created + Assert.AreEqual("uniqueidentifier", columns[6].SqlDataType); // UniqueId + } + + [TestMethod] + public void TestInferSchema_RealFile_LargeIntegers() + { + string csvPath = Path.Combine(_tempDir, "large_ints.csv"); + var sb = new StringBuilder(); + sb.AppendLine("SmallInt,RegularInt,BigInt,TooBig"); + sb.AppendLine("100,2000000000,9000000000000000000,99999999999999999999"); + sb.AppendLine("200,1500000000,8000000000000000000,88888888888888888888"); + File.WriteAllText(csvPath, sb.ToString()); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("int", columns[0].SqlDataType); // SmallInt fits in int + Assert.AreEqual("int", columns[1].SqlDataType); // RegularInt fits in int + Assert.AreEqual("bigint", columns[2].SqlDataType); // BigInt needs bigint + Assert.IsTrue(columns[3].SqlDataType.StartsWith("varchar(") || + columns[3].SqlDataType.StartsWith("decimal(")); // TooBig overflows + } + + [TestMethod] + public void TestInferSchema_RealFile_DecimalPrecision() + { + string csvPath = Path.Combine(_tempDir, "decimals.csv"); + File.WriteAllText(csvPath, @"Price,Tax,Total,Tiny +19.99,1.50,21.49,0.001 +199.99,15.00,214.99,0.002 +1999.99,150.00,2149.99,0.003 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // All should be decimal with appropriate precision + Assert.IsTrue(columns[0].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[2].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[3].SqlDataType.Contains("decimal")); + Assert.AreEqual(3, columns[3].Scale); // Tiny has 3 decimal places + } + + [TestMethod] + public void TestInferSchema_RealFile_NegativeNumbers() + { + string csvPath = Path.Combine(_tempDir, "negatives.csv"); + File.WriteAllText(csvPath, @"Temperature,Balance,Change +-10,1000.50,-5.25 +25,-500.00,10.00 +-40,-1234.56,-0.01 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("int", columns[0].SqlDataType); // Temperature - integers + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); // Balance + Assert.IsTrue(columns[2].SqlDataType.Contains("decimal")); // Change + } + + [TestMethod] + public void TestInferSchema_RealFile_UnicodeStrings() + { + string csvPath = Path.Combine(_tempDir, "unicode.csv"); + File.WriteAllText(csvPath, @"Name,City,Description +José García,São Paulo,Développeur senior +田中太郎,東京,ソフトウェアエンジニア +Müller,München,Geschäftsführer +", Encoding.UTF8); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsTrue(columns[0].SqlDataType.StartsWith("nvarchar(")); + Assert.IsTrue(columns[0].IsUnicode); + Assert.IsTrue(columns[1].SqlDataType.StartsWith("nvarchar(")); + Assert.IsTrue(columns[2].SqlDataType.StartsWith("nvarchar(")); + } + + [TestMethod] + public void TestInferSchema_RealFile_DateFormats() + { + string csvPath = Path.Combine(_tempDir, "dates.csv"); + File.WriteAllText(csvPath, @"ISO,US,WithTime +2024-01-15,01/15/2024,2024-01-15 14:30:00 +2024-02-20,02/20/2024,2024-02-20 09:15:30 +2024-03-25,03/25/2024,2024-03-25 18:45:00 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("datetime2", columns[0].SqlDataType); + Assert.AreEqual("datetime2", columns[1].SqlDataType); + Assert.AreEqual("datetime2", columns[2].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_RealFile_BooleanVariants() + { + string csvPath = Path.Combine(_tempDir, "booleans.csv"); + File.WriteAllText(csvPath, @"TrueFalse,YesNo,OnOff,TF,YN +true,yes,on,t,y +false,no,off,f,n +TRUE,YES,ON,T,Y +FALSE,NO,OFF,F,N +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + foreach (var col in columns) + { + Assert.AreEqual("bit", col.SqlDataType, $"Column {col.ColumnName} should be bit"); + } + } + + [TestMethod] + public void TestInferSchema_RealFile_NullableColumns() + { + string csvPath = Path.Combine(_tempDir, "nullable.csv"); + File.WriteAllText(csvPath, @"Id,Name,OptionalValue +1,John,100 +2,Jane, +3,,200 +4,Bob, +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsFalse(columns[0].IsNullable); // Id has all values + Assert.IsTrue(columns[1].IsNullable); // Name has empty + Assert.IsTrue(columns[2].IsNullable); // OptionalValue has empty + } + + [TestMethod] + public void TestInferSchema_RealFile_VeryLongStrings() + { + string csvPath = Path.Combine(_tempDir, "longstrings.csv"); + string longString = new string('x', 5000); + string veryLongString = new string('y', 10000); + File.WriteAllText(csvPath, $"Short,Long,VeryLong\nabc,{longString},{veryLongString}\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual("varchar(3)", columns[0].SqlDataType); + Assert.AreEqual("varchar(5000)", columns[1].SqlDataType); + Assert.AreEqual("varchar(max)", columns[2].SqlDataType); // > 8000 + } + + #endregion + + #region Full Scan Tests + + [TestMethod] + public void TestInferSchema_FullScan_10000Rows() + { + string csvPath = Path.Combine(_tempDir, "large.csv"); + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Value,Category"); + for (int i = 0; i < 10000; i++) + { + // Mix of values to test type detection + string category = i % 10 == 0 ? "A" : (i % 10 == 1 ? "B" : "C"); + writer.WriteLine($"{i},{i * 1.5m:F2},{category}"); + } + } + + var progressValues = new List(); + var columns = CsvSchemaInference.InferSchema(csvPath, null, p => progressValues.Add(p)); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(columns[2].SqlDataType.StartsWith("varchar(")); + + // Progress should have been reported + Assert.IsTrue(progressValues.Count > 0); + Assert.AreEqual(1.0, progressValues.Last(), 0.01); + + // Verify row counts + Assert.AreEqual(10000, columns[0].TotalCount); + } + + [TestMethod] + public void TestInferSchema_FullScan_WithCancellation() + { + string csvPath = Path.Combine(_tempDir, "cancellable.csv"); + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Value"); + for (int i = 0; i < 1000; i++) + { + writer.WriteLine($"{i},{i * 10}"); + } + } + + // Test 1: Pre-cancelled token should throw immediately + var preCancelledCts = new CancellationTokenSource(); + preCancelledCts.Cancel(); + + try + { + CsvSchemaInference.InferSchema(csvPath, null, null, preCancelledCts.Token); + Assert.Fail("Should have thrown OperationCanceledException for pre-cancelled token"); + } + catch (OperationCanceledException) + { + // Expected + } + + // Test 2: Stream-based inference with cancellation + var cts = new CancellationTokenSource(); + + try + { + // Create data in memory + var sb = new StringBuilder(); + sb.AppendLine("Id,Value"); + for (int i = 0; i < 10000; i++) + { + sb.AppendLine($"{i},{i * 10}"); + } + + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()))) + { + // Cancel after very short time to trigger during read + cts.CancelAfter(1); + + // This may or may not throw depending on timing - just verify it handles gracefully + var columns = CsvSchemaInference.InferSchemaFromSample(stream, null, 10000, cts.Token); + + // If we got here, data was small enough to complete before cancellation + // That's acceptable - cancellation is best-effort + } + } + catch (OperationCanceledException) + { + // Expected if cancellation kicked in + } + } + + #endregion + + #region Sample vs Full Scan Comparison + + [TestMethod] + public void TestInferSchema_SampleVsFullScan_ConsistentResults() + { + string csvPath = Path.Combine(_tempDir, "consistent.csv"); + // Use consistent value ranges so sample and full scan produce same type classifications + using (var writer = new StreamWriter(csvPath)) + { + writer.WriteLine("Id,Price,Name"); + for (int i = 0; i < 5000; i++) + { + // Keep all values in same range (1-100, price ~20) + writer.WriteLine($"{(i % 100) + 1},{19.99m + (i % 10) * 0.01m:F2},Product{(i % 10)}"); + } + } + + var sampleColumns = CsvSchemaInference.InferSchemaFromSample(csvPath, null, 100); + var fullColumns = CsvSchemaInference.InferSchema(csvPath); + + // Base type categories should match (int vs decimal vs string), precision may vary + Assert.AreEqual("int", sampleColumns[0].SqlDataType); + Assert.AreEqual("int", fullColumns[0].SqlDataType); + Assert.IsTrue(sampleColumns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(fullColumns[1].SqlDataType.Contains("decimal")); + Assert.IsTrue(sampleColumns[2].SqlDataType.StartsWith("varchar(")); + Assert.IsTrue(fullColumns[2].SqlDataType.StartsWith("varchar(")); + } + + #endregion + + #region Compressed File Tests + + [TestMethod] + public void TestInferSchema_GzipCompressed() + { + string csvPath = Path.Combine(_tempDir, "data.csv.gz"); + string csvContent = @"Id,Name,Value +1,Test,100 +2,Demo,200 +3,Sample,300 +"; + using (var fs = File.Create(csvPath)) + using (var gz = new GZipStream(fs, CompressionMode.Compress)) + using (var writer = new StreamWriter(gz)) + { + writer.Write(csvContent); + } + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); + Assert.AreEqual("int", columns[2].SqlDataType); + } + + #endregion + + #region Custom Options Tests + + [TestMethod] + public void TestInferSchema_CustomDelimiter() + { + string csvPath = Path.Combine(_tempDir, "semicolon.csv"); + File.WriteAllText(csvPath, @"Id;Name;Value +1;John;100 +2;Jane;200 +"); + + var options = new CsvReaderOptions { Delimiter = ";" }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("Id", columns[0].ColumnName); + Assert.AreEqual("Name", columns[1].ColumnName); + Assert.AreEqual("Value", columns[2].ColumnName); + } + + [TestMethod] + public void TestInferSchema_TabDelimited() + { + string csvPath = Path.Combine(_tempDir, "tabs.tsv"); + File.WriteAllText(csvPath, "Id\tName\tValue\n1\tJohn\t100\n2\tJane\t200\n"); + + var options = new CsvReaderOptions { Delimiter = "\t" }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_CustomDateFormat() + { + string csvPath = Path.Combine(_tempDir, "customdate.csv"); + File.WriteAllText(csvPath, @"Id,Date +1,25-Dec-2024 +2,15-Jan-2025 +3,01-Feb-2025 +"); + + var options = new CsvReaderOptions + { + DateTimeFormats = new[] { "dd-MMM-yyyy" } + }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual("datetime2", columns[1].SqlDataType); + } + + [TestMethod] + public void TestInferSchema_NoHeaderRow() + { + string csvPath = Path.Combine(_tempDir, "noheader.csv"); + File.WriteAllText(csvPath, @"1,John,100 +2,Jane,200 +3,Bob,300 +"); + + var options = new CsvReaderOptions { HasHeaderRow = false }; + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath, options); + + Assert.AreEqual(3, columns.Count); + // Column names are auto-generated by CsvDataReader (0-based: Column0, Column1, Column2) + Assert.AreEqual("Column0", columns[0].ColumnName); + Assert.AreEqual("Column1", columns[1].ColumnName); + Assert.AreEqual("Column2", columns[2].ColumnName); + } + + #endregion + + #region Edge Cases + + [TestMethod] + public void TestInferSchema_EmptyFile() + { + string csvPath = Path.Combine(_tempDir, "empty.csv"); + File.WriteAllText(csvPath, ""); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(0, columns.Count); + } + + [TestMethod] + public void TestInferSchema_HeaderOnly() + { + string csvPath = Path.Combine(_tempDir, "headeronly.csv"); + File.WriteAllText(csvPath, "Id,Name,Value\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("varchar(1)", columns[0].SqlDataType); + Assert.IsTrue(columns[0].IsNullable); + } + + [TestMethod] + public void TestInferSchema_SingleRow() + { + string csvPath = Path.Combine(_tempDir, "singlerow.csv"); + File.WriteAllText(csvPath, "Id,Name,Value\n1,Test,100\n"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.AreEqual(1, columns[0].TotalCount); + } + + [TestMethod] + public void TestInferSchema_ScientificNotation() + { + string csvPath = Path.Combine(_tempDir, "scientific.csv"); + File.WriteAllText(csvPath, @"Value,BigValue +1.5e2,1.0E10 +2.5e2,2.0E10 +3.5e2,3.0E10 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Scientific notation should be handled + Assert.IsTrue(columns[0].SqlDataType.Contains("decimal") || + columns[0].SqlDataType.StartsWith("varchar(")); + } + + [TestMethod] + public void TestInferSchema_MixedTypesInColumn_FallsBackToVarchar() + { + string csvPath = Path.Combine(_tempDir, "mixed.csv"); + File.WriteAllText(csvPath, @"Value +100 +abc +200 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.IsTrue(columns[0].SqlDataType.StartsWith("varchar(")); + } + + [TestMethod] + public void TestInferSchema_QuotedFields() + { + string csvPath = Path.Combine(_tempDir, "quoted.csv"); + // RFC 4180: quotes inside quoted fields are escaped by doubling them + var sb = new StringBuilder(); + sb.AppendLine("Id,Name,Description"); + sb.AppendLine("1,\"John Smith\",\"A \"\"quoted\"\" value\""); + sb.AppendLine("2,\"Jane Doe\",\"Another, with comma\""); + File.WriteAllText(csvPath, sb.ToString()); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(3, columns.Count); + Assert.IsTrue(columns[2].MaxLength > 10); // Should capture full quoted content + } + + [TestMethod] + public void TestInferSchema_LeadingZeros_ParseAsInteger() + { + string csvPath = Path.Combine(_tempDir, "leadingzeros.csv"); + File.WriteAllText(csvPath, @"ZipCode,Phone +01234,0123456789 +02345,0234567890 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Leading zeros are parsed successfully by int.TryParse (01234 -> 1234) + // so the column is inferred as integer. Note: the leading zeros are NOT + // preserved in the parsed value. If preserving leading zeros is required, + // callers should override the inferred type to varchar. + Assert.AreEqual("int", columns[0].SqlDataType); + Assert.AreEqual("int", columns[1].SqlDataType); // 0123456789 fits in int (< 2.1 billion) + } + + [TestMethod] + public void TestInferSchema_DecimalWithNoIntegerPart() + { + string csvPath = Path.Combine(_tempDir, "decimalnoint.csv"); + File.WriteAllText(csvPath, @"Value,Tiny,Mixed +.5,.001,.999 +.25,.002,1.5 +.125,.003,10.25 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Decimals without integer part (e.g., .5 instead of 0.5) should be handled + Assert.IsTrue(columns[0].SqlDataType.Contains("decimal"), $"Expected decimal, got {columns[0].SqlDataType}"); + Assert.IsTrue(columns[1].SqlDataType.Contains("decimal"), $"Expected decimal, got {columns[1].SqlDataType}"); + Assert.IsTrue(columns[2].SqlDataType.Contains("decimal"), $"Expected decimal, got {columns[2].SqlDataType}"); + + // Verify scale is tracked correctly + Assert.AreEqual(3, columns[0].Scale); // .5, .25, .125 -> max 3 digits after decimal + Assert.AreEqual(3, columns[1].Scale); // .001, .002, .003 -> 3 digits after decimal + } + + #endregion + + #region Utility Method Tests + + [TestMethod] + public void TestGenerateCreateTableStatement_ComplexTable() + { + string csvPath = Path.Combine(_tempDir, "complex.csv"); + File.WriteAllText(csvPath, @"Id,Name,Price,IsActive,Created,UniqueId +1,Widget,19.99,true,2024-01-15,550e8400-e29b-41d4-a716-446655440000 +2,,29.50,false,2024-02-20,6ba7b810-9dad-11d1-80b4-00c04fd430c8 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + string sql = CsvSchemaInference.GenerateCreateTableStatement(columns, "Products", "sales"); + + Assert.IsTrue(sql.Contains("CREATE TABLE [sales].[Products]")); + Assert.IsTrue(sql.Contains("[Id] int NOT NULL")); + Assert.IsTrue(sql.Contains("[Name]") && sql.Contains("NULL")); // Name is nullable + Assert.IsTrue(sql.Contains("[Price] decimal")); + Assert.IsTrue(sql.Contains("[IsActive] bit")); + Assert.IsTrue(sql.Contains("[Created] datetime2")); + Assert.IsTrue(sql.Contains("[UniqueId] uniqueidentifier")); + } + + [TestMethod] + public void TestToColumnTypes_Mapping() + { + string csvPath = Path.Combine(_tempDir, "types.csv"); + File.WriteAllText(csvPath, @"IntCol,DecCol,BoolCol,DateCol,GuidCol,StrCol +1,1.5,true,2024-01-01,550e8400-e29b-41d4-a716-446655440000,text +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + var typeMap = CsvSchemaInference.ToColumnTypes(columns); + + Assert.AreEqual(typeof(int), typeMap["IntCol"]); + Assert.AreEqual(typeof(decimal), typeMap["DecCol"]); + Assert.AreEqual(typeof(bool), typeMap["BoolCol"]); + Assert.AreEqual(typeof(DateTime), typeMap["DateCol"]); + Assert.AreEqual(typeof(Guid), typeMap["GuidCol"]); + Assert.AreEqual(typeof(string), typeMap["StrCol"]); + } + + [TestMethod] + public void TestInferredColumn_Properties() + { + string csvPath = Path.Combine(_tempDir, "props.csv"); + File.WriteAllText(csvPath, @"Name,Value +John,100 +Jane,200 +Bob,300 +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + // Check Name column + Assert.AreEqual("Name", columns[0].ColumnName); + Assert.AreEqual(0, columns[0].Ordinal); + Assert.AreEqual(4, columns[0].MaxLength); // "John" is longest + Assert.IsFalse(columns[0].IsNullable); + Assert.IsFalse(columns[0].IsUnicode); + Assert.AreEqual(3, columns[0].TotalCount); + Assert.AreEqual(3, columns[0].NonNullCount); + } + + #endregion + + #region Stream-Based Tests + + [TestMethod] + public void TestInferSchema_FromStream() + { + string csv = "Id,Name,Value\n1,John,100\n2,Jane,200\n"; + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(csv))) + { + var columns = CsvSchemaInference.InferSchemaFromSample(stream); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + } + + [TestMethod] + public void TestInferSchema_FromTextReader() + { + string csv = "Id,Name,Value\n1,John,100\n2,Jane,200\n"; + using (var reader = new StringReader(csv)) + { + var columns = CsvSchemaInference.InferSchemaFromSample(reader); + + Assert.AreEqual(3, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); + } + } + + #endregion + + #region Real-World Scenario Tests + + [TestMethod] + public void TestInferSchema_SalesData() + { + string csvPath = Path.Combine(_tempDir, "sales.csv"); + File.WriteAllText(csvPath, @"OrderId,CustomerId,ProductName,Quantity,UnitPrice,Discount,OrderDate,ShipCountry +10248,VINET,Queso Cabrales,12,14.00,0.00,1996-07-04,France +10249,TOMSP,Tofu,9,18.60,0.00,1996-07-05,Germany +10250,HANAR,Sir Rodney's Scones,40,8.00,0.05,1996-07-08,Brazil +10251,VICTE,Manjimup Dried Apples,35,42.40,0.15,1996-07-08,France +10252,SUPRD,Filo Mix,48,5.60,0.10,1996-07-09,Belgium +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(8, columns.Count); + Assert.AreEqual("int", columns[0].SqlDataType); // OrderId + Assert.IsTrue(columns[1].SqlDataType.StartsWith("varchar(")); // CustomerId + Assert.IsTrue(columns[2].SqlDataType.StartsWith("varchar(")); // ProductName + Assert.AreEqual("int", columns[3].SqlDataType); // Quantity + Assert.IsTrue(columns[4].SqlDataType.Contains("decimal")); // UnitPrice + Assert.IsTrue(columns[5].SqlDataType.Contains("decimal")); // Discount + Assert.AreEqual("datetime2", columns[6].SqlDataType); // OrderDate + Assert.IsTrue(columns[7].SqlDataType.StartsWith("varchar(")); // ShipCountry + } + + [TestMethod] + public void TestInferSchema_EmployeeData() + { + string csvPath = Path.Combine(_tempDir, "employees.csv"); + File.WriteAllText(csvPath, @"EmployeeId,FirstName,LastName,Email,HireDate,Salary,IsManager,DepartmentCode +E001,John,Smith,john.smith@company.com,2020-03-15,75000.00,true,IT +E002,Jane,Doe,jane.doe@company.com,2019-07-22,85000.00,true,HR +E003,Bob,Johnson,bob.j@company.com,2021-01-10,65000.00,false,IT +E004,Alice,Williams,alice.w@company.com,2018-11-05,95000.00,true,FIN +"); + + var columns = CsvSchemaInference.InferSchemaFromSample(csvPath); + + Assert.AreEqual(8, columns.Count); + Assert.IsTrue(columns[0].SqlDataType.StartsWith("varchar(")); // EmployeeId (has letter prefix) + Assert.AreEqual("datetime2", columns[4].SqlDataType); // HireDate + Assert.IsTrue(columns[5].SqlDataType.Contains("decimal")); // Salary + Assert.AreEqual("bit", columns[6].SqlDataType); // IsManager + } + + #endregion + } +} diff --git a/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs b/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs new file mode 100644 index 00000000..c1d8e171 --- /dev/null +++ b/project/dbatools/Csv/Reader/ColumnTypeAnalyzer.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Analyzes values for a single column to determine the optimal SQL Server data type. + /// Uses incremental analysis with early exit when types are eliminated. + /// + internal sealed class ColumnTypeAnalyzer + { + // Type flags - tracks which types are still possible + [Flags] + private enum PossibleTypes + { + None = 0, + Guid = 1 << 0, + Boolean = 1 << 1, + Int = 1 << 2, + BigInt = 1 << 3, + Decimal = 1 << 4, + DateTime = 1 << 5, + String = 1 << 6, // Always possible as fallback + All = Guid | Boolean | Int | BigInt | Decimal | DateTime | String + } + + private static readonly HashSet BooleanTrueValues = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "true", "yes", "1", "on", "y", "t" + }; + + private static readonly HashSet BooleanFalseValues = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "false", "no", "0", "off", "n", "f" + }; + + // Standard DateTime formats to try + private static readonly string[] StandardDateTimeFormats = new[] + { + "yyyy-MM-dd HH:mm:ss.fff", + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd", + "yyyy/MM/dd HH:mm:ss", + "yyyy/MM/dd", + "MM/dd/yyyy HH:mm:ss", + "MM/dd/yyyy", + "dd/MM/yyyy HH:mm:ss", + "dd/MM/yyyy", + "dd-MM-yyyy HH:mm:ss", + "dd-MM-yyyy", + "M/d/yyyy HH:mm:ss", + "M/d/yyyy", + "yyyyMMdd", + "yyyyMMddHHmmss", + "yyyy-MM-ddTHH:mm:ss", + "yyyy-MM-ddTHH:mm:ss.fff", + "yyyy-MM-ddTHH:mm:ssZ", + "yyyy-MM-ddTHH:mm:ss.fffZ", + }; + + private readonly string _columnName; + private readonly int _ordinal; + private readonly string[] _customDateTimeFormats; + private readonly CultureInfo _culture; + + private PossibleTypes _possibleTypes = PossibleTypes.All; + private long _totalCount; + private long _nullCount; + private int _maxLength; + private bool _hasUnicode; + + // Decimal tracking + private int _maxPrecision; // Max total digits + private int _maxScale; // Max digits after decimal point + private int _maxIntegerDigits; // Max digits before decimal point + + /// + /// Creates a new column type analyzer. + /// + /// The column name from the CSV header. + /// The zero-based column position. + /// Optional custom DateTime formats to try first. + /// The culture for parsing numbers and dates. + public ColumnTypeAnalyzer(string columnName, int ordinal, string[] customDateTimeFormats, CultureInfo culture) + { + _columnName = columnName; + _ordinal = ordinal; + _customDateTimeFormats = customDateTimeFormats; + _culture = culture ?? CultureInfo.InvariantCulture; + } + + /// + /// Analyzes a single value and updates type statistics. + /// + /// The string value to analyze. + public void AnalyzeValue(string value) + { + _totalCount++; + + // Handle null/empty + if (string.IsNullOrEmpty(value) || string.IsNullOrWhiteSpace(value)) + { + _nullCount++; + return; + } + + string trimmed = value.Trim(); + if (trimmed.Length == 0) + { + _nullCount++; + return; + } + + // Track max length for string types + if (trimmed.Length > _maxLength) + { + _maxLength = trimmed.Length; + } + + // Check for Unicode characters (non-ASCII) + if (!_hasUnicode) + { + foreach (char c in trimmed) + { + if (c > 127) + { + _hasUnicode = true; + break; + } + } + } + + // Try each type in priority order, eliminating as they fail + // Early exit: if a type is already eliminated, skip checking it + + // 1. GUID check (most specific) + if ((_possibleTypes & PossibleTypes.Guid) != 0) + { + if (!Guid.TryParse(trimmed, out _)) + { + _possibleTypes &= ~PossibleTypes.Guid; + } + } + + // 2. Boolean check + if ((_possibleTypes & PossibleTypes.Boolean) != 0) + { + if (!BooleanTrueValues.Contains(trimmed) && !BooleanFalseValues.Contains(trimmed)) + { + _possibleTypes &= ~PossibleTypes.Boolean; + } + } + + // 3. Integer checks (int then bigint) + if ((_possibleTypes & PossibleTypes.Int) != 0) + { + if (!int.TryParse(trimmed, NumberStyles.Integer, _culture, out _)) + { + _possibleTypes &= ~PossibleTypes.Int; + } + } + + if ((_possibleTypes & PossibleTypes.BigInt) != 0) + { + if (!long.TryParse(trimmed, NumberStyles.Integer, _culture, out _)) + { + _possibleTypes &= ~PossibleTypes.BigInt; + } + } + + // 4. Decimal check - track precision and scale + if ((_possibleTypes & PossibleTypes.Decimal) != 0) + { + if (decimal.TryParse(trimmed, NumberStyles.Number, _culture, out decimal decVal)) + { + // Calculate precision and scale + AnalyzeDecimalPrecision(trimmed, decVal); + } + else + { + _possibleTypes &= ~PossibleTypes.Decimal; + } + } + + // 5. DateTime check + if ((_possibleTypes & PossibleTypes.DateTime) != 0) + { + if (!TryParseDateTime(trimmed)) + { + _possibleTypes &= ~PossibleTypes.DateTime; + } + } + } + + /// + /// Analyzes decimal precision and scale from a parsed value. + /// + private void AnalyzeDecimalPrecision(string original, decimal value) + { + // Use the string representation to count actual digits + // This handles scientific notation and trailing zeros correctly + + string normalized = original.Trim(); + + // Remove sign + if (normalized.StartsWith("-") || normalized.StartsWith("+")) + { + normalized = normalized.Substring(1); + } + + // Handle scientific notation - fall back to string analysis of the decimal + if (normalized.IndexOf('e') >= 0 || normalized.IndexOf('E') >= 0) + { + // Use decimal's string representation for scientific notation + normalized = Math.Abs(value).ToString(CultureInfo.InvariantCulture); + } + + // Remove thousands separators + normalized = normalized.Replace(",", "").Replace(" ", ""); + + // Find decimal point + int decimalIndex = normalized.IndexOf('.'); + if (decimalIndex < 0) + { + // Use culture-specific decimal separator + string decSep = _culture.NumberFormat.NumberDecimalSeparator; + decimalIndex = normalized.IndexOf(decSep, StringComparison.Ordinal); + } + + int integerDigits; + int fractionalDigits; + + if (decimalIndex >= 0) + { + // Count integer part digits (excluding leading zeros for values < 1) + string intPart = normalized.Substring(0, decimalIndex); + integerDigits = CountSignificantDigits(intPart, true); + + // Count fractional digits (including trailing zeros as they indicate precision) + string fracPart = normalized.Substring(decimalIndex + 1); + fractionalDigits = fracPart.Length; + } + else + { + integerDigits = CountSignificantDigits(normalized, true); + fractionalDigits = 0; + } + + // Track maximums + if (integerDigits > _maxIntegerDigits) + { + _maxIntegerDigits = integerDigits; + } + if (fractionalDigits > _maxScale) + { + _maxScale = fractionalDigits; + } + + int totalPrecision = integerDigits + fractionalDigits; + if (totalPrecision > _maxPrecision) + { + _maxPrecision = totalPrecision; + } + } + + /// + /// Counts significant digits in a numeric string. + /// + private static int CountSignificantDigits(string value, bool isIntegerPart) + { + int count = 0; + bool foundNonZero = false; + + foreach (char c in value) + { + if (char.IsDigit(c)) + { + if (isIntegerPart) + { + // For integer part, count after first non-zero (or all if it's just "0") + if (c != '0') + { + foundNonZero = true; + } + if (foundNonZero || value.Length == 1) + { + count++; + } + } + else + { + // For fractional part, count all digits + count++; + } + } + } + + return count > 0 ? count : 1; // At least 1 digit + } + + /// + /// Attempts to parse a value as DateTime using custom and standard formats. + /// + private bool TryParseDateTime(string value) + { + DateTimeStyles styles = DateTimeStyles.AllowWhiteSpaces; + + // Try custom formats first + if (_customDateTimeFormats != null && _customDateTimeFormats.Length > 0) + { + if (DateTime.TryParseExact(value, _customDateTimeFormats, _culture, styles, out _)) + { + return true; + } + } + + // Try standard formats + if (DateTime.TryParseExact(value, StandardDateTimeFormats, _culture, styles, out _)) + { + return true; + } + + // Try general parsing as fallback + return DateTime.TryParse(value, _culture, styles, out _); + } + + /// + /// Returns the inferred column based on all analyzed values. + /// + public InferredColumn GetInferredColumn() + { + var column = new InferredColumn + { + ColumnName = _columnName, + Ordinal = _ordinal, + TotalCount = _totalCount, + NonNullCount = _totalCount - _nullCount, + IsNullable = _nullCount > 0, + IsUnicode = _hasUnicode, + MaxLength = _maxLength + }; + + // If all values were null/empty + if (_totalCount == _nullCount) + { + column.SqlDataType = "varchar(1)"; + column.IsNullable = true; + return column; + } + + // Determine type in priority order + // Priority: GUID > Int > BigInt > Decimal > DateTime > Boolean > String + // Note: Int/BigInt are checked before Boolean because "1" and "0" are valid for both, + // and integer types are more restrictive (if we saw "2", boolean is eliminated but int remains) + + if ((_possibleTypes & PossibleTypes.Guid) != 0) + { + column.SqlDataType = "uniqueidentifier"; + } + else if ((_possibleTypes & PossibleTypes.Int) != 0) + { + column.SqlDataType = "int"; + } + else if ((_possibleTypes & PossibleTypes.BigInt) != 0) + { + column.SqlDataType = "bigint"; + } + else if ((_possibleTypes & PossibleTypes.Decimal) != 0) + { + // Calculate SQL decimal precision and scale + // SQL Server decimal: precision 1-38, scale 0-precision + int precision = _maxIntegerDigits + _maxScale; + int scale = _maxScale; + + // Ensure valid SQL Server decimal bounds + if (precision < 1) precision = 1; + if (precision > 38) precision = 38; + if (scale > precision) scale = precision; + if (scale < 0) scale = 0; + + // If it's effectively an integer in decimal form + if (scale == 0 && precision <= 10 && (_possibleTypes & PossibleTypes.Int) != 0) + { + column.SqlDataType = "int"; + } + else if (scale == 0 && precision <= 19 && (_possibleTypes & PossibleTypes.BigInt) != 0) + { + column.SqlDataType = "bigint"; + } + else + { + column.SqlDataType = $"decimal({precision},{scale})"; + column.Precision = precision; + column.Scale = scale; + } + } + else if ((_possibleTypes & PossibleTypes.Boolean) != 0) + { + column.SqlDataType = "bit"; + } + else if ((_possibleTypes & PossibleTypes.DateTime) != 0) + { + column.SqlDataType = "datetime2"; + } + else + { + // Fall back to string type + column.SqlDataType = GetStringType(column); + } + + return column; + } + + /// + /// Determines the appropriate string type (varchar/nvarchar with length). + /// + private string GetStringType(InferredColumn column) + { + string baseType = _hasUnicode ? "nvarchar" : "varchar"; + int maxAllowed = _hasUnicode ? 4000 : 8000; + + if (_maxLength == 0) + { + return $"{baseType}(1)"; + } + else if (_maxLength > maxAllowed) + { + return $"{baseType}(max)"; + } + else + { + return $"{baseType}({_maxLength})"; + } + } + } +} diff --git a/project/dbatools/Csv/Reader/CsvDataReader.cs b/project/dbatools/Csv/Reader/CsvDataReader.cs index e9a9630f..db821a43 100644 --- a/project/dbatools/Csv/Reader/CsvDataReader.cs +++ b/project/dbatools/Csv/Reader/CsvDataReader.cs @@ -3802,6 +3802,11 @@ public IReadOnlyList Columns /// public IReadOnlyList StaticColumnsList => _staticColumns; + /// + /// Gets the options used by this reader. + /// + public CsvReaderOptions Options => _options; + /// /// Gets the field headers. /// diff --git a/project/dbatools/Csv/Reader/CsvSchemaInference.cs b/project/dbatools/Csv/Reader/CsvSchemaInference.cs new file mode 100644 index 00000000..326fd3fa --- /dev/null +++ b/project/dbatools/Csv/Reader/CsvSchemaInference.cs @@ -0,0 +1,468 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; + +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Provides SQL Server schema inference for CSV files. + /// Analyzes CSV data to determine optimal column types for database import. + /// + public static class CsvSchemaInference + { + /// + /// Default number of rows to sample for schema inference. + /// + public const int DefaultSampleRows = 1000; + + /// + /// Default progress report interval (percentage points). + /// + private const double ProgressReportInterval = 0.01; // 1% + + #region Sample-Based Inference + + /// + /// Infers SQL Server schema by sampling the first N rows of a CSV file. + /// Fast but has a small risk if data patterns change after the sample. + /// + /// Path to the CSV file. + /// CSV reader options (delimiter, encoding, etc.). If null, defaults are used. + /// Number of rows to sample. Default is 1000. + /// List of inferred column definitions. + /// Thrown when path is null. + /// Thrown when the file does not exist. + public static List InferSchemaFromSample(string path, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + if (!File.Exists(path)) + throw new FileNotFoundException("CSV file not found.", path); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + // Create options copy to avoid modifying the caller's options + var inferOptions = options.Clone(); + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(path, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, inferOptions.CancellationToken); + } + } + + /// + /// Infers SQL Server schema by sampling the first N rows from a stream. + /// + /// Stream containing CSV data. + /// CSV reader options. + /// Number of rows to sample. + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchemaFromSample(Stream stream, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows, CancellationToken cancellationToken = default) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(stream, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, cancellationToken); + } + } + + /// + /// Infers SQL Server schema by sampling the first N rows from a TextReader. + /// + /// TextReader containing CSV data. + /// CSV reader options. + /// Number of rows to sample. + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchemaFromSample(TextReader textReader, CsvReaderOptions options = null, int sampleRows = DefaultSampleRows, CancellationToken cancellationToken = default) + { + if (textReader == null) + throw new ArgumentNullException(nameof(textReader)); + if (sampleRows < 1) + throw new ArgumentOutOfRangeException(nameof(sampleRows), "Sample rows must be at least 1."); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(textReader, inferOptions)) + { + return InferSchemaCore(reader, sampleRows, null, cancellationToken); + } + } + + #endregion + + #region Full Scan Inference + + /// + /// Infers SQL Server schema by scanning the entire CSV file. + /// Slower but guarantees no import failures due to type mismatches. + /// + /// Path to the CSV file. + /// CSV reader options (delimiter, encoding, etc.). If null, defaults are used. + /// Optional callback receiving progress (0.0 to 1.0). + /// Cancellation token. + /// List of inferred column definitions. + /// Thrown when path is null. + /// Thrown when the file does not exist. + public static List InferSchema(string path, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + if (!File.Exists(path)) + throw new FileNotFoundException("CSV file not found.", path); + + options = options ?? new CsvReaderOptions(); + + // Get file size for progress reporting + long fileSize = new FileInfo(path).Length; + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(path, inferOptions)) + { + return InferSchemaCore(reader, int.MaxValue, WrapProgressCallback(progressCallback, fileSize), cancellationToken); + } + } + + /// + /// Infers SQL Server schema by scanning the entire stream. + /// + /// Stream containing CSV data. + /// CSV reader options. + /// Optional callback receiving progress (0.0 to 1.0). + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchema(Stream stream, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + options = options ?? new CsvReaderOptions(); + + // Try to get stream length for progress + long streamLength = -1; + try + { + if (stream.CanSeek) + { + streamLength = stream.Length; + } + } + catch + { + // Ignore - some streams don't support Length + } + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(stream, inferOptions)) + { + return InferSchemaCore(reader, int.MaxValue, WrapProgressCallback(progressCallback, streamLength), cancellationToken); + } + } + + /// + /// Infers SQL Server schema by scanning the entire TextReader content. + /// Note: Progress callback will not report accurate percentages for TextReader. + /// + /// TextReader containing CSV data. + /// CSV reader options. + /// Optional callback (will not provide accurate progress for TextReader). + /// Cancellation token. + /// List of inferred column definitions. + public static List InferSchema(TextReader textReader, CsvReaderOptions options = null, Action progressCallback = null, CancellationToken cancellationToken = default) + { + if (textReader == null) + throw new ArgumentNullException(nameof(textReader)); + + options = options ?? new CsvReaderOptions(); + + var inferOptions = options.Clone(); + inferOptions.CancellationToken = cancellationToken; + inferOptions.ProgressCallback = null; + + using (var reader = new CsvDataReader(textReader, inferOptions)) + { + // For TextReader, we can't report accurate progress since we don't know total size + return InferSchemaCore(reader, int.MaxValue, null, cancellationToken); + } + } + + #endregion + + #region Core Implementation + + /// + /// Core implementation for schema inference using CsvDataReader. + /// + private static List InferSchemaCore(CsvDataReader csvReader, int maxRows, Action progressCallback, CancellationToken cancellationToken) + { + var result = new List(); + ColumnTypeAnalyzer[] analyzers = null; + + long rowCount = 0; + long lastProgressReport = 0; + const long progressInterval = 10000; // Report every 10K rows + + while (csvReader.Read() && rowCount < maxRows) + { + cancellationToken.ThrowIfCancellationRequested(); + + // Initialize analyzers on first row (after Read() populates field count) + if (analyzers == null) + { + int fieldCount = csvReader.FieldCount; + analyzers = new ColumnTypeAnalyzer[fieldCount]; + + var readerOptions = GetReaderOptions(csvReader); + for (int i = 0; i < fieldCount; i++) + { + string columnName = csvReader.GetName(i); + analyzers[i] = new ColumnTypeAnalyzer( + columnName, + i, + readerOptions != null ? readerOptions.DateTimeFormats : null, + readerOptions != null ? readerOptions.Culture : System.Globalization.CultureInfo.InvariantCulture); + } + } + + // Analyze each field + for (int i = 0; i < analyzers.Length; i++) + { + string value = csvReader.GetString(i); + analyzers[i].AnalyzeValue(value); + } + + rowCount++; + + // Report progress periodically + if (progressCallback != null && rowCount - lastProgressReport >= progressInterval) + { + progressCallback(rowCount, maxRows < int.MaxValue ? maxRows : -1); + lastProgressReport = rowCount; + } + } + + // Build results + if (analyzers != null) + { + foreach (var analyzer in analyzers) + { + result.Add(analyzer.GetInferredColumn()); + } + } + else if (csvReader.FieldCount > 0) + { + // Headers only (no data rows) - return varchar(1) NULL for each column + for (int i = 0; i < csvReader.FieldCount; i++) + { + result.Add(new InferredColumn + { + ColumnName = csvReader.GetName(i), + Ordinal = i, + SqlDataType = "varchar(1)", + IsNullable = true, + TotalCount = 0, + NonNullCount = 0, + MaxLength = 0 + }); + } + } + + // Final progress report + if (progressCallback != null) + { + progressCallback(rowCount, rowCount); + } + + return result; + } + + /// + /// Gets the options from a CsvDataReader. + /// + private static CsvReaderOptions GetReaderOptions(CsvDataReader reader) + { + return reader.Options; + } + + /// + /// Wraps a user progress callback with file-size based progress calculation. + /// + private static Action WrapProgressCallback(Action userCallback, long totalSize) + { + if (userCallback == null) + return null; + + double lastReported = -1; + + return (rowsRead, totalRows) => + { + double progress; + if (totalRows > 0) + { + progress = (double)rowsRead / totalRows; + } + else if (totalSize > 0) + { + // Estimate based on rows read (assume average row size) + // This is rough but better than nothing + progress = Math.Min(0.99, rowsRead * 100.0 / totalSize); + } + else + { + // Can't calculate progress + return; + } + + if (progress > 1.0) progress = 1.0; + + // Only report if progress changed significantly + if (progress - lastReported >= ProgressReportInterval || progress >= 1.0) + { + userCallback(progress); + lastReported = progress; + } + }; + } + + #endregion + + #region Utility Methods + + /// + /// Generates a CREATE TABLE statement from inferred columns. + /// + /// The inferred column definitions. + /// The name of the table to create. + /// Optional schema name (default: dbo). + /// A CREATE TABLE SQL statement. + public static string GenerateCreateTableStatement(List columns, string tableName, string schemaName = "dbo") + { + if (columns == null) + throw new ArgumentNullException(nameof(columns)); + if (string.IsNullOrWhiteSpace(tableName)) + throw new ArgumentException("Table name is required.", nameof(tableName)); + + var sb = new StringBuilder(); + // Escape ] as ]] to prevent SQL injection via identifier names + string escapedSchema = schemaName?.Replace("]", "]]") ?? "dbo"; + string escapedTable = tableName.Replace("]", "]]"); + sb.AppendLine(string.Format("CREATE TABLE [{0}].[{1}]", escapedSchema, escapedTable)); + sb.AppendLine("("); + + bool first = true; + foreach (var column in columns) + { + if (!first) + { + sb.AppendLine(","); + } + first = false; + + sb.Append(string.Format(" {0}", column.ToSqlDefinition())); + } + + sb.AppendLine(); + sb.AppendLine(");"); + + return sb.ToString(); + } + + /// + /// Converts inferred columns to a ColumnTypes dictionary for use with CsvReaderOptions. + /// + /// The inferred column definitions. + /// A dictionary mapping column names to .NET types. + public static Dictionary ToColumnTypes(List columns) + { + if (columns == null) + throw new ArgumentNullException(nameof(columns)); + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var column in columns) + { + Type netType = SqlTypeToNetType(column.SqlDataType); + result[column.ColumnName] = netType; + } + + return result; + } + + /// + /// Maps SQL Server data type strings to .NET types. + /// + private static Type SqlTypeToNetType(string sqlType) + { + if (string.IsNullOrEmpty(sqlType)) + return typeof(string); + + // Normalize: remove parentheses and content + string baseType = sqlType.ToLowerInvariant(); + int parenIndex = baseType.IndexOf('('); + if (parenIndex > 0) + { + baseType = baseType.Substring(0, parenIndex); + } + + switch (baseType) + { + case "bit": + return typeof(bool); + case "int": + return typeof(int); + case "bigint": + return typeof(long); + case "smallint": + return typeof(short); + case "tinyint": + return typeof(byte); + case "decimal": + case "numeric": + case "money": + case "smallmoney": + return typeof(decimal); + case "float": + return typeof(double); + case "real": + return typeof(float); + case "datetime": + case "datetime2": + case "date": + case "smalldatetime": + return typeof(DateTime); + case "uniqueidentifier": + return typeof(Guid); + default: + return typeof(string); + } + } + + #endregion + } +} diff --git a/project/dbatools/Csv/Reader/InferredColumn.cs b/project/dbatools/Csv/Reader/InferredColumn.cs new file mode 100644 index 00000000..c6119d6a --- /dev/null +++ b/project/dbatools/Csv/Reader/InferredColumn.cs @@ -0,0 +1,88 @@ +namespace Dataplat.Dbatools.Csv.Reader +{ + /// + /// Represents the inferred SQL Server schema for a CSV column. + /// + public sealed class InferredColumn + { + /// + /// Gets or sets the column name from the CSV header. + /// + public string ColumnName { get; set; } + + /// + /// Gets or sets the inferred SQL Server data type. + /// Examples: "int", "bigint", "varchar(47)", "nvarchar(255)", "datetime2", "bit", "uniqueidentifier", "decimal(18,4)" + /// + public string SqlDataType { get; set; } + + /// + /// Gets or sets the maximum length observed for string types. + /// For non-string types, this is 0. + /// + public int MaxLength { get; set; } + + /// + /// Gets or sets whether the column contains null or empty values. + /// When true, the SQL column should allow NULLs. + /// + public bool IsNullable { get; set; } + + /// + /// Gets or sets whether non-ASCII (Unicode) characters were detected. + /// When true, nvarchar should be used instead of varchar. + /// + public bool IsUnicode { get; set; } + + /// + /// Gets or sets the precision for decimal types. + /// Total number of digits (before + after decimal point). + /// + public int Precision { get; set; } + + /// + /// Gets or sets the scale for decimal types. + /// Number of digits after the decimal point. + /// + public int Scale { get; set; } + + /// + /// Gets the zero-based ordinal position of this column. + /// + public int Ordinal { get; internal set; } + + /// + /// Gets or sets the number of distinct non-null values sampled. + /// Useful for estimating cardinality. + /// + public long NonNullCount { get; set; } + + /// + /// Gets or sets the total number of values examined for this column. + /// + public long TotalCount { get; set; } + + /// + /// Returns a string representation of this inferred column. + /// + public override string ToString() + { + string nullability = IsNullable ? " NULL" : " NOT NULL"; + return $"{ColumnName} {SqlDataType}{nullability}"; + } + + /// + /// Gets the full SQL column definition suitable for CREATE TABLE. + /// + /// Whether to quote the column name with square brackets. + /// A SQL column definition string. + public string ToSqlDefinition(bool quoted = true) + { + // Escape ] as ]] to prevent SQL injection via column names + string escapedName = ColumnName?.Replace("]", "]]") ?? ""; + string name = quoted ? $"[{escapedName}]" : escapedName; + string nullability = IsNullable ? "NULL" : "NOT NULL"; + return $"{name} {SqlDataType} {nullability}"; + } + } +} diff --git a/project/dbatools/dbatools.csproj b/project/dbatools/dbatools.csproj index 999b576e..4d702408 100644 --- a/project/dbatools/dbatools.csproj +++ b/project/dbatools/dbatools.csproj @@ -7,8 +7,8 @@ dbatools The dbatools PowerShell Module library Copyright © 2025 - 0.10.0.78 - 0.10.0.78 + 0.10.0.79 + 0.10.0.79 dbatools false true @@ -58,6 +58,8 @@ + + diff --git a/tests/test-avoidconflicts.ps1 b/tests/test-avoidconflicts.ps1 index 4a03c525..1479d9c9 100644 --- a/tests/test-avoidconflicts.ps1 +++ b/tests/test-avoidconflicts.ps1 @@ -1,105 +1,285 @@ -# Test the AvoidConflicts parameter -Write-Host "=== Testing AvoidConflicts Parameter ===" -ForegroundColor Cyan +#!/usr/bin/env pwsh +# Test the AvoidConflicts parameter for dbatools.library +# +# IMPORTANT: Test order matters due to system-level assembly caching. +# Tests that check for conflicts must run BEFORE tests that load dbatools.library +# successfully, otherwise the cached assemblies may prevent conflicts from occurring. + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host " AvoidConflicts Parameter Test Suite " -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan $testsPassed = 0 $testsFailed = 0 -# Test 1: Import dbatools.library normally (without AvoidConflicts) -Write-Host "`n--- Test 1: Default import without AvoidConflicts ---" -ForegroundColor Yellow -try { - Remove-Module dbatools.library -ErrorAction SilentlyContinue - Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -Force +# Get the module path (relative to this script) +$modulePath = Join-Path $PSScriptRoot "..\artifacts\dbatools.library\dbatools.library.psd1" +if (-not (Test-Path $modulePath)) { + Write-Host "[ERROR] Module not found at: $modulePath" -ForegroundColor Red + Write-Host "Please build the module first using: dotnet build" -ForegroundColor Yellow + exit 1 +} - $loaded = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } - if ($loaded) { - Write-Host "[PASS] Microsoft.Data.SqlClient loaded by default" -ForegroundColor Green - $testsPassed++ - } else { - Write-Host "[FAIL] Microsoft.Data.SqlClient not loaded" -ForegroundColor Red - $testsFailed++ +# ============================================================================ +# CONFLICT TESTS FIRST - These must run before any successful dbatools.library load +# ============================================================================ + +# Test 1: SqlServer first, then dbatools.library WITHOUT AvoidConflicts +# Note: With the AssemblyLoadContext resolver (Core) or Redirector (Desktop), this may succeed +# because the resolver handles version mismatches automatically +Write-Host "`n--- Test 1: SqlServer first, WITHOUT AvoidConflicts ---" -ForegroundColor Yellow +Write-Host "Expected: May fail on Desktop (no resolver for some assemblies) or pass on Core (resolver handles it)" -ForegroundColor Gray +Write-Host "(This test validates baseline behavior)" -ForegroundColor DarkGray + +$result1 = pwsh -NoProfile -Command { + param($modulePath) + try { + Import-Module SqlServer -ErrorAction Stop + $sqlclient = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + if (-not $sqlclient) { + Write-Output "SKIP: SqlServer did not load SqlClient" + return + } + Import-Module $modulePath -Force -ErrorAction Stop + # If we get here, the resolver handled it + Write-Output "PASS_WITH_RESOLVER" + } catch { + if ($_.Exception.Message -match "Assembly with same name is already loaded|already loaded|SqlClient") { + Write-Output "EXPECTED_FAIL" + } else { + Write-Output "FAIL: Unexpected error: $($_.Exception.Message)" + } } -} catch { - Write-Host "[FAIL] Exception: $_" -ForegroundColor Red +} -args $modulePath + +Write-Host " Result: $result1" -ForegroundColor Gray + +if ($result1 -eq "EXPECTED_FAIL") { + Write-Host "[PASS] Fails when SqlServer loads conflicting DLLs (expected on Desktop without resolver)" -ForegroundColor Green + $testsPassed++ +} elseif ($result1 -eq "PASS_WITH_RESOLVER") { + Write-Host "[PASS] Module loaded successfully (AssemblyLoadContext resolver handled conflicts)" -ForegroundColor Green + $testsPassed++ +} elseif ($result1 -match "^SKIP") { + Write-Host "[SKIP] $result1" -ForegroundColor Yellow +} else { + Write-Host "[FAIL] $result1" -ForegroundColor Red $testsFailed++ } -# Test 2: Import SqlServer module first, then dbatools.library with AvoidConflicts -Write-Host "`n--- Test 2: Import with AvoidConflicts after SqlServer module ---" -ForegroundColor Yellow -try { - Remove-Module dbatools.library -ErrorAction SilentlyContinue - Remove-Module SqlServer -ErrorAction SilentlyContinue - - # Import SqlServer module first - Write-Host "Importing SqlServer module..." -ForegroundColor Gray - Import-Module SqlServer -ErrorAction Stop +# Test 2: Wrong ArgumentList syntax (hashtable) - should fail +Write-Host "`n--- Test 2: Wrong ArgumentList syntax (hashtable) ---" -ForegroundColor Yellow +Write-Host "Expected: Error about converting hashtable to SwitchParameter" -ForegroundColor Gray - # Get the SqlClient version loaded by SqlServer - $sqlServerClient = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } - if ($sqlServerClient) { - Write-Host "SqlServer module loaded Microsoft.Data.SqlClient version: $($sqlServerClient.GetName().Version)" -ForegroundColor Gray +$result2 = pwsh -NoProfile -Command { + param($modulePath) + try { + Import-Module $modulePath -ArgumentList @{AvoidConflicts=$true} -Force -ErrorAction Stop + Write-Output "UNEXPECTED_PASS" + } catch { + if ($_.Exception.Message -match "Cannot convert.*Hashtable.*SwitchParameter|Boolean parameters") { + Write-Output "EXPECTED_FAIL" + } else { + Write-Output "FAIL: Unexpected error: $($_.Exception.Message)" + } } +} -args $modulePath - # Import dbatools.library with AvoidConflicts and capture verbose output - Write-Host "Importing dbatools.library with -AvoidConflicts..." -ForegroundColor Gray - $verboseOutput = Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -ArgumentList $true -Verbose 4>&1 | Out-String +if ($result2 -eq "EXPECTED_FAIL") { + Write-Host "[PASS] Hashtable syntax correctly rejected with helpful error" -ForegroundColor Green + $testsPassed++ +} elseif ($result2 -eq "UNEXPECTED_PASS") { + Write-Host "[FAIL] Hashtable syntax unexpectedly worked" -ForegroundColor Red + $testsFailed++ +} else { + Write-Host "[FAIL] $result2" -ForegroundColor Red + $testsFailed++ +} - if ($verboseOutput -match "Skipping.*already loaded") { - Write-Host "[PASS] Verbose output shows assemblies were skipped" -ForegroundColor Green - Write-Host " Verbose message: $($verboseOutput -split "`n" | Where-Object { $_ -match 'Skipping' } | Select-Object -First 1)" -ForegroundColor Gray - $testsPassed++ - } else { - Write-Host "[FAIL] No verbose output about skipping assemblies" -ForegroundColor Red - Write-Host " Verbose output was: $verboseOutput" -ForegroundColor Gray - $testsFailed++ +# ============================================================================ +# SUCCESS TESTS - These can run after conflict tests +# ============================================================================ + +# Test 3: SqlServer first, then dbatools.library WITH AvoidConflicts (expect success) +Write-Host "`n--- Test 3: SqlServer first, WITH AvoidConflicts ---" -ForegroundColor Yellow +Write-Host "Expected: SUCCESS - AvoidConflicts skips conflicting assemblies" -ForegroundColor Gray + +$result3 = pwsh -NoProfile -Command { + param($modulePath) + try { + Import-Module SqlServer -ErrorAction Stop + + # Capture verbose output + $verboseMessages = @() + Import-Module $modulePath -ArgumentList $true -Force -Verbose -ErrorAction Stop 4>&1 | ForEach-Object { + if ($_ -is [System.Management.Automation.VerboseRecord]) { + $verboseMessages += $_.Message + } + } + + # Check if SMO is available + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + $skipped = $verboseMessages | Where-Object { $_ -match "Skipping.*already loaded" } + if ($skipped) { + Write-Output "PASS: Module loaded, assemblies skipped" + } else { + Write-Output "PASS: Module loaded successfully" + } + } else { + Write-Output "FAIL: SMO types not available" + } + } catch { + Write-Output "FAIL: $($_.Exception.Message)" } -} catch { - Write-Host "[FAIL] Exception: $_" -ForegroundColor Red +} -args $modulePath + +if ($result3 -match "^PASS") { + Write-Host "[PASS] Module loads with AvoidConflicts after SqlServer" -ForegroundColor Green + $testsPassed++ +} else { + Write-Host "[FAIL] $result3" -ForegroundColor Red $testsFailed++ } -# Test 3: Verify module still functions with AvoidConflicts -Write-Host "`n--- Test 3: Module functional after AvoidConflicts ---" -ForegroundColor Yellow -try { - if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { - Write-Host "[PASS] SMO types available after AvoidConflicts" -ForegroundColor Green - $testsPassed++ - } else { - Write-Host "[FAIL] SMO types not available" -ForegroundColor Red - $testsFailed++ +# Test 4: Verify verbose output shows skipped assemblies +Write-Host "`n--- Test 4: Verbose output shows skipped assemblies ---" -ForegroundColor Yellow +Write-Host "Expected: Verbose messages indicate SqlClient was skipped" -ForegroundColor Gray + +$result4 = pwsh -NoProfile -Command { + param($modulePath) + try { + Import-Module SqlServer -ErrorAction Stop + + $verboseOutput = "" + Import-Module $modulePath -ArgumentList $true -Force -Verbose 4>&1 | ForEach-Object { + if ($_ -is [System.Management.Automation.VerboseRecord]) { + $verboseOutput += $_.Message + "`n" + } + } + + if ($verboseOutput -match "Skipping Microsoft\.Data\.SqlClient\.dll.*already loaded") { + Write-Output "PASS: SqlClient skip message found" + } else { + Write-Output "INFO: SqlClient skip message not found (module may use compatible versions)" + } + } catch { + Write-Output "FAIL: $($_.Exception.Message)" } -} catch { - Write-Host "[FAIL] Exception: $_" -ForegroundColor Red +} -args $modulePath + +if ($result4 -match "^PASS") { + Write-Host "[PASS] Verbose output confirms SqlClient was skipped" -ForegroundColor Green + $testsPassed++ +} elseif ($result4 -match "^INFO") { + Write-Host "[INFO] $result4" -ForegroundColor Yellow + # Informational - don't count as pass or fail +} else { + Write-Host "[FAIL] $result4" -ForegroundColor Red $testsFailed++ } -# Test 4: Verify no conflicts when importing both modules -Write-Host "`n--- Test 4: No conflicts with both modules loaded ---" -ForegroundColor Yellow -try { - # Try using both modules - $smoType = [Microsoft.SqlServer.Management.Smo.Server] - $sqlClientType = [Microsoft.Data.SqlClient.SqlConnection] +# Test 5: Default import without SqlServer (expect success) +Write-Host "`n--- Test 5: Default import without SqlServer ---" -ForegroundColor Yellow +Write-Host "Expected: SUCCESS - Module loads normally" -ForegroundColor Gray - Write-Host "[PASS] Both modules can coexist without conflicts" -ForegroundColor Green - Write-Host " SMO type available: $($smoType.FullName)" -ForegroundColor Gray - Write-Host " SqlClient type available: $($sqlClientType.FullName)" -ForegroundColor Gray +$result5 = pwsh -NoProfile -Command { + param($modulePath) + try { + Import-Module $modulePath -Force -ErrorAction Stop + $loaded = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + if ($loaded) { + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Output "PASS" + } else { + Write-Output "FAIL: SMO types not available" + } + } else { + Write-Output "FAIL: Microsoft.Data.SqlClient not loaded" + } + } catch { + Write-Output "FAIL: $($_.Exception.Message)" + } +} -args $modulePath + +if ($result5 -eq "PASS") { + Write-Host "[PASS] dbatools.library loads successfully without SqlServer" -ForegroundColor Green $testsPassed++ -} catch { - Write-Host "[FAIL] Exception: $_" -ForegroundColor Red +} else { + Write-Host "[FAIL] $result5" -ForegroundColor Red $testsFailed++ } +# Test 6: Verify AssemblyLoadContext resolver works (PowerShell Core specific) +if ($PSVersionTable.PSEdition -eq "Core") { + Write-Host "`n--- Test 6: AssemblyLoadContext resolver (Core only) ---" -ForegroundColor Yellow + Write-Host "Expected: SUCCESS - Version mismatches resolved by AssemblyLoadContext handler" -ForegroundColor Gray + + $result6 = pwsh -NoProfile -Command { + param($modulePath) + try { + # Import SqlServer to get different assembly versions loaded + Import-Module SqlServer -ErrorAction Stop + + # Get the ConnectionInfo version loaded by SqlServer + $connectionInfo = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.SqlServer.ConnectionInfo' } + + if ($connectionInfo) { + Write-Host "SqlServer loaded ConnectionInfo: $($connectionInfo.GetName().Version)" -ForegroundColor Gray + } + + # Import dbatools.library - this will trigger assembly resolution + # If the resolver works, it should use the already-loaded assemblies + Import-Module $modulePath -ArgumentList $true -Force -ErrorAction Stop + + # Verify we can use SMO (which depends on ConnectionInfo) + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + # Verify the ConnectionInfo is still the one from SqlServer (not reloaded) + $connectionInfoAfter = [System.AppDomain]::CurrentDomain.GetAssemblies() | + Where-Object { $_.GetName().Name -eq 'Microsoft.SqlServer.ConnectionInfo' } + + $count = @($connectionInfoAfter).Count + if ($count -eq 1) { + Write-Output "PASS: Single ConnectionInfo assembly (resolver working)" + } else { + Write-Output "WARN: Multiple ConnectionInfo assemblies loaded ($count)" + } + } else { + Write-Output "FAIL: SMO types not available" + } + } catch { + Write-Output "FAIL: $($_.Exception.Message)" + } + } -args $modulePath + + if ($result6 -match "^PASS") { + Write-Host "[PASS] AssemblyLoadContext resolver correctly handles version mismatches" -ForegroundColor Green + $testsPassed++ + } elseif ($result6 -match "^WARN") { + Write-Host "[WARN] $result6" -ForegroundColor Yellow + # Don't count as failure - multiple assemblies might be OK in some scenarios + } else { + Write-Host "[FAIL] $result6" -ForegroundColor Red + $testsFailed++ + } +} else { + Write-Host "`n--- Test 6: AssemblyLoadContext resolver (Core only) ---" -ForegroundColor Yellow + Write-Host "[SKIP] Test only applicable to PowerShell Core" -ForegroundColor DarkGray +} + # Summary Write-Host "`n========================================" -ForegroundColor White -Write-Host " TEST SUMMARY" -ForegroundColor White +Write-Host " TEST SUMMARY " -ForegroundColor White Write-Host "========================================" -ForegroundColor White Write-Host "Passed: $testsPassed" -ForegroundColor Green Write-Host "Failed: $testsFailed" -ForegroundColor $(if ($testsFailed -gt 0) { 'Red' } else { 'Green' }) Write-Host "Total: $($testsPassed + $testsFailed)" -ForegroundColor White +Write-Host "" if ($testsFailed -eq 0) { - Write-Host "`nALL AVOIDCONFLICTS TESTS PASSED!" -ForegroundColor Green + Write-Host "ALL TESTS PASSED!" -ForegroundColor Green exit 0 } else { - Write-Host "`nSOME TESTS FAILED - REVIEW BEFORE MERGE" -ForegroundColor Red + Write-Host "SOME TESTS FAILED" -ForegroundColor Red exit 1 } diff --git a/tests/verify-fix.ps1 b/tests/verify-fix.ps1 new file mode 100644 index 00000000..a5f5db4a --- /dev/null +++ b/tests/verify-fix.ps1 @@ -0,0 +1,70 @@ +#!/usr/bin/env pwsh +# Verification that both fixes work: +# 1. Boolean logic fix for $skipSqlClient +# 2. AssemblyLoadContext resolver for PowerShell Core (handles version mismatches) + +Write-Host "=== Verifying AvoidConflicts Fixes ===" -ForegroundColor Cyan +Write-Host "PowerShell Edition: $($PSVersionTable.PSEdition)" -ForegroundColor Gray +Write-Host "PowerShell Version: $($PSVersionTable.PSVersion)" -ForegroundColor Gray +Write-Host "" + +try { + Import-Module SqlServer -ErrorAction Stop + Write-Host "[OK] SqlServer loaded" -ForegroundColor Green + + # Check what SqlServer loaded + $sqlClient = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + $connectionInfo = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.SqlServer.ConnectionInfo' } + + if ($sqlClient) { + Write-Host "[OK] SqlClient loaded by SqlServer: $($sqlClient.GetName().Version)" -ForegroundColor Yellow + } + if ($connectionInfo) { + Write-Host "[OK] ConnectionInfo loaded by SqlServer: $($connectionInfo.GetName().Version)" -ForegroundColor Yellow + } + + if (-not $sqlClient -and -not $connectionInfo) { + Write-Host "[SKIP] SqlServer did not load any conflicting assemblies - test not applicable" -ForegroundColor Yellow + exit 0 + } + + Write-Host "" + + # This is the critical test - with the bugs: + # - Bug #1: $skipSqlClient boolean logic would fail + # - Bug #2: AssemblyLoadContext would fail to resolve version mismatches (Core only) + Import-Module ./artifacts/dbatools.library/dbatools.library.psd1 -ArgumentList $true -Force -Verbose -ErrorAction Stop + Write-Host "[OK] dbatools.library loaded successfully with AvoidConflicts" -ForegroundColor Green + + if ([Microsoft.SqlServer.Management.Smo.Server] -as [type]) { + Write-Host "[OK] SMO types are available" -ForegroundColor Green + } else { + Write-Host "[WARN] SMO types not available" -ForegroundColor Yellow + } + + # Verify the assemblies that are loaded are the ones from SqlServer (not dbatools.library) + $sqlClientAfter = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.Data.SqlClient' } + $connectionInfoAfter = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'Microsoft.SqlServer.ConnectionInfo' } + + Write-Host "" + Write-Host "Assembly versions after import:" -ForegroundColor Gray + if ($sqlClientAfter) { + Write-Host " SqlClient: $($sqlClientAfter.GetName().Version)" -ForegroundColor Gray + } + if ($connectionInfoAfter) { + Write-Host " ConnectionInfo: $($connectionInfoAfter.GetName().Version)" -ForegroundColor Gray + } + + Write-Host "" + Write-Host "=== ALL FIXES VERIFIED ===" -ForegroundColor Green + exit 0 + +} catch { + Write-Host "[FAIL] $($_.Exception.Message)" -ForegroundColor Red + Write-Host "" + Write-Host "Full error:" -ForegroundColor Red + Write-Host $_.Exception.ToString() -ForegroundColor DarkRed + Write-Host "" + Write-Host "=== FIX VERIFICATION FAILED ===" -ForegroundColor Red + exit 1 +}