diff --git a/packages/http-client-csharp/eng/pipeline/publish.yml b/packages/http-client-csharp/eng/pipeline/publish.yml index 5eed3ef4f11..64dacab88ef 100644 --- a/packages/http-client-csharp/eng/pipeline/publish.yml +++ b/packages/http-client-csharp/eng/pipeline/publish.yml @@ -248,6 +248,7 @@ extends: ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} + ${{ replace(replace('True', ne(variables['Build.Reason'], 'Manual'), ''), 'True', '-UseParallelRegeneration') }} -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} -BuildReason '$(Build.Reason)' diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 5ab1b823fa5..056a4134b38 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -311,93 +311,6 @@ function Update-UnbrandedGeneratorVersion { } } -# Compute libraries to regenerate by scanning the repository -function Get-LibrariesToRegenerate { - param([string]$SdkRepoPath) - - $EmitterMap = @{ - 'eng/azure-typespec-http-client-csharp-emitter-package.json' = '@azure-typespec/http-client-csharp' - 'eng/azure-typespec-http-client-csharp-mgmt-emitter-package.json' = '@azure-typespec/http-client-csharp-mgmt' - 'eng/http-client-csharp-emitter-package.json' = '@typespec/http-client-csharp' - } - - function Get-GeneratorType { - param([string]$LibraryPath) - - # Check for tsp-location.yaml files to identify TypeSpec libraries - $tspLocationFiles = Get-ChildItem -Path $LibraryPath -Recurse -Filter "tsp-location.yaml" -ErrorAction SilentlyContinue - - foreach ($tspLocationFile in $tspLocationFiles) { - try { - $content = Get-Content $tspLocationFile.FullName -Raw -ErrorAction SilentlyContinue - if ($content -and $content -match 'emitterPackageJsonPath:\s*(?"[^"]+"|[^,\s]+)\s*,?') { - $emitterPath = $matches['val'].Trim('"') - - if ($EmitterMap.ContainsKey($emitterPath)) { - return $EmitterMap[$emitterPath] - } - } - } - catch { - # Continue to next file if error - } - } - - return $null - } - - $libraries = @() - $sdkRoot = Join-Path $SdkRepoPath "sdk" - - if (-not (Test-Path $sdkRoot)) { - Write-Warning "SDK directory not found at: $sdkRoot" - return @() - } - - # Scan through all service directories - $serviceDirs = Get-ChildItem -Path $sdkRoot -Directory -Force -ErrorAction SilentlyContinue - foreach ($serviceDir in $serviceDirs) { - # Look for library directories - $libraryDirs = Get-ChildItem -Path $serviceDir.FullName -Directory -Force -ErrorAction SilentlyContinue - foreach ($libraryDir in $libraryDirs) { - # Skip directories that don't look like libraries - if ($libraryDir.Name -in @("tests", "samples", "perf", "assets", "docs")) { - continue - } - - # Skip libraries that start with "Microsoft." or don't start with "Azure." - if ($libraryDir.Name.StartsWith("Microsoft.") -or -not $libraryDir.Name.StartsWith("Azure.")) { - continue - } - - # If it has a /src directory, it's likely a library - $srcPath = Join-Path $libraryDir.FullName "src" - if (-not (Test-Path $srcPath)) { - continue - } - - # Check if this library uses TypeSpec with one of our generators - $generator = Get-GeneratorType $libraryDir.FullName - if (-not $generator) { - continue - } - - # Calculate relative path from SDK repo root - $relativePath = $libraryDir.FullName.Substring($SdkRepoPath.Length + 1) - $relativePath = $relativePath -replace "\\", "/" - - $libraries += @{ - Service = $serviceDir.Name - Library = $libraryDir.Name - Path = $relativePath - Generator = $generator - } - } - } - - return @($libraries) -} - # Interactive library selection function Select-LibrariesToRegenerate { param([array]$Libraries) @@ -519,60 +432,6 @@ function Select-LibrariesToRegenerate { return @($selectedLibraries) } -# Generate final report -function Write-RegenerationReport { - param( - [array]$Results, - [TimeSpan]$ElapsedTime, - [string]$DebugFolder - ) - - $passed = @($Results | Where-Object { $_.Success -eq $true }) - $failed = @($Results | Where-Object { $_.Success -eq $false }) - - Write-Host "`n==================== REGENERATION REPORT ====================" -ForegroundColor Cyan - Write-Host "Total Libraries: $($Results.Count)" -ForegroundColor White - Write-Host "Passed: $($passed.Count)" -ForegroundColor Green - Write-Host "Failed: $($failed.Count)" -ForegroundColor Red - - if ($ElapsedTime) { - $elapsedFormatted = "{0:hh\:mm\:ss}" -f $ElapsedTime - Write-Host "Execution Time: $elapsedFormatted" -ForegroundColor Cyan - } - Write-Host "" - - if ($passed.Count -gt 0) { - Write-Host "PASSED LIBRARIES:" -ForegroundColor Green - foreach ($result in $passed) { - Write-Host " ✓ $($result.Library) ($($result.Service))" -ForegroundColor Green - } - Write-Host "" - } - - if ($failed.Count -gt 0) { - Write-Host "FAILED LIBRARIES:" -ForegroundColor Red - foreach ($result in $failed) { - Write-Host " ✗ $($result.Library) ($($result.Service))" -ForegroundColor Red - Write-Host " Error: $($result.Error)" -ForegroundColor Gray - if ($result.Output) { - Write-Host " Details: $($result.Output.Substring(0, [Math]::Min(200, $result.Output.Length)))..." -ForegroundColor DarkGray - } - } - Write-Host "" - } - - Write-Host "=============================================================" -ForegroundColor Cyan - - # Save detailed report to debug folder - $reportPath = if ($DebugFolder) { - Join-Path $DebugFolder "regen-report.json" - } else { - Join-Path $packageRoot "regen-report.json" - } - $Results | ConvertTo-Json -Depth 10 | Set-Content $reportPath -Encoding utf8 - Write-Host "Detailed report saved to: $reportPath" -ForegroundColor Gray -} - # ============================================================================ # Main Script Execution # ============================================================================ @@ -585,7 +444,7 @@ try { if ($Select -and -not $isOpenAIMode) { Write-Host "`n[1/5] Loading TypeSpec libraries from repository..." -ForegroundColor Cyan - $allLibraries = Get-LibrariesToRegenerate -SdkRepoPath $sdkRepoPath + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath # Apply generator filter before interactive selection $filteredLibraries = @(Filter-LibrariesByGenerator ` @@ -626,6 +485,11 @@ try { } Write-Host "Debug folder: $debugFolder" -ForegroundColor Gray + $isCiRun = [bool]$env:BUILD_ARTIFACTSTAGINGDIRECTORY + $regenerationReportPath = Join-Path $debugFolder 'regen-report.json' + if ($isCiRun) { + $regenerationReportPath = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY 'regen-report.json' + } Write-Host "" # Step 1: Build the unbranded generator @@ -746,7 +610,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -DebugFolder $debugFolder + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath -PrintJson:$isCiRun # Exit with appropriate code if ($result.Success) { @@ -794,7 +658,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -DebugFolder $debugFolder + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath -PrintJson:$isCiRun if ($result.Success) { Write-Host "`nScript completed successfully." -ForegroundColor Cyan @@ -828,7 +692,7 @@ try { $librariesToAnalyze = $librariesToRegenerate } else { # Load all libraries and apply filters to determine what would be regenerated - $allLibraries = Get-LibrariesToRegenerate -SdkRepoPath $sdkRepoPath + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath $librariesToAnalyze = Filter-LibrariesByGenerator ` -Libraries $allLibraries ` -Azure:$Azure ` @@ -973,7 +837,7 @@ try { if (-not $Select) { # Load all libraries if not using -Select flag - $allLibraries = Get-LibrariesToRegenerate -SdkRepoPath $sdkRepoPath + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath # Apply generator filter $librariesToRegenerate = Filter-LibrariesByGenerator ` @@ -1013,145 +877,43 @@ try { Write-Host "No libraries selected for regeneration" -ForegroundColor Yellow $failedCount = 0 } else { - - # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 - $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { - (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum - } elseif ($IsMacOS) { - [int](sysctl -n hw.ncpu) - } else { - [int](nproc) - } - - $throttleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) - - Write-Host "Using $throttleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray - Write-Host "" - - # Pre-install tsp-client to avoid concurrent npm operations - $sdkForNetEngFolder = Join-Path $sdkRepoPath "eng" - Write-Host "Pre-installing tsp-client..." -ForegroundColor Gray - $tspClientDir = Join-Path $sdkForNetEngFolder "common" "tsp-client" - Invoke "npm ci --prefix $tspClientDir --registry $artifactFeedRegistry" $tspClientDir - if ($LASTEXITCODE -ne 0) { - throw "Failed to install tsp-client" - } - Write-Host " tsp-client ready" -ForegroundColor Green - Write-Host "" - - # Pre-build the client plugin to avoid concurrent builds - $codeGenerationTargetPath = Join-Path $sdkForNetEngFolder "CodeGeneration.targets" - if (-not (Test-Path $codeGenerationTargetPath)) { - throw "CodeGeneration.targets not found at: $codeGenerationTargetPath" - } - Write-Host "Pre-building client plugin..." -ForegroundColor Gray - Invoke "dotnet build $codeGenerationTargetPath /t:BuildPlugin /p:TypeSpecInput=temp" $sdkForNetEngFolder - if ($LASTEXITCODE -ne 0) { - throw "Failed to build client plugin" - } - Write-Host " Client plugin ready" -ForegroundColor Green - Write-Host "" - - # Thread-safe collections for progress tracking - $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() - $totalCount = $librariesToRegenerate.Count - - Write-Host "Configuring npm registry for tsp-client (temporary .env)..." -ForegroundColor Gray - $sdkEnvFile = Join-Path $sdkRepoPath ".env" - $originalSdkEnv = if (Test-Path $sdkEnvFile) { Get-Content $sdkEnvFile -Raw } else { $null } - Set-Content $sdkEnvFile "npm_config_registry=$artifactFeedRegistry`n" -Encoding utf8 -NoNewline - Write-Host " Wrote $sdkEnvFile" -ForegroundColor Green - Write-Host "" - - try { - Write-Host "Dispatching $totalCount regeneration jobs ($throttleLimit at a time)..." -ForegroundColor Cyan - # Run regeneration in parallel - $results = $librariesToRegenerate | ForEach-Object -ThrottleLimit $throttleLimit -Parallel { - $library = $_ - $azureSdkPath = $using:sdkRepoPath - $completedBag = $using:completed - $total = $using:totalCount - - Write-Host " -> Starting $($library.Library) ($($library.Service))" -ForegroundColor DarkGray - - # Determine build path (check for src subdirectory) - $libraryPath = Join-Path $azureSdkPath $library.Path - $srcPath = Join-Path $libraryPath "src" - $buildPath = if ((Test-Path $srcPath) -and (Get-ChildItem -Path $srcPath -Filter "*.csproj" -ErrorAction SilentlyContinue)) { - $srcPath - } else { - $libraryPath - } - - # Regenerate library - $result = try { - if (-not (Test-Path $libraryPath)) { - @{ Success = $false; Error = "Library path not found"; Output = "" } + $regenerationThrottleLimit = 0 + if (-not $isCiRun) { + # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 + $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { + (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum + } elseif ($IsMacOS) { + [int](sysctl -n hw.ncpu) } else { - Push-Location $buildPath - try { - $output = & dotnet build /t:GenerateCode /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true 2>&1 - $exitCode = $LASTEXITCODE - - if ($exitCode -ne 0) { - @{ Success = $false; Error = "Generation failed with exit code $exitCode"; Output = ($output -join "`n") } - } else { - @{ Success = $true; Output = ($output -join "`n") } - } - } - finally { - Pop-Location - } + [int](nproc) } + + $regenerationThrottleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) } - catch { - @{ Success = $false; Error = $_.Exception.Message; Output = $_.Exception.ToString() } - } - - # Update progress counter - $completedBag.Add(1) - $currentCount = $completedBag.Count - - # Thread-safe console output with progress - $status = if ($result.Success) { "✓" } else { "✗" } - $color = if ($result.Success) { "Green" } else { "White" } - - $progressMsg = "[$currentCount/$total] $status $($library.Library)" - Write-Host $progressMsg -ForegroundColor $color - - # Return result with library metadata - return @{ - Service = $library.Service - Library = $library.Library - Path = $library.Path - Generator = $library.Generator - Success = if ($result.ContainsKey('Success')) { $result.Success } else { $false } - Error = if ($result.ContainsKey('Error')) { $result.Error } else { "" } - Output = if ($result.ContainsKey('Output')) { $result.Output } else { "" } - } - } - } - finally { - # Remove/restore the temporary .env used to redirect tsp-client's npm registry - if ($null -eq $originalSdkEnv) { - Remove-Item $sdkEnvFile -Force -ErrorAction SilentlyContinue - } else { - Set-Content $sdkEnvFile $originalSdkEnv -Encoding utf8 -NoNewline - } - } - - # Generate final report - $scriptEndTime = Get-Date - $elapsedTime = $scriptEndTime - $scriptStartTime - - Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -DebugFolder $debugFolder - - # Check if any libraries failed - $failedLibraries = @($results | Where-Object { -not $_.Success }) - $failedCount = $failedLibraries.Count + + $results = @(Invoke-SdkLibraryRegeneration ` + -SdkRepoPath $sdkRepoPath ` + -Libraries $librariesToRegenerate ` + -NpmRegistry $artifactFeedRegistry ` + -ThrottleLimit $regenerationThrottleLimit ` + -StopOnFailure:$isCiRun) + + # Generate final report + $scriptEndTime = Get-Date + $elapsedTime = $scriptEndTime - $scriptStartTime + + Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath -PrintJson:$isCiRun + + # Check if any libraries failed + $failedLibraries = @($results | Where-Object { -not $_.Success }) + $failedCount = $failedLibraries.Count } if ($failedCount -gt 0) { + if ($isCiRun) { + throw "Validation failed: $failedCount libraries failed to regenerate. Check the detailed report above for error information." + } + Write-Host "`nValidation completed with warnings: $failedCount libraries failed to regenerate" -ForegroundColor Yellow Write-Host "Check the detailed report above for error information" -ForegroundColor Yellow } else { diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index af1d00126e2..118e58a3756 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1006,4 +1006,487 @@ function Update-AzureSpectorScenarios { return $generationOutput } -Export-ModuleMember -Function "Update-MgmtGenerator", "Update-AzureGenerator", "Filter-LibrariesByGenerator", "Filter-LibrariesByName", "Update-OpenAIGenerator", "Add-LocalNuGetSource", "Update-AzureSpectorScenarios" +function Get-SdkLibrariesToRegenerate { + <# + .SYNOPSIS + Discovers the SDK libraries in azure-sdk-for-net that are generated by the TypeSpec C# generators. + + .DESCRIPTION + Scans the sdk directory of the given repository for libraries containing a tsp-location.yaml + that references one of the TypeSpec C# emitter package json artifacts and returns metadata for + each matching library (Service, Library, Path and Generator). + + .PARAMETER SdkRepoPath + Path to the local azure-sdk-for-net repository. + + .PARAMETER EmitterPackageJsonPaths + Optional. Restricts the results to libraries referencing the specified emitter package json paths + (for example 'eng/http-client-csharp-emitter-package.json'). When omitted, all known emitters match. + + #> + param( + [Parameter(Mandatory = $true)] + [string]$SdkRepoPath, + + [Parameter(Mandatory = $false)] + [string[]]$EmitterPackageJsonPaths + ) + + $ErrorActionPreference = 'Stop' + + $emitterMap = @{ + 'eng/azure-typespec-http-client-csharp-emitter-package.json' = '@azure-typespec/http-client-csharp' + 'eng/azure-typespec-http-client-csharp-mgmt-emitter-package.json' = '@azure-typespec/http-client-csharp-mgmt' + 'eng/http-client-csharp-emitter-package.json' = '@typespec/http-client-csharp' + } + + if ($EmitterPackageJsonPaths -and $EmitterPackageJsonPaths.Count -gt 0) { + $unknownEmitters = @($EmitterPackageJsonPaths | Where-Object { -not $emitterMap.ContainsKey($_) }) + if ($unknownEmitters.Count -gt 0) { + throw "Unknown emitter package json path(s): $($unknownEmitters -join ', ')" + } + + $filteredMap = @{} + foreach ($emitterPath in $EmitterPackageJsonPaths) { + $filteredMap[$emitterPath] = $emitterMap[$emitterPath] + } + $emitterMap = $filteredMap + } + + # Resolves the generator used by a library by inspecting its tsp-location.yaml files + function Get-GeneratorType { + param( + [string]$LibraryPath, + [hashtable]$EmitterMap + ) + + $tspLocationFiles = Get-ChildItem -Path $LibraryPath -Recurse -Filter "tsp-location.yaml" -ErrorAction SilentlyContinue + + foreach ($tspLocationFile in $tspLocationFiles) { + try { + $content = Get-Content $tspLocationFile.FullName -Raw -ErrorAction SilentlyContinue + if ($content -and $content -match 'emitterPackageJsonPath:\s*(?"[^"]+"|[^,\s]+)\s*,?') { + $emitterPath = $matches['val'].Trim('"') + + if ($EmitterMap.ContainsKey($emitterPath)) { + return $EmitterMap[$emitterPath] + } + } + } + catch { + # Continue to next file if error + } + } + + return $null + } + + $libraries = @() + $sdkRoot = Join-Path $SdkRepoPath "sdk" + + if (-not (Test-Path $sdkRoot)) { + Write-Warning "SDK directory not found at: $sdkRoot" + return @() + } + + # Scan through all service directories + $serviceDirs = Get-ChildItem -Path $sdkRoot -Directory -Force -ErrorAction SilentlyContinue + foreach ($serviceDir in $serviceDirs) { + # Look for library directories + $libraryDirs = Get-ChildItem -Path $serviceDir.FullName -Directory -Force -ErrorAction SilentlyContinue + foreach ($libraryDir in $libraryDirs) { + # Skip directories that don't look like libraries + if ($libraryDir.Name -in @("tests", "samples", "perf", "assets", "docs")) { + continue + } + + # If it has a /src directory, it's likely a library + $srcPath = Join-Path $libraryDir.FullName "src" + if (-not (Test-Path $srcPath)) { + continue + } + + # Check if this library uses TypeSpec with one of our generators + $generator = Get-GeneratorType -LibraryPath $libraryDir.FullName -EmitterMap $emitterMap + if (-not $generator) { + continue + } + + # Calculate relative path from SDK repo root + $relativePath = $libraryDir.FullName.Substring($SdkRepoPath.Length + 1) + $relativePath = $relativePath -replace "\\", "/" + + $libraries += @{ + Service = $serviceDir.Name + Library = $libraryDir.Name + Path = $relativePath + Generator = $generator + } + } + } + + return @($libraries) +} + +function Invoke-SdkLibraryRegeneration { + <# + .SYNOPSIS + Regenerates the specified azure-sdk-for-net libraries in parallel. + + .DESCRIPTION + Pre-installs tsp-client and pre-builds the code generation plugin once, then invokes + 'dotnet build /t:GenerateCode' for each library concurrently. Returns one result object per + library containing the library metadata, a Success flag, and any error output. + + .PARAMETER SdkRepoPath + Path to the local azure-sdk-for-net repository. + + .PARAMETER Libraries + The libraries to regenerate, as returned by Get-SdkLibrariesToRegenerate. + + .PARAMETER ThrottleLimit + Optional. Number of concurrent regeneration jobs. Defaults to 3x the logical processors, clamped to 4-12, + since each job is dominated by child process and IO wait rather than CPU work. + + .PARAMETER NpmRegistry + Optional. When specified, a temporary .env file is written to the repository root so tsp-client + restores npm packages from the given registry. The original .env is restored afterwards. + + .PARAMETER AdditionalBuildArgs + Optional. Extra msbuild arguments appended to the 'dotnet build /t:GenerateCode' invocation. + + .PARAMETER SerialServiceDirectories + Optional. Names of service directories whose libraries share a code generation plugin and therefore + must be regenerated one at a time. Those libraries are regenerated serially after the parallel batch. + + .PARAMETER StopOnFailure + Optional. Stops dispatching later regeneration batches after a batch reports any failures. + #> + param( + [Parameter(Mandatory = $true)] + [string]$SdkRepoPath, + + [Parameter(Mandatory = $true)] + [array]$Libraries, + + [Parameter(Mandatory = $false)] + [int]$ThrottleLimit = 0, + + [Parameter(Mandatory = $false)] + [string]$NpmRegistry, + + [Parameter(Mandatory = $false)] + [string[]]$AdditionalBuildArgs = @(), + + [Parameter(Mandatory = $false)] + [string[]]$SerialServiceDirectories = @(), + + [Parameter(Mandatory = $false)] + [switch]$StopOnFailure + ) + + $ErrorActionPreference = 'Stop' + + if (-not $Libraries -or $Libraries.Count -eq 0) { + return @() + } + + # Determine parallel execution throttle limit. Each regeneration job spends most of its time + # waiting on child processes (tsp-client spec sync, NuGet/npm restore, file IO), so the machine + # can run more jobs than it has cores without saturating the CPU. Oversubscribe by 3x the + # logical processors, with a floor of 4 so low-core CI agents still get useful concurrency and a + # cap of 12 to keep memory usage on those same agents bounded. + if ($ThrottleLimit -le 0) { + $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { + (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum + } elseif ($IsMacOS) { + [int](sysctl -n hw.ncpu) + } else { + [int](nproc) + } + + $ThrottleLimit = [Math]::Max(4, [Math]::Min(12, $cpuCores * 3)) + Write-Host "Using $ThrottleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray + } else { + Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray + } + Write-Host "" + + $engFolder = Join-Path $SdkRepoPath "eng" + + # Pre-install tsp-client to avoid concurrent npm operations + Write-Host "Pre-installing tsp-client..." -ForegroundColor Gray + $tspClientDir = Join-Path $engFolder "common" "tsp-client" + $npmCiCommand = "npm ci --prefix $tspClientDir" + if ($NpmRegistry) { + $npmCiCommand += " --registry $NpmRegistry" + } + # Pipe to Out-Host so the command output is not captured as this function's return value + Invoke $npmCiCommand $tspClientDir | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Failed to install tsp-client" + } + Write-Host " tsp-client ready" -ForegroundColor Green + Write-Host "" + + # Pre-build the client plugin to avoid concurrent builds + $codeGenerationTargetPath = Join-Path $engFolder "CodeGeneration.targets" + if (-not (Test-Path $codeGenerationTargetPath)) { + throw "CodeGeneration.targets not found at: $codeGenerationTargetPath" + } + Write-Host "Pre-building client plugin..." -ForegroundColor Gray + Invoke "dotnet build $codeGenerationTargetPath /t:BuildPlugin /p:TypeSpecInput=temp" $engFolder | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Failed to build client plugin" + } + Write-Host " Client plugin ready" -ForegroundColor Green + Write-Host "" + + # Thread-safe collections for progress tracking + $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() + $totalCount = $Libraries.Count + $buildArgs = @($AdditionalBuildArgs | Where-Object { $_ }) + + $sdkEnvFile = Join-Path $SdkRepoPath ".env" + $originalSdkEnv = $null + $wroteSdkEnv = $false + if ($NpmRegistry) { + Write-Host "Configuring npm registry for tsp-client (temporary .env)..." -ForegroundColor Gray + $originalSdkEnv = if (Test-Path $sdkEnvFile) { Get-Content $sdkEnvFile -Raw } else { $null } + Set-Content $sdkEnvFile "npm_config_registry=$NpmRegistry`n" -Encoding utf8 -NoNewline + $wroteSdkEnv = $true + Write-Host " Wrote $sdkEnvFile" -ForegroundColor Green + Write-Host "" + } + + # Libraries in service directories that share a code generation plugin must not be built + # concurrently with each other, so they are regenerated in a serial batch after the parallel one. + $parallelLibraries = @($Libraries | Where-Object { $_.Service -notin $SerialServiceDirectories }) + $serialLibraries = @($Libraries | Where-Object { $_.Service -in $SerialServiceDirectories }) + + $batches = @() + if ($parallelLibraries.Count -gt 0) { + $batches += , @{ Libraries = $parallelLibraries; Throttle = $ThrottleLimit } + } + if ($serialLibraries.Count -gt 0) { + $batches += , @{ Libraries = $serialLibraries; Throttle = 1 } + } + + $results = @() + + try { + for ($batchIndex = 0; $batchIndex -lt $batches.Count; $batchIndex++) { + $batch = $batches[$batchIndex] + $batchLibraries = $batch.Libraries + $batchThrottle = $batch.Throttle + Write-Host "Dispatching $($batchLibraries.Count) regeneration jobs ($batchThrottle at a time)..." -ForegroundColor Cyan + # Run regeneration in parallel + $batchResults = @($batchLibraries | ForEach-Object -ThrottleLimit $batchThrottle -Parallel { + $library = $_ + $azureSdkPath = $using:SdkRepoPath + $completedBag = $using:completed + $total = $using:totalCount + $extraArgs = $using:buildArgs + + Write-Host " -> Starting $($library.Library) ($($library.Service))" -ForegroundColor DarkGray + $libraryStopwatch = [System.Diagnostics.Stopwatch]::StartNew() + + # Determine build path (check for src subdirectory) + $libraryPath = Join-Path $azureSdkPath $library.Path + $srcPath = Join-Path $libraryPath "src" + $buildPath = if ((Test-Path $srcPath) -and (Get-ChildItem -Path $srcPath -Filter "*.csproj" -ErrorAction SilentlyContinue)) { + $srcPath + } else { + $libraryPath + } + + # Regenerate library + $result = try { + if (-not (Test-Path $libraryPath)) { + @{ Success = $false; Error = "Library path not found"; Output = "" } + } else { + Push-Location $buildPath + try { + $output = & dotnet build /t:GenerateCode /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true @extraArgs 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + @{ Success = $false; Error = "Generation failed with exit code $exitCode"; Output = ($output -join "`n") } + } else { + @{ Success = $true; Output = ($output -join "`n") } + } + } + finally { + Pop-Location + } + } + } + catch { + @{ Success = $false; Error = $_.Exception.Message; Output = $_.Exception.ToString() } + } + + # Update progress counter + $libraryStopwatch.Stop() + $durationSeconds = [Math]::Round($libraryStopwatch.Elapsed.TotalSeconds, 1) + $completedBag.Add(1) + $currentCount = $completedBag.Count + + # Thread-safe console output with progress + $status = if ($result.Success) { "✓" } else { "✗" } + $color = if ($result.Success) { "Green" } else { "White" } + + $progressMsg = "[$currentCount/$total] $status $($library.Library) ($($durationSeconds)s)" + Write-Host $progressMsg -ForegroundColor $color + + # Return result with library metadata + return @{ + Service = $library.Service + Library = $library.Library + Path = $library.Path + Generator = $library.Generator + DurationSeconds = $durationSeconds + Success = if ($result.ContainsKey('Success')) { $result.Success } else { $false } + Error = if ($result.ContainsKey('Error')) { $result.Error } else { "" } + Output = if ($result.ContainsKey('Output')) { $result.Output } else { "" } + } + }) + $results += $batchResults + + if ($StopOnFailure -and @($batchResults | Where-Object { -not $_.Success }).Count -gt 0) { + $remainingLibraries = @() + for ($remainingBatchIndex = $batchIndex + 1; $remainingBatchIndex -lt $batches.Count; $remainingBatchIndex++) { + $remainingLibraries += $batches[$remainingBatchIndex].Libraries + } + foreach ($library in $remainingLibraries) { + $results += @{ + Service = $library.Service + Library = $library.Library + Path = $library.Path + Generator = $library.Generator + DurationSeconds = 0 + Success = $false + Error = "Skipped because an earlier regeneration batch failed" + Output = "" + Skipped = $true + } + } + break + } + } + } + finally { + # Remove/restore the temporary .env used to redirect tsp-client's npm registry + if ($wroteSdkEnv) { + if ($null -eq $originalSdkEnv) { + Remove-Item $sdkEnvFile -Force -ErrorAction SilentlyContinue + } else { + Set-Content $sdkEnvFile $originalSdkEnv -Encoding utf8 -NoNewline + } + } + } + + return @($results) +} + +function Write-RegenerationReport { + <# + .SYNOPSIS + Writes a summary of the regeneration results to the console and optionally to a JSON file. + + .PARAMETER Results + The result objects returned by Invoke-SdkLibraryRegeneration. + + .PARAMETER ElapsedTime + Optional. The total time taken to produce the results. + + .PARAMETER ReportPath + Optional. When specified, the detailed results are also written as JSON to this path. + + .PARAMETER PrintJson + Optional. Prints the detailed report as human-readable JSON. + #> + param( + [Parameter(Mandatory = $true)] + [array]$Results, + + [Parameter(Mandatory = $false)] + [TimeSpan]$ElapsedTime, + + [Parameter(Mandatory = $false)] + [string]$ReportPath, + + [Parameter(Mandatory = $false)] + [switch]$PrintJson + ) + + $passed = @($Results | Where-Object { $_.Success -eq $true }) + $failed = @($Results | Where-Object { $_.Success -eq $false }) + + $elapsedFormatted = if ($ElapsedTime) { "{0:hh\:mm\:ss}" -f $ElapsedTime } else { $null } + $report = [ordered]@{ + Summary = [ordered]@{ + TotalLibraries = $Results.Count + Passed = $passed.Count + Failed = $failed.Count + ExecutionTime = $elapsedFormatted + } + Results = @($Results) + } + $reportJson = $report | ConvertTo-Json -Depth 10 + + Write-Host "`n==================== REGENERATION REPORT ====================" -ForegroundColor Cyan + Write-Host "Total Libraries: $($Results.Count)" -ForegroundColor White + Write-Host "Passed: $($passed.Count)" -ForegroundColor Green + Write-Host "Failed: $($failed.Count)" -ForegroundColor Red + + if ($ElapsedTime) { + Write-Host "Execution Time: $elapsedFormatted" -ForegroundColor Cyan + } + Write-Host "" + + if ($passed.Count -gt 0) { + Write-Host "PASSED LIBRARIES:" -ForegroundColor Green + foreach ($result in $passed) { + $duration = if ($result.DurationSeconds) { " - $($result.DurationSeconds)s" } else { "" } + Write-Host " ✓ $($result.Library) ($($result.Service))$duration" -ForegroundColor Green + } + Write-Host "" + } + + $timed = @($Results | Where-Object { $_.DurationSeconds } | Sort-Object -Property DurationSeconds -Descending | Select-Object -First 5) + if ($timed.Count -gt 0) { + Write-Host "SLOWEST LIBRARIES:" -ForegroundColor Cyan + foreach ($result in $timed) { + Write-Host " $($result.Library) ($($result.Service)) - $($result.DurationSeconds)s" -ForegroundColor Gray + } + Write-Host "" + } + + if ($failed.Count -gt 0) { + Write-Host "FAILED LIBRARIES:" -ForegroundColor Red + foreach ($result in $failed) { + Write-Host " ✗ $($result.Library) ($($result.Service))" -ForegroundColor Red + Write-Host " Error: $($result.Error)" -ForegroundColor Gray + if ($result.Output) { + Write-Host " Details: $($result.Output.Substring(0, [Math]::Min(200, $result.Output.Length)))..." -ForegroundColor DarkGray + } + } + Write-Host "" + } + + Write-Host "=============================================================" -ForegroundColor Cyan + + # Save detailed report + if ($ReportPath) { + $reportJson | Set-Content $ReportPath -Encoding utf8 + Write-Host "Detailed report saved to: $ReportPath" -ForegroundColor Gray + } + + if ($PrintJson) { + Write-Host "" + Write-Host "REGENERATION REPORT JSON:" -ForegroundColor Cyan + Write-Host $reportJson + } +} + +Export-ModuleMember -Function "Update-MgmtGenerator", "Update-AzureGenerator", "Filter-LibrariesByGenerator", "Filter-LibrariesByName", "Update-OpenAIGenerator", "Add-LocalNuGetSource", "Update-AzureSpectorScenarios", "Get-SdkLibrariesToRegenerate", "Invoke-SdkLibraryRegeneration", "Write-RegenerationReport" diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 3e2b94ff1a2..2078ddb098a 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -23,6 +23,8 @@ Path to the build artifacts directory containing the published .tgz and .nupkg f The URL of the pipeline run that triggered this PR. When provided, it is included in the PR description for traceability. .PARAMETER BuildReason The reason the pipeline was triggered (for example, 'Manual', 'Schedule', or 'IndividualCI'). When set to 'Manual', step failures fail the pipeline instead of being downgraded to warnings and opening a PR. +.PARAMETER UseParallelRegeneration +When specified, SDK libraries are regenerated per library in parallel using the shared RegenPreview helpers instead of running 'dotnet msbuild service.proj /t:GenerateCode' once per service directory. This is intended for manual pipeline runs where turnaround time matters. #> [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -60,7 +62,10 @@ param( [switch]$UseTypeSpecNext, [Parameter(Mandatory = $false)] - [string]$BuildReason + [string]$BuildReason, + + [Parameter(Mandatory = $false)] + [switch]$UseParallelRegeneration ) # When the pipeline is triggered manually, failures should fail the pipeline with an @@ -68,6 +73,7 @@ param( # false positive to reviewers. For automated (scheduled/CI) runs, keep the existing # behavior of reporting SucceededWithIssues and continuing. $FailOnError = $BuildReason -eq 'Manual' +$isCiRun = [bool]$env:BUILD_ARTIFACTSTAGINGDIRECTORY # Tracks non-fatal step failures that were downgraded to warnings so a manual run can # fail before creating a pull request. @@ -580,6 +586,11 @@ try { } } + # Service directories whose libraries share a single code generator plugin that each + # library's generation builds into a common output folder. These services are regenerated + # serially since they share a common plugin project that shouldn't be built in parallel. + $serialCodeGenServiceDirectories = @("ai") + # Discover service directories with tsp-location.yaml referencing any of the matched emitter patterns $tspLocations = Get-ChildItem -Path (Join-Path $tempDir "sdk") -Filter "tsp-location.yaml" -Recurse $serviceDirectories = @() @@ -603,14 +614,43 @@ try { if ($serviceDirectories.Count -eq 0) { Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." + } elseif ($UseParallelRegeneration) { + # Manual runs regenerate each library directly (in parallel) instead of building an + # entire service directory at a time, which is significantly faster. + Write-Host "##[section]Regenerating SDK libraries in parallel..." + $allLibraries = @(Get-SdkLibrariesToRegenerate -SdkRepoPath $tempDir) + $librariesToRegenerate = @(Filter-LibrariesByGenerator ` + -Libraries $allLibraries ` + -Azure:$RegenerateAzureLibraries ` + -Unbranded ` + -Mgmt:$RegenerateMgmtLibraries) + + if ($librariesToRegenerate.Count -eq 0) { + Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." + } else { + $regeneratedServiceCount = @($librariesToRegenerate | ForEach-Object { $_.Service } | Sort-Object -Unique).Count + Write-Host "Regenerating $($librariesToRegenerate.Count) libraries across $regeneratedServiceCount service directories" + $regenerationStartTime = Get-Date + $regenerationResults = @(Invoke-SdkLibraryRegeneration ` + -SdkRepoPath $tempDir ` + -Libraries $librariesToRegenerate ` + -AdditionalBuildArgs @("/p:Trace=true") ` + -SerialServiceDirectories $serialCodeGenServiceDirectories ` + -StopOnFailure:$isCiRun) + + Write-RegenerationReport -Results $regenerationResults -ElapsedTime ((Get-Date) - $regenerationStartTime) -PrintJson:$isCiRun + + $failedLibraries = @($regenerationResults | Where-Object { -not $_.Success }) + foreach ($failedLibrary in $failedLibraries) { + Register-StepFailure "Code generation failed for $($failedLibrary.Path): $($failedLibrary.Error)" + } + if ($failedLibraries.Count -gt 0) { + throw "Parallel code generation failed for $($failedLibraries.Count) libraries." + } + } } else { $serviceProj = Join-Path $tempDir "eng/service.proj" - # Service directories whose libraries share a single code generator plugin that each - # library's generation builds into a common output folder. These services are regenerated - # serially since they share a common plugin project that shouldn't be built in parallel. - $serialCodeGenServiceDirectories = @("ai") - foreach ($serviceDirectory in $serviceDirectories) { Write-Host "Regenerating code for service directory: $serviceDirectory" $previousErrorAction = $ErrorActionPreference diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index d7513081d8b..e52f3e75c5b 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -392,6 +392,16 @@ If all libraries regenerate successfully, the script restores modified files: **Note:** If any libraries fail, artifacts are NOT restored, allowing you to debug the issue with the modified configuration intact. +## Shared Regeneration Helpers + +The library discovery and parallel regeneration logic lives in `RegenPreview.psm1` so it can be reused outside of local validation runs: + +| Function | Description | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | +| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit` (defaults to 3x the logical processors, clamped to 4-12), `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | +| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | + ### Error Handling If the script encounters an error during pre-requisite steps (Steps 1-6), it will: @@ -411,7 +421,8 @@ All packaged artifacts are stored in the `debug` folder at the root of the unbra - `Microsoft.TypeSpec.Generator.{version}.nupkg` - Core generator NuGet package - `Microsoft.TypeSpec.Generator.Input.{version}.nupkg` - Input models NuGet package - `Microsoft.TypeSpec.Generator.ClientModel.{version}.nupkg` - Client model NuGet package -- `regen-report.json` - Detailed JSON report of regeneration results +- `regen-report.json` - Detailed JSON report of regeneration results (written to the ADO artifact staging + directory during CI runs) ### Console Output @@ -448,6 +459,12 @@ FAILED LIBRARIES: Detailed report saved to: C:\...\debug\regen-report.json ``` +In ADO CI runs, the detailed JSON report is written to the artifact staging directory instead and is +also printed to the console in human-readable JSON. If any library fails to regenerate, later regeneration +batches are marked as skipped and the script exits with an error after printing the report. Local runs keep +the existing behavior: all regeneration batches run, and any library failures are reported as warnings at +the end. + ## Common Scenarios ### Scenario 1: Test OpenAI Library Changes