From 0adf38863c73fa3bcdd16d089d21f5b0067d5c99 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:14:04 +0000 Subject: [PATCH 01/17] Use RegenPreview parallel regeneration for manual publish pipeline runs Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/pipeline/publish.yml | 1 + .../eng/scripts/RegenPreview.ps1 | 299 +------------ .../eng/scripts/RegenPreview.psm1 | 423 +++++++++++++++++- .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 54 ++- .../eng/scripts/docs/RegenPreview.md | 15 + 5 files changed, 504 insertions(+), 288 deletions(-) diff --git a/packages/http-client-csharp/eng/pipeline/publish.yml b/packages/http-client-csharp/eng/pipeline/publish.yml index 8e480d9943f..239ddc46b51 100644 --- a/packages/http-client-csharp/eng/pipeline/publish.yml +++ b/packages/http-client-csharp/eng/pipeline/publish.yml @@ -248,6 +248,7 @@ extends: ${{ replace(replace('True', eq(variables['Build.SourceBranchName'], 'main'), ''), 'True', '-Internal') }} ${{ replace(replace('True', eq(parameters.RegenerateAzureLibraries, false), ''), 'True', '-RegenerateAzureLibraries') }} ${{ replace(replace('True', eq(parameters.RegenerateMgmtLibraries, false), ''), 'True', '-RegenerateMgmtLibraries') }} + ${{ replace(replace('True', ne(variables['Build.Reason'], 'Manual'), ''), 'True', '-UseParallelRegeneration') }} -BuildArtifactsPath '$(Pipeline.Workspace)/build_artifacts_csharp/packages' -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} -BuildReason '$(Build.Reason)' diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 5ab1b823fa5..766b39be535 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -311,93 +311,6 @@ function Update-UnbrandedGeneratorVersion { } } -# Compute libraries to regenerate by scanning the repository -function Get-LibrariesToRegenerate { - param([string]$SdkRepoPath) - - $EmitterMap = @{ - 'eng/azure-typespec-http-client-csharp-emitter-package.json' = '@azure-typespec/http-client-csharp' - 'eng/azure-typespec-http-client-csharp-mgmt-emitter-package.json' = '@azure-typespec/http-client-csharp-mgmt' - 'eng/http-client-csharp-emitter-package.json' = '@typespec/http-client-csharp' - } - - function Get-GeneratorType { - param([string]$LibraryPath) - - # Check for tsp-location.yaml files to identify TypeSpec libraries - $tspLocationFiles = Get-ChildItem -Path $LibraryPath -Recurse -Filter "tsp-location.yaml" -ErrorAction SilentlyContinue - - foreach ($tspLocationFile in $tspLocationFiles) { - try { - $content = Get-Content $tspLocationFile.FullName -Raw -ErrorAction SilentlyContinue - if ($content -and $content -match 'emitterPackageJsonPath:\s*(?"[^"]+"|[^,\s]+)\s*,?') { - $emitterPath = $matches['val'].Trim('"') - - if ($EmitterMap.ContainsKey($emitterPath)) { - return $EmitterMap[$emitterPath] - } - } - } - catch { - # Continue to next file if error - } - } - - return $null - } - - $libraries = @() - $sdkRoot = Join-Path $SdkRepoPath "sdk" - - if (-not (Test-Path $sdkRoot)) { - Write-Warning "SDK directory not found at: $sdkRoot" - return @() - } - - # Scan through all service directories - $serviceDirs = Get-ChildItem -Path $sdkRoot -Directory -Force -ErrorAction SilentlyContinue - foreach ($serviceDir in $serviceDirs) { - # Look for library directories - $libraryDirs = Get-ChildItem -Path $serviceDir.FullName -Directory -Force -ErrorAction SilentlyContinue - foreach ($libraryDir in $libraryDirs) { - # Skip directories that don't look like libraries - if ($libraryDir.Name -in @("tests", "samples", "perf", "assets", "docs")) { - continue - } - - # Skip libraries that start with "Microsoft." or don't start with "Azure." - if ($libraryDir.Name.StartsWith("Microsoft.") -or -not $libraryDir.Name.StartsWith("Azure.")) { - continue - } - - # If it has a /src directory, it's likely a library - $srcPath = Join-Path $libraryDir.FullName "src" - if (-not (Test-Path $srcPath)) { - continue - } - - # Check if this library uses TypeSpec with one of our generators - $generator = Get-GeneratorType $libraryDir.FullName - if (-not $generator) { - continue - } - - # Calculate relative path from SDK repo root - $relativePath = $libraryDir.FullName.Substring($SdkRepoPath.Length + 1) - $relativePath = $relativePath -replace "\\", "/" - - $libraries += @{ - Service = $serviceDir.Name - Library = $libraryDir.Name - Path = $relativePath - Generator = $generator - } - } - } - - return @($libraries) -} - # Interactive library selection function Select-LibrariesToRegenerate { param([array]$Libraries) @@ -519,60 +432,6 @@ function Select-LibrariesToRegenerate { return @($selectedLibraries) } -# Generate final report -function Write-RegenerationReport { - param( - [array]$Results, - [TimeSpan]$ElapsedTime, - [string]$DebugFolder - ) - - $passed = @($Results | Where-Object { $_.Success -eq $true }) - $failed = @($Results | Where-Object { $_.Success -eq $false }) - - Write-Host "`n==================== REGENERATION REPORT ====================" -ForegroundColor Cyan - Write-Host "Total Libraries: $($Results.Count)" -ForegroundColor White - Write-Host "Passed: $($passed.Count)" -ForegroundColor Green - Write-Host "Failed: $($failed.Count)" -ForegroundColor Red - - if ($ElapsedTime) { - $elapsedFormatted = "{0:hh\:mm\:ss}" -f $ElapsedTime - Write-Host "Execution Time: $elapsedFormatted" -ForegroundColor Cyan - } - Write-Host "" - - if ($passed.Count -gt 0) { - Write-Host "PASSED LIBRARIES:" -ForegroundColor Green - foreach ($result in $passed) { - Write-Host " ✓ $($result.Library) ($($result.Service))" -ForegroundColor Green - } - Write-Host "" - } - - if ($failed.Count -gt 0) { - Write-Host "FAILED LIBRARIES:" -ForegroundColor Red - foreach ($result in $failed) { - Write-Host " ✗ $($result.Library) ($($result.Service))" -ForegroundColor Red - Write-Host " Error: $($result.Error)" -ForegroundColor Gray - if ($result.Output) { - Write-Host " Details: $($result.Output.Substring(0, [Math]::Min(200, $result.Output.Length)))..." -ForegroundColor DarkGray - } - } - Write-Host "" - } - - Write-Host "=============================================================" -ForegroundColor Cyan - - # Save detailed report to debug folder - $reportPath = if ($DebugFolder) { - Join-Path $DebugFolder "regen-report.json" - } else { - Join-Path $packageRoot "regen-report.json" - } - $Results | ConvertTo-Json -Depth 10 | Set-Content $reportPath -Encoding utf8 - Write-Host "Detailed report saved to: $reportPath" -ForegroundColor Gray -} - # ============================================================================ # Main Script Execution # ============================================================================ @@ -585,7 +444,7 @@ try { if ($Select -and -not $isOpenAIMode) { Write-Host "`n[1/5] Loading TypeSpec libraries from repository..." -ForegroundColor Cyan - $allLibraries = Get-LibrariesToRegenerate -SdkRepoPath $sdkRepoPath + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath -AzureLibrariesOnly # Apply generator filter before interactive selection $filteredLibraries = @(Filter-LibrariesByGenerator ` @@ -746,7 +605,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -DebugFolder $debugFolder + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath (Join-Path $debugFolder 'regen-report.json') # Exit with appropriate code if ($result.Success) { @@ -794,7 +653,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -DebugFolder $debugFolder + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath (Join-Path $debugFolder 'regen-report.json') if ($result.Success) { Write-Host "`nScript completed successfully." -ForegroundColor Cyan @@ -828,7 +687,7 @@ try { $librariesToAnalyze = $librariesToRegenerate } else { # Load all libraries and apply filters to determine what would be regenerated - $allLibraries = Get-LibrariesToRegenerate -SdkRepoPath $sdkRepoPath + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath -AzureLibrariesOnly $librariesToAnalyze = Filter-LibrariesByGenerator ` -Libraries $allLibraries ` -Azure:$Azure ` @@ -973,7 +832,7 @@ try { if (-not $Select) { # Load all libraries if not using -Select flag - $allLibraries = Get-LibrariesToRegenerate -SdkRepoPath $sdkRepoPath + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath -AzureLibrariesOnly # Apply generator filter $librariesToRegenerate = Filter-LibrariesByGenerator ` @@ -1013,142 +872,20 @@ try { Write-Host "No libraries selected for regeneration" -ForegroundColor Yellow $failedCount = 0 } else { - - # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 - $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { - (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum - } elseif ($IsMacOS) { - [int](sysctl -n hw.ncpu) - } else { - [int](nproc) - } - - $throttleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) - - Write-Host "Using $throttleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray - Write-Host "" - - # Pre-install tsp-client to avoid concurrent npm operations - $sdkForNetEngFolder = Join-Path $sdkRepoPath "eng" - Write-Host "Pre-installing tsp-client..." -ForegroundColor Gray - $tspClientDir = Join-Path $sdkForNetEngFolder "common" "tsp-client" - Invoke "npm ci --prefix $tspClientDir --registry $artifactFeedRegistry" $tspClientDir - if ($LASTEXITCODE -ne 0) { - throw "Failed to install tsp-client" - } - Write-Host " tsp-client ready" -ForegroundColor Green - Write-Host "" + $results = @(Invoke-SdkLibraryRegeneration ` + -SdkRepoPath $sdkRepoPath ` + -Libraries $librariesToRegenerate ` + -NpmRegistry $artifactFeedRegistry) - # Pre-build the client plugin to avoid concurrent builds - $codeGenerationTargetPath = Join-Path $sdkForNetEngFolder "CodeGeneration.targets" - if (-not (Test-Path $codeGenerationTargetPath)) { - throw "CodeGeneration.targets not found at: $codeGenerationTargetPath" - } - Write-Host "Pre-building client plugin..." -ForegroundColor Gray - Invoke "dotnet build $codeGenerationTargetPath /t:BuildPlugin /p:TypeSpecInput=temp" $sdkForNetEngFolder - if ($LASTEXITCODE -ne 0) { - throw "Failed to build client plugin" - } - Write-Host " Client plugin ready" -ForegroundColor Green - Write-Host "" - - # Thread-safe collections for progress tracking - $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() - $totalCount = $librariesToRegenerate.Count - - Write-Host "Configuring npm registry for tsp-client (temporary .env)..." -ForegroundColor Gray - $sdkEnvFile = Join-Path $sdkRepoPath ".env" - $originalSdkEnv = if (Test-Path $sdkEnvFile) { Get-Content $sdkEnvFile -Raw } else { $null } - Set-Content $sdkEnvFile "npm_config_registry=$artifactFeedRegistry`n" -Encoding utf8 -NoNewline - Write-Host " Wrote $sdkEnvFile" -ForegroundColor Green - Write-Host "" - - try { - Write-Host "Dispatching $totalCount regeneration jobs ($throttleLimit at a time)..." -ForegroundColor Cyan - # Run regeneration in parallel - $results = $librariesToRegenerate | ForEach-Object -ThrottleLimit $throttleLimit -Parallel { - $library = $_ - $azureSdkPath = $using:sdkRepoPath - $completedBag = $using:completed - $total = $using:totalCount - - Write-Host " -> Starting $($library.Library) ($($library.Service))" -ForegroundColor DarkGray - - # Determine build path (check for src subdirectory) - $libraryPath = Join-Path $azureSdkPath $library.Path - $srcPath = Join-Path $libraryPath "src" - $buildPath = if ((Test-Path $srcPath) -and (Get-ChildItem -Path $srcPath -Filter "*.csproj" -ErrorAction SilentlyContinue)) { - $srcPath - } else { - $libraryPath - } - - # Regenerate library - $result = try { - if (-not (Test-Path $libraryPath)) { - @{ Success = $false; Error = "Library path not found"; Output = "" } - } else { - Push-Location $buildPath - try { - $output = & dotnet build /t:GenerateCode /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true 2>&1 - $exitCode = $LASTEXITCODE - - if ($exitCode -ne 0) { - @{ Success = $false; Error = "Generation failed with exit code $exitCode"; Output = ($output -join "`n") } - } else { - @{ Success = $true; Output = ($output -join "`n") } - } - } - finally { - Pop-Location - } - } - } - catch { - @{ Success = $false; Error = $_.Exception.Message; Output = $_.Exception.ToString() } - } - - # Update progress counter - $completedBag.Add(1) - $currentCount = $completedBag.Count - - # Thread-safe console output with progress - $status = if ($result.Success) { "✓" } else { "✗" } - $color = if ($result.Success) { "Green" } else { "White" } - - $progressMsg = "[$currentCount/$total] $status $($library.Library)" - Write-Host $progressMsg -ForegroundColor $color - - # Return result with library metadata - return @{ - Service = $library.Service - Library = $library.Library - Path = $library.Path - Generator = $library.Generator - Success = if ($result.ContainsKey('Success')) { $result.Success } else { $false } - Error = if ($result.ContainsKey('Error')) { $result.Error } else { "" } - Output = if ($result.ContainsKey('Output')) { $result.Output } else { "" } - } - } - } - finally { - # Remove/restore the temporary .env used to redirect tsp-client's npm registry - if ($null -eq $originalSdkEnv) { - Remove-Item $sdkEnvFile -Force -ErrorAction SilentlyContinue - } else { - Set-Content $sdkEnvFile $originalSdkEnv -Encoding utf8 -NoNewline - } - } - - # Generate final report - $scriptEndTime = Get-Date - $elapsedTime = $scriptEndTime - $scriptStartTime - - Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -DebugFolder $debugFolder - - # Check if any libraries failed - $failedLibraries = @($results | Where-Object { -not $_.Success }) - $failedCount = $failedLibraries.Count + # Generate final report + $scriptEndTime = Get-Date + $elapsedTime = $scriptEndTime - $scriptStartTime + + Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -ReportPath (Join-Path $debugFolder 'regen-report.json') + + # Check if any libraries failed + $failedLibraries = @($results | Where-Object { -not $_.Success }) + $failedCount = $failedLibraries.Count } if ($failedCount -gt 0) { diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index af1d00126e2..8cadec613ee 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1006,4 +1006,425 @@ function Update-AzureSpectorScenarios { return $generationOutput } -Export-ModuleMember -Function "Update-MgmtGenerator", "Update-AzureGenerator", "Filter-LibrariesByGenerator", "Filter-LibrariesByName", "Update-OpenAIGenerator", "Add-LocalNuGetSource", "Update-AzureSpectorScenarios" +function Get-SdkLibrariesToRegenerate { + <# + .SYNOPSIS + Discovers the SDK libraries in azure-sdk-for-net that are generated by the TypeSpec C# generators. + + .DESCRIPTION + Scans the sdk directory of the given repository for libraries containing a tsp-location.yaml + that references one of the TypeSpec C# emitter package json artifacts and returns metadata for + each matching library (Service, Library, Path and Generator). + + .PARAMETER SdkRepoPath + Path to the local azure-sdk-for-net repository. + + .PARAMETER EmitterPackageJsonPaths + Optional. Restricts the results to libraries referencing the specified emitter package json paths + (for example 'eng/http-client-csharp-emitter-package.json'). When omitted, all known emitters match. + + .PARAMETER AzureLibrariesOnly + Optional. When specified, only libraries whose directory name starts with 'Azure.' are returned. + #> + param( + [Parameter(Mandatory = $true)] + [string]$SdkRepoPath, + + [Parameter(Mandatory = $false)] + [string[]]$EmitterPackageJsonPaths, + + [Parameter(Mandatory = $false)] + [switch]$AzureLibrariesOnly + ) + + $ErrorActionPreference = 'Stop' + + $emitterMap = @{ + 'eng/azure-typespec-http-client-csharp-emitter-package.json' = '@azure-typespec/http-client-csharp' + 'eng/azure-typespec-http-client-csharp-mgmt-emitter-package.json' = '@azure-typespec/http-client-csharp-mgmt' + 'eng/http-client-csharp-emitter-package.json' = '@typespec/http-client-csharp' + } + + if ($EmitterPackageJsonPaths -and $EmitterPackageJsonPaths.Count -gt 0) { + $unknownEmitters = @($EmitterPackageJsonPaths | Where-Object { -not $emitterMap.ContainsKey($_) }) + if ($unknownEmitters.Count -gt 0) { + throw "Unknown emitter package json path(s): $($unknownEmitters -join ', ')" + } + + $filteredMap = @{} + foreach ($emitterPath in $EmitterPackageJsonPaths) { + $filteredMap[$emitterPath] = $emitterMap[$emitterPath] + } + $emitterMap = $filteredMap + } + + # Resolves the generator used by a library by inspecting its tsp-location.yaml files + function Get-GeneratorType { + param( + [string]$LibraryPath, + [hashtable]$EmitterMap + ) + + $tspLocationFiles = Get-ChildItem -Path $LibraryPath -Recurse -Filter "tsp-location.yaml" -ErrorAction SilentlyContinue + + foreach ($tspLocationFile in $tspLocationFiles) { + try { + $content = Get-Content $tspLocationFile.FullName -Raw -ErrorAction SilentlyContinue + if ($content -and $content -match 'emitterPackageJsonPath:\s*(?"[^"]+"|[^,\s]+)\s*,?') { + $emitterPath = $matches['val'].Trim('"') + + if ($EmitterMap.ContainsKey($emitterPath)) { + return $EmitterMap[$emitterPath] + } + } + } + catch { + # Continue to next file if error + } + } + + return $null + } + + $libraries = @() + $sdkRoot = Join-Path $SdkRepoPath "sdk" + + if (-not (Test-Path $sdkRoot)) { + Write-Warning "SDK directory not found at: $sdkRoot" + return @() + } + + # Scan through all service directories + $serviceDirs = Get-ChildItem -Path $sdkRoot -Directory -Force -ErrorAction SilentlyContinue + foreach ($serviceDir in $serviceDirs) { + # Look for library directories + $libraryDirs = Get-ChildItem -Path $serviceDir.FullName -Directory -Force -ErrorAction SilentlyContinue + foreach ($libraryDir in $libraryDirs) { + # Skip directories that don't look like libraries + if ($libraryDir.Name -in @("tests", "samples", "perf", "assets", "docs")) { + continue + } + + if ($AzureLibrariesOnly -and -not $libraryDir.Name.StartsWith("Azure.")) { + continue + } + + # If it has a /src directory, it's likely a library + $srcPath = Join-Path $libraryDir.FullName "src" + if (-not (Test-Path $srcPath)) { + continue + } + + # Check if this library uses TypeSpec with one of our generators + $generator = Get-GeneratorType -LibraryPath $libraryDir.FullName -EmitterMap $emitterMap + if (-not $generator) { + continue + } + + # Calculate relative path from SDK repo root + $relativePath = $libraryDir.FullName.Substring($SdkRepoPath.Length + 1) + $relativePath = $relativePath -replace "\\", "/" + + $libraries += @{ + Service = $serviceDir.Name + Library = $libraryDir.Name + Path = $relativePath + Generator = $generator + } + } + } + + return @($libraries) +} + +function Invoke-SdkLibraryRegeneration { + <# + .SYNOPSIS + Regenerates the specified azure-sdk-for-net libraries in parallel. + + .DESCRIPTION + Pre-installs tsp-client and pre-builds the code generation plugin once, then invokes + 'dotnet build /t:GenerateCode' for each library concurrently. Returns one result object per + library containing the library metadata, a Success flag, and any error output. + + .PARAMETER SdkRepoPath + Path to the local azure-sdk-for-net repository. + + .PARAMETER Libraries + The libraries to regenerate, as returned by Get-SdkLibrariesToRegenerate. + + .PARAMETER ThrottleLimit + Optional. Number of concurrent regeneration jobs. Defaults to (logical processors - 2), clamped to 1-8. + + .PARAMETER NpmRegistry + Optional. When specified, a temporary .env file is written to the repository root so tsp-client + restores npm packages from the given registry. The original .env is restored afterwards. + + .PARAMETER AdditionalBuildArgs + Optional. Extra msbuild arguments appended to the 'dotnet build /t:GenerateCode' invocation. + + .PARAMETER SerialServiceDirectories + Optional. Names of service directories whose libraries share a code generation plugin and therefore + must be regenerated one at a time. Those libraries are regenerated serially after the parallel batch. + #> + param( + [Parameter(Mandatory = $true)] + [string]$SdkRepoPath, + + [Parameter(Mandatory = $true)] + [array]$Libraries, + + [Parameter(Mandatory = $false)] + [int]$ThrottleLimit = 0, + + [Parameter(Mandatory = $false)] + [string]$NpmRegistry, + + [Parameter(Mandatory = $false)] + [string[]]$AdditionalBuildArgs = @(), + + [Parameter(Mandatory = $false)] + [string[]]$SerialServiceDirectories = @() + ) + + $ErrorActionPreference = 'Stop' + + if (-not $Libraries -or $Libraries.Count -eq 0) { + return @() + } + + # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 + if ($ThrottleLimit -le 0) { + $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { + (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum + } elseif ($IsMacOS) { + [int](sysctl -n hw.ncpu) + } else { + [int](nproc) + } + + $ThrottleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + Write-Host "Using $ThrottleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray + } else { + Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray + } + Write-Host "" + + $engFolder = Join-Path $SdkRepoPath "eng" + + # Pre-install tsp-client to avoid concurrent npm operations + Write-Host "Pre-installing tsp-client..." -ForegroundColor Gray + $tspClientDir = Join-Path $engFolder "common" "tsp-client" + $npmCiCommand = "npm ci --prefix $tspClientDir" + if ($NpmRegistry) { + $npmCiCommand += " --registry $NpmRegistry" + } + # Pipe to Out-Host so the command output is not captured as this function's return value + Invoke $npmCiCommand $tspClientDir | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Failed to install tsp-client" + } + Write-Host " tsp-client ready" -ForegroundColor Green + Write-Host "" + + # Pre-build the client plugin to avoid concurrent builds + $codeGenerationTargetPath = Join-Path $engFolder "CodeGeneration.targets" + if (-not (Test-Path $codeGenerationTargetPath)) { + throw "CodeGeneration.targets not found at: $codeGenerationTargetPath" + } + Write-Host "Pre-building client plugin..." -ForegroundColor Gray + Invoke "dotnet build $codeGenerationTargetPath /t:BuildPlugin /p:TypeSpecInput=temp" $engFolder | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Failed to build client plugin" + } + Write-Host " Client plugin ready" -ForegroundColor Green + Write-Host "" + + # Thread-safe collections for progress tracking + $completed = [System.Collections.Concurrent.ConcurrentBag[int]]::new() + $totalCount = $Libraries.Count + $buildArgs = @($AdditionalBuildArgs | Where-Object { $_ }) + + $sdkEnvFile = Join-Path $SdkRepoPath ".env" + $originalSdkEnv = $null + $wroteSdkEnv = $false + if ($NpmRegistry) { + Write-Host "Configuring npm registry for tsp-client (temporary .env)..." -ForegroundColor Gray + $originalSdkEnv = if (Test-Path $sdkEnvFile) { Get-Content $sdkEnvFile -Raw } else { $null } + Set-Content $sdkEnvFile "npm_config_registry=$NpmRegistry`n" -Encoding utf8 -NoNewline + $wroteSdkEnv = $true + Write-Host " Wrote $sdkEnvFile" -ForegroundColor Green + Write-Host "" + } + + # Libraries in service directories that share a code generation plugin must not be built + # concurrently with each other, so they are regenerated in a serial batch after the parallel one. + $parallelLibraries = @($Libraries | Where-Object { $_.Service -notin $SerialServiceDirectories }) + $serialLibraries = @($Libraries | Where-Object { $_.Service -in $SerialServiceDirectories }) + + $batches = @() + if ($parallelLibraries.Count -gt 0) { + $batches += , @{ Libraries = $parallelLibraries; Throttle = $ThrottleLimit } + } + if ($serialLibraries.Count -gt 0) { + $batches += , @{ Libraries = $serialLibraries; Throttle = 1 } + } + + $results = @() + + try { + foreach ($batch in $batches) { + $batchLibraries = $batch.Libraries + $batchThrottle = $batch.Throttle + Write-Host "Dispatching $($batchLibraries.Count) regeneration jobs ($batchThrottle at a time)..." -ForegroundColor Cyan + # Run regeneration in parallel + $results += $batchLibraries | ForEach-Object -ThrottleLimit $batchThrottle -Parallel { + $library = $_ + $azureSdkPath = $using:SdkRepoPath + $completedBag = $using:completed + $total = $using:totalCount + $extraArgs = $using:buildArgs + + Write-Host " -> Starting $($library.Library) ($($library.Service))" -ForegroundColor DarkGray + + # Determine build path (check for src subdirectory) + $libraryPath = Join-Path $azureSdkPath $library.Path + $srcPath = Join-Path $libraryPath "src" + $buildPath = if ((Test-Path $srcPath) -and (Get-ChildItem -Path $srcPath -Filter "*.csproj" -ErrorAction SilentlyContinue)) { + $srcPath + } else { + $libraryPath + } + + # Regenerate library + $result = try { + if (-not (Test-Path $libraryPath)) { + @{ Success = $false; Error = "Library path not found"; Output = "" } + } else { + Push-Location $buildPath + try { + $output = & dotnet build /t:GenerateCode /p:SkipTspClientInstall=true /p:SkipBuildPlugin=true @extraArgs 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + @{ Success = $false; Error = "Generation failed with exit code $exitCode"; Output = ($output -join "`n") } + } else { + @{ Success = $true; Output = ($output -join "`n") } + } + } + finally { + Pop-Location + } + } + } + catch { + @{ Success = $false; Error = $_.Exception.Message; Output = $_.Exception.ToString() } + } + + # Update progress counter + $completedBag.Add(1) + $currentCount = $completedBag.Count + + # Thread-safe console output with progress + $status = if ($result.Success) { "✓" } else { "✗" } + $color = if ($result.Success) { "Green" } else { "White" } + + $progressMsg = "[$currentCount/$total] $status $($library.Library)" + Write-Host $progressMsg -ForegroundColor $color + + # Return result with library metadata + return @{ + Service = $library.Service + Library = $library.Library + Path = $library.Path + Generator = $library.Generator + Success = if ($result.ContainsKey('Success')) { $result.Success } else { $false } + Error = if ($result.ContainsKey('Error')) { $result.Error } else { "" } + Output = if ($result.ContainsKey('Output')) { $result.Output } else { "" } + } + } + } + } + finally { + # Remove/restore the temporary .env used to redirect tsp-client's npm registry + if ($wroteSdkEnv) { + if ($null -eq $originalSdkEnv) { + Remove-Item $sdkEnvFile -Force -ErrorAction SilentlyContinue + } else { + Set-Content $sdkEnvFile $originalSdkEnv -Encoding utf8 -NoNewline + } + } + } + + return @($results) +} + +function Write-RegenerationReport { + <# + .SYNOPSIS + Writes a summary of the regeneration results to the console and optionally to a JSON file. + + .PARAMETER Results + The result objects returned by Invoke-SdkLibraryRegeneration. + + .PARAMETER ElapsedTime + Optional. The total time taken to produce the results. + + .PARAMETER ReportPath + Optional. When specified, the detailed results are also written as JSON to this path. + #> + param( + [Parameter(Mandatory = $true)] + [array]$Results, + + [Parameter(Mandatory = $false)] + [TimeSpan]$ElapsedTime, + + [Parameter(Mandatory = $false)] + [string]$ReportPath + ) + + $passed = @($Results | Where-Object { $_.Success -eq $true }) + $failed = @($Results | Where-Object { $_.Success -eq $false }) + + Write-Host "`n==================== REGENERATION REPORT ====================" -ForegroundColor Cyan + Write-Host "Total Libraries: $($Results.Count)" -ForegroundColor White + Write-Host "Passed: $($passed.Count)" -ForegroundColor Green + Write-Host "Failed: $($failed.Count)" -ForegroundColor Red + + if ($ElapsedTime) { + $elapsedFormatted = "{0:hh\:mm\:ss}" -f $ElapsedTime + Write-Host "Execution Time: $elapsedFormatted" -ForegroundColor Cyan + } + Write-Host "" + + if ($passed.Count -gt 0) { + Write-Host "PASSED LIBRARIES:" -ForegroundColor Green + foreach ($result in $passed) { + Write-Host " ✓ $($result.Library) ($($result.Service))" -ForegroundColor Green + } + Write-Host "" + } + + if ($failed.Count -gt 0) { + Write-Host "FAILED LIBRARIES:" -ForegroundColor Red + foreach ($result in $failed) { + Write-Host " ✗ $($result.Library) ($($result.Service))" -ForegroundColor Red + Write-Host " Error: $($result.Error)" -ForegroundColor Gray + if ($result.Output) { + Write-Host " Details: $($result.Output.Substring(0, [Math]::Min(200, $result.Output.Length)))..." -ForegroundColor DarkGray + } + } + Write-Host "" + } + + Write-Host "=============================================================" -ForegroundColor Cyan + + # Save detailed report + if ($ReportPath) { + $Results | ConvertTo-Json -Depth 10 | Set-Content $ReportPath -Encoding utf8 + Write-Host "Detailed report saved to: $ReportPath" -ForegroundColor Gray + } +} + +Export-ModuleMember -Function "Update-MgmtGenerator", "Update-AzureGenerator", "Filter-LibrariesByGenerator", "Filter-LibrariesByName", "Update-OpenAIGenerator", "Add-LocalNuGetSource", "Update-AzureSpectorScenarios", "Get-SdkLibrariesToRegenerate", "Invoke-SdkLibraryRegeneration", "Write-RegenerationReport" diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 3e2b94ff1a2..842650e9c7e 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -23,6 +23,10 @@ Path to the build artifacts directory containing the published .tgz and .nupkg f The URL of the pipeline run that triggered this PR. When provided, it is included in the PR description for traceability. .PARAMETER BuildReason The reason the pipeline was triggered (for example, 'Manual', 'Schedule', or 'IndividualCI'). When set to 'Manual', step failures fail the pipeline instead of being downgraded to warnings and opening a PR. +.PARAMETER UseParallelRegeneration +When specified, SDK libraries are regenerated per library in parallel using the shared RegenPreview helpers instead of running 'dotnet msbuild service.proj /t:GenerateCode' once per service directory. This is intended for manual pipeline runs where turnaround time matters. +.PARAMETER RegenerationThrottleLimit +Optional. The number of concurrent library regenerations when -UseParallelRegeneration is specified. Defaults to (logical processors - 2), clamped between 1 and 8. #> [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -60,7 +64,13 @@ param( [switch]$UseTypeSpecNext, [Parameter(Mandatory = $false)] - [string]$BuildReason + [string]$BuildReason, + + [Parameter(Mandatory = $false)] + [switch]$UseParallelRegeneration, + + [Parameter(Mandatory = $false)] + [int]$RegenerationThrottleLimit = 0 ) # When the pipeline is triggered manually, failures should fail the pipeline with an @@ -580,6 +590,11 @@ try { } } + # Service directories whose libraries share a single code generator plugin that each + # library's generation builds into a common output folder. These services are regenerated + # serially since they share a common plugin project that shouldn't be built in parallel. + $serialCodeGenServiceDirectories = @("ai") + # Discover service directories with tsp-location.yaml referencing any of the matched emitter patterns $tspLocations = Get-ChildItem -Path (Join-Path $tempDir "sdk") -Filter "tsp-location.yaml" -Recurse $serviceDirectories = @() @@ -603,14 +618,41 @@ try { if ($serviceDirectories.Count -eq 0) { Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." + } elseif ($UseParallelRegeneration) { + # Manual runs regenerate each library directly (in parallel) instead of building an + # entire service directory at a time, which is significantly faster. + Write-Host "##[section]Regenerating SDK libraries in parallel..." + $librariesToRegenerate = @(Get-SdkLibrariesToRegenerate -SdkRepoPath $tempDir -EmitterPackageJsonPaths $emitterPatterns) + + if ($librariesToRegenerate.Count -eq 0) { + Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." + } else { + Write-Host "Regenerating $($librariesToRegenerate.Count) libraries across $($serviceDirectories.Count) service directories" + $regenerationStartTime = Get-Date + $previousErrorAction = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + $regenerationResults = @(Invoke-SdkLibraryRegeneration ` + -SdkRepoPath $tempDir ` + -Libraries $librariesToRegenerate ` + -ThrottleLimit $RegenerationThrottleLimit ` + -AdditionalBuildArgs @("/p:Trace=true") ` + -SerialServiceDirectories $serialCodeGenServiceDirectories) + + Write-RegenerationReport -Results $regenerationResults -ElapsedTime ((Get-Date) - $regenerationStartTime) + + foreach ($failedLibrary in @($regenerationResults | Where-Object { -not $_.Success })) { + Register-StepFailure "Code generation failed for $($failedLibrary.Path): $($failedLibrary.Error)" + } + } catch { + Register-StepFailure "Parallel code generation failed: $($_.Exception.Message). Continuing with PR creation." + } finally { + $ErrorActionPreference = $previousErrorAction + } + } } else { $serviceProj = Join-Path $tempDir "eng/service.proj" - # Service directories whose libraries share a single code generator plugin that each - # library's generation builds into a common output folder. These services are regenerated - # serially since they share a common plugin project that shouldn't be built in parallel. - $serialCodeGenServiceDirectories = @("ai") - foreach ($serviceDirectory in $serviceDirectories) { Write-Host "Regenerating code for service directory: $serviceDirectory" $previousErrorAction = $ErrorActionPreference diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index d7513081d8b..034179554f8 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -392,6 +392,21 @@ If all libraries regenerate successfully, the script restores modified files: **Note:** If any libraries fail, artifacts are NOT restored, allowing you to debug the issue with the modified configuration intact. +## Shared Regeneration Helpers + +The library discovery and parallel regeneration logic lives in `RegenPreview.psm1` so it can be reused outside of local validation runs: + +| Function | Description | +| --- | --- | +| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`) and to `Azure.*` libraries only (`-AzureLibrariesOnly`). | +| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, and `-SerialServiceDirectories` (service directories that must be regenerated one library at a time). | +| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. | + +`Submit-AzureSdkForNetPr.ps1` uses these helpers when it is invoked with `-UseParallelRegeneration`, which the +`packages/http-client-csharp/eng/pipeline/publish.yml` pipeline only passes for **manual** runs. Automated (CI and +scheduled) runs continue to regenerate one service directory at a time with +`dotnet msbuild eng/service.proj /t:GenerateCode`. + ### Error Handling If the script encounters an error during pre-requisite steps (Steps 1-6), it will: From c72ce0d619b8a0884d3421f247d197f42390a041 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:15:01 +0000 Subject: [PATCH 02/17] Report regenerated service count from discovered libraries Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 842650e9c7e..e7d338ce042 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -627,7 +627,8 @@ try { if ($librariesToRegenerate.Count -eq 0) { Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." } else { - Write-Host "Regenerating $($librariesToRegenerate.Count) libraries across $($serviceDirectories.Count) service directories" + $regeneratedServiceCount = @($librariesToRegenerate | ForEach-Object { $_.Service } | Sort-Object -Unique).Count + Write-Host "Regenerating $($librariesToRegenerate.Count) libraries across $regeneratedServiceCount service directories" $regenerationStartTime = Get-Date $previousErrorAction = $ErrorActionPreference $ErrorActionPreference = "Continue" From b9c091a8c01c741168220c85e7d5008288602a30 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:36:36 +0000 Subject: [PATCH 03/17] fix: address publish regen review feedback Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 8 +------- .../http-client-csharp/eng/scripts/docs/RegenPreview.md | 8 ++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index e7d338ce042..4aadc0fd376 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -25,8 +25,6 @@ The URL of the pipeline run that triggered this PR. When provided, it is include The reason the pipeline was triggered (for example, 'Manual', 'Schedule', or 'IndividualCI'). When set to 'Manual', step failures fail the pipeline instead of being downgraded to warnings and opening a PR. .PARAMETER UseParallelRegeneration When specified, SDK libraries are regenerated per library in parallel using the shared RegenPreview helpers instead of running 'dotnet msbuild service.proj /t:GenerateCode' once per service directory. This is intended for manual pipeline runs where turnaround time matters. -.PARAMETER RegenerationThrottleLimit -Optional. The number of concurrent library regenerations when -UseParallelRegeneration is specified. Defaults to (logical processors - 2), clamped between 1 and 8. #> [CmdletBinding(SupportsShouldProcess = $true)] param( @@ -67,10 +65,7 @@ param( [string]$BuildReason, [Parameter(Mandatory = $false)] - [switch]$UseParallelRegeneration, - - [Parameter(Mandatory = $false)] - [int]$RegenerationThrottleLimit = 0 + [switch]$UseParallelRegeneration ) # When the pipeline is triggered manually, failures should fail the pipeline with an @@ -636,7 +631,6 @@ try { $regenerationResults = @(Invoke-SdkLibraryRegeneration ` -SdkRepoPath $tempDir ` -Libraries $librariesToRegenerate ` - -ThrottleLimit $RegenerationThrottleLimit ` -AdditionalBuildArgs @("/p:Trace=true") ` -SerialServiceDirectories $serialCodeGenServiceDirectories) diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index 034179554f8..36555205968 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -396,11 +396,11 @@ If all libraries regenerate successfully, the script restores modified files: The library discovery and parallel regeneration logic lives in `RegenPreview.psm1` so it can be reused outside of local validation runs: -| Function | Description | -| --- | --- | -| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`) and to `Azure.*` libraries only (`-AzureLibrariesOnly`). | +| Function | Description | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`) and to `Azure.*` libraries only (`-AzureLibrariesOnly`). | | `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, and `-SerialServiceDirectories` (service directories that must be regenerated one library at a time). | -| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. | +| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. | `Submit-AzureSdkForNetPr.ps1` uses these helpers when it is invoked with `-UseParallelRegeneration`, which the `packages/http-client-csharp/eng/pipeline/publish.yml` pipeline only passes for **manual** runs. Automated (CI and From 59ac76f39e625a84775d9e1993c89c99aa3c0e7e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:51:12 +0000 Subject: [PATCH 04/17] fix: remove Azure-only regeneration helper flag Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../http-client-csharp/eng/scripts/RegenPreview.ps1 | 6 +++--- .../http-client-csharp/eng/scripts/RegenPreview.psm1 | 11 +---------- .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 7 ++++++- .../eng/scripts/docs/RegenPreview.md | 2 +- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 766b39be535..fa75201c434 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -444,7 +444,7 @@ try { if ($Select -and -not $isOpenAIMode) { Write-Host "`n[1/5] Loading TypeSpec libraries from repository..." -ForegroundColor Cyan - $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath -AzureLibrariesOnly + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath # Apply generator filter before interactive selection $filteredLibraries = @(Filter-LibrariesByGenerator ` @@ -687,7 +687,7 @@ try { $librariesToAnalyze = $librariesToRegenerate } else { # Load all libraries and apply filters to determine what would be regenerated - $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath -AzureLibrariesOnly + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath $librariesToAnalyze = Filter-LibrariesByGenerator ` -Libraries $allLibraries ` -Azure:$Azure ` @@ -832,7 +832,7 @@ try { if (-not $Select) { # Load all libraries if not using -Select flag - $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath -AzureLibrariesOnly + $allLibraries = Get-SdkLibrariesToRegenerate -SdkRepoPath $sdkRepoPath # Apply generator filter $librariesToRegenerate = Filter-LibrariesByGenerator ` diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index 8cadec613ee..a9ebe23ef34 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1023,18 +1023,13 @@ function Get-SdkLibrariesToRegenerate { Optional. Restricts the results to libraries referencing the specified emitter package json paths (for example 'eng/http-client-csharp-emitter-package.json'). When omitted, all known emitters match. - .PARAMETER AzureLibrariesOnly - Optional. When specified, only libraries whose directory name starts with 'Azure.' are returned. #> param( [Parameter(Mandatory = $true)] [string]$SdkRepoPath, [Parameter(Mandatory = $false)] - [string[]]$EmitterPackageJsonPaths, - - [Parameter(Mandatory = $false)] - [switch]$AzureLibrariesOnly + [string[]]$EmitterPackageJsonPaths ) $ErrorActionPreference = 'Stop' @@ -1105,10 +1100,6 @@ function Get-SdkLibrariesToRegenerate { continue } - if ($AzureLibrariesOnly -and -not $libraryDir.Name.StartsWith("Azure.")) { - continue - } - # If it has a /src directory, it's likely a library $srcPath = Join-Path $libraryDir.FullName "src" if (-not (Test-Path $srcPath)) { diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 4aadc0fd376..06926e75bc8 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -617,7 +617,12 @@ try { # Manual runs regenerate each library directly (in parallel) instead of building an # entire service directory at a time, which is significantly faster. Write-Host "##[section]Regenerating SDK libraries in parallel..." - $librariesToRegenerate = @(Get-SdkLibrariesToRegenerate -SdkRepoPath $tempDir -EmitterPackageJsonPaths $emitterPatterns) + $allLibraries = @(Get-SdkLibrariesToRegenerate -SdkRepoPath $tempDir) + $librariesToRegenerate = @(Filter-LibrariesByGenerator ` + -Libraries $allLibraries ` + -Azure:$RegenerateAzureLibraries ` + -Unbranded ` + -Mgmt:$RegenerateMgmtLibraries) if ($librariesToRegenerate.Count -eq 0) { Write-Host "No SDK libraries found matching emitter patterns. Skipping SDK regeneration." diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index 36555205968..b0104a8b84b 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -398,7 +398,7 @@ The library discovery and parallel regeneration logic lives in `RegenPreview.psm | Function | Description | | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`) and to `Azure.*` libraries only (`-AzureLibrariesOnly`). | +| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | | `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, and `-SerialServiceDirectories` (service directories that must be regenerated one library at a time). | | `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. | From 382058b351df833ac20e448caa35cab1093a764b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:03:52 +0000 Subject: [PATCH 05/17] fix: stage regen preview report in CI artifacts Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.ps1 | 14 +++++++++++--- .../eng/scripts/docs/RegenPreview.md | 5 ++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index fa75201c434..f0a8332c80a 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -485,6 +485,14 @@ try { } Write-Host "Debug folder: $debugFolder" -ForegroundColor Gray + $regenerationReportPath = Join-Path $debugFolder 'regen-report.json' + if ($env:TF_BUILD -and $env:BUILD_ARTIFACTSTAGINGDIRECTORY) { + $regenerationReportPath = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY 'regen-report.json' + $reportFolder = Split-Path $regenerationReportPath -Parent + if (-not (Test-Path $reportFolder)) { + New-Item -ItemType Directory -Path $reportFolder -Force | Out-Null + } + } Write-Host "" # Step 1: Build the unbranded generator @@ -605,7 +613,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath (Join-Path $debugFolder 'regen-report.json') + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath # Exit with appropriate code if ($result.Success) { @@ -653,7 +661,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath (Join-Path $debugFolder 'regen-report.json') + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath if ($result.Success) { Write-Host "`nScript completed successfully." -ForegroundColor Cyan @@ -881,7 +889,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -ReportPath (Join-Path $debugFolder 'regen-report.json') + Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath # Check if any libraries failed $failedLibraries = @($results | Where-Object { -not $_.Success }) diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index b0104a8b84b..3d39c0fec59 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -426,7 +426,8 @@ All packaged artifacts are stored in the `debug` folder at the root of the unbra - `Microsoft.TypeSpec.Generator.{version}.nupkg` - Core generator NuGet package - `Microsoft.TypeSpec.Generator.Input.{version}.nupkg` - Input models NuGet package - `Microsoft.TypeSpec.Generator.ClientModel.{version}.nupkg` - Client model NuGet package -- `regen-report.json` - Detailed JSON report of regeneration results +- `regen-report.json` - Detailed JSON report of regeneration results (written to the ADO artifact staging + directory during CI runs) ### Console Output @@ -463,6 +464,8 @@ FAILED LIBRARIES: Detailed report saved to: C:\...\debug\regen-report.json ``` +In ADO CI runs, the detailed JSON report is written to the artifact staging directory instead. + ## Common Scenarios ### Scenario 1: Test OpenAI Library Changes From bdc384851ddaf93263e70cb1c3c0353e21d5bf2c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:04:41 +0000 Subject: [PATCH 06/17] fix: simplify CI regen report path Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- packages/http-client-csharp/eng/scripts/RegenPreview.ps1 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index f0a8332c80a..6c2706753c8 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -488,10 +488,6 @@ try { $regenerationReportPath = Join-Path $debugFolder 'regen-report.json' if ($env:TF_BUILD -and $env:BUILD_ARTIFACTSTAGINGDIRECTORY) { $regenerationReportPath = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY 'regen-report.json' - $reportFolder = Split-Path $regenerationReportPath -Parent - if (-not (Test-Path $reportFolder)) { - New-Item -ItemType Directory -Path $reportFolder -Force | Out-Null - } } Write-Host "" From a2ccf5e73b21fbd496134ab997c1edc29829ebc5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:12:50 +0000 Subject: [PATCH 07/17] fix: simplify regen report staging condition Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- packages/http-client-csharp/eng/scripts/RegenPreview.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 6c2706753c8..98cc94af910 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -486,7 +486,7 @@ try { Write-Host "Debug folder: $debugFolder" -ForegroundColor Gray $regenerationReportPath = Join-Path $debugFolder 'regen-report.json' - if ($env:TF_BUILD -and $env:BUILD_ARTIFACTSTAGINGDIRECTORY) { + if ($env:BUILD_ARTIFACTSTAGINGDIRECTORY) { $regenerationReportPath = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY 'regen-report.json' } Write-Host "" From dc2b7c0617d9af10e4087d28c4f3abf8a5ab22af Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:15:13 +0000 Subject: [PATCH 08/17] fix: default SDK regen parallelism to 8 Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.psm1 | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index a9ebe23ef34..e97d806e63c 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1145,7 +1145,7 @@ function Invoke-SdkLibraryRegeneration { The libraries to regenerate, as returned by Get-SdkLibrariesToRegenerate. .PARAMETER ThrottleLimit - Optional. Number of concurrent regeneration jobs. Defaults to (logical processors - 2), clamped to 1-8. + Optional. Number of concurrent regeneration jobs. Defaults to 8. .PARAMETER NpmRegistry Optional. When specified, a temporary .env file is written to the repository root so tsp-client @@ -1184,21 +1184,10 @@ function Invoke-SdkLibraryRegeneration { return @() } - # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 if ($ThrottleLimit -le 0) { - $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { - (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum - } elseif ($IsMacOS) { - [int](sysctl -n hw.ncpu) - } else { - [int](nproc) - } - - $ThrottleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) - Write-Host "Using $ThrottleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray - } else { - Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray + $ThrottleLimit = 8 } + Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray Write-Host "" $engFolder = Join-Path $SdkRepoPath "eng" From 57ecc8f8ccdc6d75f704405ff6c0c5b094f107b8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:16:54 +0000 Subject: [PATCH 09/17] fix: make regen failures fatal and print JSON report Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.ps1 | 3 +- .../eng/scripts/RegenPreview.psm1 | 26 +++++++++++++++-- .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 28 ++++++++----------- .../eng/scripts/docs/RegenPreview.md | 28 +++++++++++++++++-- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 98cc94af910..153f1257374 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -893,8 +893,7 @@ try { } if ($failedCount -gt 0) { - Write-Host "`nValidation completed with warnings: $failedCount libraries failed to regenerate" -ForegroundColor Yellow - Write-Host "Check the detailed report above for error information" -ForegroundColor Yellow + throw "Validation failed: $failedCount libraries failed to regenerate. Check the detailed report above for error information." } else { Write-Host "`nValidation completed successfully! All libraries regenerated without errors." -ForegroundColor Green diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index e97d806e63c..cd22662ad38 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1258,7 +1258,7 @@ function Invoke-SdkLibraryRegeneration { $batchThrottle = $batch.Throttle Write-Host "Dispatching $($batchLibraries.Count) regeneration jobs ($batchThrottle at a time)..." -ForegroundColor Cyan # Run regeneration in parallel - $results += $batchLibraries | ForEach-Object -ThrottleLimit $batchThrottle -Parallel { + $batchResults = @($batchLibraries | ForEach-Object -ThrottleLimit $batchThrottle -Parallel { $library = $_ $azureSdkPath = $using:SdkRepoPath $completedBag = $using:completed @@ -1322,6 +1322,11 @@ function Invoke-SdkLibraryRegeneration { Error = if ($result.ContainsKey('Error')) { $result.Error } else { "" } Output = if ($result.ContainsKey('Output')) { $result.Output } else { "" } } + }) + $results += $batchResults + + if (@($batchResults | Where-Object { -not $_.Success }).Count -gt 0) { + break } } } @@ -1367,13 +1372,24 @@ function Write-RegenerationReport { $passed = @($Results | Where-Object { $_.Success -eq $true }) $failed = @($Results | Where-Object { $_.Success -eq $false }) + $elapsedFormatted = if ($ElapsedTime) { "{0:hh\:mm\:ss}" -f $ElapsedTime } else { $null } + $report = [ordered]@{ + Summary = [ordered]@{ + TotalLibraries = $Results.Count + Passed = $passed.Count + Failed = $failed.Count + ExecutionTime = $elapsedFormatted + } + Results = @($Results) + } + $reportJson = $report | ConvertTo-Json -Depth 10 + Write-Host "`n==================== REGENERATION REPORT ====================" -ForegroundColor Cyan Write-Host "Total Libraries: $($Results.Count)" -ForegroundColor White Write-Host "Passed: $($passed.Count)" -ForegroundColor Green Write-Host "Failed: $($failed.Count)" -ForegroundColor Red if ($ElapsedTime) { - $elapsedFormatted = "{0:hh\:mm\:ss}" -f $ElapsedTime Write-Host "Execution Time: $elapsedFormatted" -ForegroundColor Cyan } Write-Host "" @@ -1402,9 +1418,13 @@ function Write-RegenerationReport { # Save detailed report if ($ReportPath) { - $Results | ConvertTo-Json -Depth 10 | Set-Content $ReportPath -Encoding utf8 + $reportJson | Set-Content $ReportPath -Encoding utf8 Write-Host "Detailed report saved to: $ReportPath" -ForegroundColor Gray } + + Write-Host "" + Write-Host "REGENERATION REPORT JSON:" -ForegroundColor Cyan + Write-Host $reportJson } Export-ModuleMember -Function "Update-MgmtGenerator", "Update-AzureGenerator", "Filter-LibrariesByGenerator", "Filter-LibrariesByName", "Update-OpenAIGenerator", "Add-LocalNuGetSource", "Update-AzureSpectorScenarios", "Get-SdkLibrariesToRegenerate", "Invoke-SdkLibraryRegeneration", "Write-RegenerationReport" diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 06926e75bc8..5ffe283c288 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -630,24 +630,20 @@ try { $regeneratedServiceCount = @($librariesToRegenerate | ForEach-Object { $_.Service } | Sort-Object -Unique).Count Write-Host "Regenerating $($librariesToRegenerate.Count) libraries across $regeneratedServiceCount service directories" $regenerationStartTime = Get-Date - $previousErrorAction = $ErrorActionPreference - $ErrorActionPreference = "Continue" - try { - $regenerationResults = @(Invoke-SdkLibraryRegeneration ` - -SdkRepoPath $tempDir ` - -Libraries $librariesToRegenerate ` - -AdditionalBuildArgs @("/p:Trace=true") ` - -SerialServiceDirectories $serialCodeGenServiceDirectories) + $regenerationResults = @(Invoke-SdkLibraryRegeneration ` + -SdkRepoPath $tempDir ` + -Libraries $librariesToRegenerate ` + -AdditionalBuildArgs @("/p:Trace=true") ` + -SerialServiceDirectories $serialCodeGenServiceDirectories) - Write-RegenerationReport -Results $regenerationResults -ElapsedTime ((Get-Date) - $regenerationStartTime) + Write-RegenerationReport -Results $regenerationResults -ElapsedTime ((Get-Date) - $regenerationStartTime) - foreach ($failedLibrary in @($regenerationResults | Where-Object { -not $_.Success })) { - Register-StepFailure "Code generation failed for $($failedLibrary.Path): $($failedLibrary.Error)" - } - } catch { - Register-StepFailure "Parallel code generation failed: $($_.Exception.Message). Continuing with PR creation." - } finally { - $ErrorActionPreference = $previousErrorAction + $failedLibraries = @($regenerationResults | Where-Object { -not $_.Success }) + foreach ($failedLibrary in $failedLibraries) { + Register-StepFailure "Code generation failed for $($failedLibrary.Path): $($failedLibrary.Error)" + } + if ($failedLibraries.Count -gt 0) { + throw "Parallel code generation failed for $($failedLibraries.Count) libraries." } } } else { diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index 3d39c0fec59..dc49bcb8bce 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -400,7 +400,7 @@ The library discovery and parallel regeneration logic lives in `RegenPreview.psm | ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | | `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, and `-SerialServiceDirectories` (service directories that must be regenerated one library at a time). | -| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. | +| `Write-RegenerationReport` | Prints the pass/fail summary and a human-readable JSON report, and optionally writes the detailed JSON report. | `Submit-AzureSdkForNetPr.ps1` uses these helpers when it is invoked with `-UseParallelRegeneration`, which the `packages/http-client-csharp/eng/pipeline/publish.yml` pipeline only passes for **manual** runs. Automated (CI and @@ -442,7 +442,7 @@ The script provides colored console output with: ### Regeneration Report -After regeneration completes, a summary report is displayed: +After regeneration completes, a summary report is displayed, followed by the same report in human-readable JSON: ``` ==================== REGENERATION REPORT ==================== @@ -462,9 +462,31 @@ FAILED LIBRARIES: ============================================================= Detailed report saved to: C:\...\debug\regen-report.json + +REGENERATION REPORT JSON: +{ + "Summary": { + "TotalLibraries": 3, + "Passed": 2, + "Failed": 1, + "ExecutionTime": "00:02:45" + }, + "Results": [ + { + "Service": "ai", + "Library": "Azure.AI.VoiceLive", + "Path": "sdk/ai/Azure.AI.VoiceLive", + "Generator": "@azure-typespec/http-client-csharp", + "Success": true, + "Error": "", + "Output": "..." + } + ] +} ``` -In ADO CI runs, the detailed JSON report is written to the artifact staging directory instead. +In ADO CI runs, the detailed JSON report is written to the artifact staging directory instead. If any +library fails to regenerate, the script exits with an error after printing the report. ## Common Scenarios From 1567b7d6b934612bc22427a34edc9fd3dff5ed23 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:19:02 +0000 Subject: [PATCH 10/17] fix: report skipped regen batches after failures Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.psm1 | 19 ++++++++++++++++++- .../eng/scripts/docs/RegenPreview.md | 3 ++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index cd22662ad38..3decf69e073 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1253,7 +1253,8 @@ function Invoke-SdkLibraryRegeneration { $results = @() try { - foreach ($batch in $batches) { + for ($batchIndex = 0; $batchIndex -lt $batches.Count; $batchIndex++) { + $batch = $batches[$batchIndex] $batchLibraries = $batch.Libraries $batchThrottle = $batch.Throttle Write-Host "Dispatching $($batchLibraries.Count) regeneration jobs ($batchThrottle at a time)..." -ForegroundColor Cyan @@ -1326,6 +1327,22 @@ function Invoke-SdkLibraryRegeneration { $results += $batchResults if (@($batchResults | Where-Object { -not $_.Success }).Count -gt 0) { + $remainingLibraries = @() + for ($remainingBatchIndex = $batchIndex + 1; $remainingBatchIndex -lt $batches.Count; $remainingBatchIndex++) { + $remainingLibraries += $batches[$remainingBatchIndex].Libraries + } + foreach ($library in $remainingLibraries) { + $results += @{ + Service = $library.Service + Library = $library.Library + Path = $library.Path + Generator = $library.Generator + Success = $false + Error = "Skipped because an earlier regeneration batch failed" + Output = "" + Skipped = $true + } + } break } } diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index dc49bcb8bce..3ae516e54e4 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -486,7 +486,8 @@ REGENERATION REPORT JSON: ``` In ADO CI runs, the detailed JSON report is written to the artifact staging directory instead. If any -library fails to regenerate, the script exits with an error after printing the report. +library fails to regenerate, later regeneration batches are marked as skipped and the script exits with an +error after printing the report. ## Common Scenarios From be774cf17fd4b4b0034d82b371feb7579a91efaf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:30:52 +0000 Subject: [PATCH 11/17] fix: keep local regen preview failures nonfatal Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.ps1 | 19 ++++++--- .../eng/scripts/RegenPreview.psm1 | 33 +++++++++++---- .../eng/scripts/Submit-AzureSdkForNetPr.ps1 | 6 ++- .../eng/scripts/docs/RegenPreview.md | 41 +++++-------------- 4 files changed, 54 insertions(+), 45 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 153f1257374..2fdd4609b19 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -485,8 +485,9 @@ try { } Write-Host "Debug folder: $debugFolder" -ForegroundColor Gray + $isCiRun = [bool]$env:BUILD_ARTIFACTSTAGINGDIRECTORY $regenerationReportPath = Join-Path $debugFolder 'regen-report.json' - if ($env:BUILD_ARTIFACTSTAGINGDIRECTORY) { + if ($isCiRun) { $regenerationReportPath = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY 'regen-report.json' } Write-Host "" @@ -609,7 +610,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath -PrintJson:$isCiRun # Exit with appropriate code if ($result.Success) { @@ -657,7 +658,7 @@ try { $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath + Write-RegenerationReport -Results @($result) -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath -PrintJson:$isCiRun if ($result.Success) { Write-Host "`nScript completed successfully." -ForegroundColor Cyan @@ -879,13 +880,14 @@ try { $results = @(Invoke-SdkLibraryRegeneration ` -SdkRepoPath $sdkRepoPath ` -Libraries $librariesToRegenerate ` - -NpmRegistry $artifactFeedRegistry) + -NpmRegistry $artifactFeedRegistry ` + -StopOnFailure:$isCiRun) # Generate final report $scriptEndTime = Get-Date $elapsedTime = $scriptEndTime - $scriptStartTime - Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath + Write-RegenerationReport -Results $results -ElapsedTime $elapsedTime -ReportPath $regenerationReportPath -PrintJson:$isCiRun # Check if any libraries failed $failedLibraries = @($results | Where-Object { -not $_.Success }) @@ -893,7 +895,12 @@ try { } if ($failedCount -gt 0) { - throw "Validation failed: $failedCount libraries failed to regenerate. Check the detailed report above for error information." + if ($isCiRun) { + throw "Validation failed: $failedCount libraries failed to regenerate. Check the detailed report above for error information." + } + + Write-Host "`nValidation completed with warnings: $failedCount libraries failed to regenerate" -ForegroundColor Yellow + Write-Host "Check the detailed report above for error information" -ForegroundColor Yellow } else { Write-Host "`nValidation completed successfully! All libraries regenerated without errors." -ForegroundColor Green diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index 3decf69e073..e7ba72b4f75 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1157,6 +1157,9 @@ function Invoke-SdkLibraryRegeneration { .PARAMETER SerialServiceDirectories Optional. Names of service directories whose libraries share a code generation plugin and therefore must be regenerated one at a time. Those libraries are regenerated serially after the parallel batch. + + .PARAMETER StopOnFailure + Optional. Stops dispatching later regeneration batches after a batch reports any failures. #> param( [Parameter(Mandatory = $true)] @@ -1175,7 +1178,10 @@ function Invoke-SdkLibraryRegeneration { [string[]]$AdditionalBuildArgs = @(), [Parameter(Mandatory = $false)] - [string[]]$SerialServiceDirectories = @() + [string[]]$SerialServiceDirectories = @(), + + [Parameter(Mandatory = $false)] + [switch]$StopOnFailure ) $ErrorActionPreference = 'Stop' @@ -1326,7 +1332,7 @@ function Invoke-SdkLibraryRegeneration { }) $results += $batchResults - if (@($batchResults | Where-Object { -not $_.Success }).Count -gt 0) { + if ($StopOnFailure -and @($batchResults | Where-Object { -not $_.Success }).Count -gt 0) { $remainingLibraries = @() for ($remainingBatchIndex = $batchIndex + 1; $remainingBatchIndex -lt $batches.Count; $remainingBatchIndex++) { $remainingLibraries += $batches[$remainingBatchIndex].Libraries @@ -1374,6 +1380,9 @@ function Write-RegenerationReport { .PARAMETER ReportPath Optional. When specified, the detailed results are also written as JSON to this path. + + .PARAMETER PrintJson + Optional. Prints the detailed report as human-readable JSON. #> param( [Parameter(Mandatory = $true)] @@ -1383,7 +1392,10 @@ function Write-RegenerationReport { [TimeSpan]$ElapsedTime, [Parameter(Mandatory = $false)] - [string]$ReportPath + [string]$ReportPath, + + [Parameter(Mandatory = $false)] + [switch]$PrintJson ) $passed = @($Results | Where-Object { $_.Success -eq $true }) @@ -1435,13 +1447,20 @@ function Write-RegenerationReport { # Save detailed report if ($ReportPath) { - $reportJson | Set-Content $ReportPath -Encoding utf8 + $jsonToWrite = if ($PrintJson) { + $reportJson + } else { + $Results | ConvertTo-Json -Depth 10 + } + $jsonToWrite | Set-Content $ReportPath -Encoding utf8 Write-Host "Detailed report saved to: $ReportPath" -ForegroundColor Gray } - Write-Host "" - Write-Host "REGENERATION REPORT JSON:" -ForegroundColor Cyan - Write-Host $reportJson + if ($PrintJson) { + Write-Host "" + Write-Host "REGENERATION REPORT JSON:" -ForegroundColor Cyan + Write-Host $reportJson + } } Export-ModuleMember -Function "Update-MgmtGenerator", "Update-AzureGenerator", "Filter-LibrariesByGenerator", "Filter-LibrariesByName", "Update-OpenAIGenerator", "Add-LocalNuGetSource", "Update-AzureSpectorScenarios", "Get-SdkLibrariesToRegenerate", "Invoke-SdkLibraryRegeneration", "Write-RegenerationReport" diff --git a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 index 5ffe283c288..2078ddb098a 100755 --- a/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 +++ b/packages/http-client-csharp/eng/scripts/Submit-AzureSdkForNetPr.ps1 @@ -73,6 +73,7 @@ param( # false positive to reviewers. For automated (scheduled/CI) runs, keep the existing # behavior of reporting SucceededWithIssues and continuing. $FailOnError = $BuildReason -eq 'Manual' +$isCiRun = [bool]$env:BUILD_ARTIFACTSTAGINGDIRECTORY # Tracks non-fatal step failures that were downgraded to warnings so a manual run can # fail before creating a pull request. @@ -634,9 +635,10 @@ try { -SdkRepoPath $tempDir ` -Libraries $librariesToRegenerate ` -AdditionalBuildArgs @("/p:Trace=true") ` - -SerialServiceDirectories $serialCodeGenServiceDirectories) + -SerialServiceDirectories $serialCodeGenServiceDirectories ` + -StopOnFailure:$isCiRun) - Write-RegenerationReport -Results $regenerationResults -ElapsedTime ((Get-Date) - $regenerationStartTime) + Write-RegenerationReport -Results $regenerationResults -ElapsedTime ((Get-Date) - $regenerationStartTime) -PrintJson:$isCiRun $failedLibraries = @($regenerationResults | Where-Object { -not $_.Success }) foreach ($failedLibrary in $failedLibraries) { diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index 3ae516e54e4..05287ae6bf0 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -396,11 +396,11 @@ If all libraries regenerate successfully, the script restores modified files: The library discovery and parallel regeneration logic lives in `RegenPreview.psm1` so it can be reused outside of local validation runs: -| Function | Description | -| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | -| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, and `-SerialServiceDirectories` (service directories that must be regenerated one library at a time). | -| `Write-RegenerationReport` | Prints the pass/fail summary and a human-readable JSON report, and optionally writes the detailed JSON report. | +| Function | Description | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | +| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | +| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | `Submit-AzureSdkForNetPr.ps1` uses these helpers when it is invoked with `-UseParallelRegeneration`, which the `packages/http-client-csharp/eng/pipeline/publish.yml` pipeline only passes for **manual** runs. Automated (CI and @@ -442,7 +442,7 @@ The script provides colored console output with: ### Regeneration Report -After regeneration completes, a summary report is displayed, followed by the same report in human-readable JSON: +After regeneration completes, a summary report is displayed: ``` ==================== REGENERATION REPORT ==================== @@ -462,32 +462,13 @@ FAILED LIBRARIES: ============================================================= Detailed report saved to: C:\...\debug\regen-report.json - -REGENERATION REPORT JSON: -{ - "Summary": { - "TotalLibraries": 3, - "Passed": 2, - "Failed": 1, - "ExecutionTime": "00:02:45" - }, - "Results": [ - { - "Service": "ai", - "Library": "Azure.AI.VoiceLive", - "Path": "sdk/ai/Azure.AI.VoiceLive", - "Generator": "@azure-typespec/http-client-csharp", - "Success": true, - "Error": "", - "Output": "..." - } - ] -} ``` -In ADO CI runs, the detailed JSON report is written to the artifact staging directory instead. If any -library fails to regenerate, later regeneration batches are marked as skipped and the script exits with an -error after printing the report. +In ADO CI runs, the detailed JSON report is written to the artifact staging directory instead and is +also printed to the console in human-readable JSON. If any library fails to regenerate, later regeneration +batches are marked as skipped and the script exits with an error after printing the report. Local runs keep +the existing behavior: all regeneration batches run, and any library failures are reported as warnings at +the end. ## Common Scenarios From 64f8336edbde0ce0129fe06da49a17cb7e65ff3b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:31:53 +0000 Subject: [PATCH 12/17] fix: keep regen report file shape consistent Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- packages/http-client-csharp/eng/scripts/RegenPreview.psm1 | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index e7ba72b4f75..c43b2c1c74c 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1447,12 +1447,7 @@ function Write-RegenerationReport { # Save detailed report if ($ReportPath) { - $jsonToWrite = if ($PrintJson) { - $reportJson - } else { - $Results | ConvertTo-Json -Depth 10 - } - $jsonToWrite | Set-Content $ReportPath -Encoding utf8 + $reportJson | Set-Content $ReportPath -Encoding utf8 Write-Host "Detailed report saved to: $ReportPath" -ForegroundColor Gray } From f944a217b8a7092d038c62ad4e47551cc409dd4e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:52:16 +0000 Subject: [PATCH 13/17] fix: address regen preview review feedback Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.ps1 | 15 +++++++++++++++ .../eng/scripts/docs/RegenPreview.md | 5 ----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 index 2fdd4609b19..056a4134b38 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.ps1 @@ -877,10 +877,25 @@ try { Write-Host "No libraries selected for regeneration" -ForegroundColor Yellow $failedCount = 0 } else { + $regenerationThrottleLimit = 0 + if (-not $isCiRun) { + # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 + $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { + (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum + } elseif ($IsMacOS) { + [int](sysctl -n hw.ncpu) + } else { + [int](nproc) + } + + $regenerationThrottleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + } + $results = @(Invoke-SdkLibraryRegeneration ` -SdkRepoPath $sdkRepoPath ` -Libraries $librariesToRegenerate ` -NpmRegistry $artifactFeedRegistry ` + -ThrottleLimit $regenerationThrottleLimit ` -StopOnFailure:$isCiRun) # Generate final report diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index 05287ae6bf0..20d19f51edb 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -402,11 +402,6 @@ The library discovery and parallel regeneration logic lives in `RegenPreview.psm | `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | | `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | -`Submit-AzureSdkForNetPr.ps1` uses these helpers when it is invoked with `-UseParallelRegeneration`, which the -`packages/http-client-csharp/eng/pipeline/publish.yml` pipeline only passes for **manual** runs. Automated (CI and -scheduled) runs continue to regenerate one service directory at a time with -`dotnet msbuild eng/service.proj /t:GenerateCode`. - ### Error Handling If the script encounters an error during pre-requisite steps (Steps 1-6), it will: From 074575e4abcfcec2846e94c5d5762ade2e356403 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:33:36 +0000 Subject: [PATCH 14/17] Revert forced default of 8 concurrent regen jobs Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.psm1 | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index c43b2c1c74c..d00c90ded1e 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1145,7 +1145,7 @@ function Invoke-SdkLibraryRegeneration { The libraries to regenerate, as returned by Get-SdkLibrariesToRegenerate. .PARAMETER ThrottleLimit - Optional. Number of concurrent regeneration jobs. Defaults to 8. + Optional. Number of concurrent regeneration jobs. Defaults to (logical processors - 2), clamped to 1-8. .PARAMETER NpmRegistry Optional. When specified, a temporary .env file is written to the repository root so tsp-client @@ -1190,10 +1190,21 @@ function Invoke-SdkLibraryRegeneration { return @() } + # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 if ($ThrottleLimit -le 0) { - $ThrottleLimit = 8 + $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { + (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum + } elseif ($IsMacOS) { + [int](sysctl -n hw.ncpu) + } else { + [int](nproc) + } + + $ThrottleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + Write-Host "Using $ThrottleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray + } else { + Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray } - Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray Write-Host "" $engFolder = Join-Path $SdkRepoPath "eng" From 1d4d59ebe7438c176280517a786a13e87c926463 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:56:18 +0000 Subject: [PATCH 15/17] Oversubscribe CI regen throttle to 1.5x cores (min 4, max 8) Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../http-client-csharp/eng/scripts/RegenPreview.psm1 | 11 ++++++++--- .../eng/scripts/docs/RegenPreview.md | 10 +++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index d00c90ded1e..22ace690768 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1145,7 +1145,8 @@ function Invoke-SdkLibraryRegeneration { The libraries to regenerate, as returned by Get-SdkLibrariesToRegenerate. .PARAMETER ThrottleLimit - Optional. Number of concurrent regeneration jobs. Defaults to (logical processors - 2), clamped to 1-8. + Optional. Number of concurrent regeneration jobs. Defaults to 1.5x the logical processors, clamped to 4-8, + since each job is dominated by child process and IO wait rather than CPU work. .PARAMETER NpmRegistry Optional. When specified, a temporary .env file is written to the repository root so tsp-client @@ -1190,7 +1191,11 @@ function Invoke-SdkLibraryRegeneration { return @() } - # Determine parallel execution throttle limit: (CPU cores - 2), min 1, max 8 + # Determine parallel execution throttle limit. Each regeneration job spends most of its time + # waiting on child processes (tsp-client spec sync, NuGet/npm restore, file IO), so the machine + # can run more jobs than it has cores without saturating the CPU. Oversubscribe by 1.5x the + # logical processors, with a floor of 4 so low-core CI agents still get useful concurrency and a + # cap of 8 to avoid thrashing memory on those same agents. if ($ThrottleLimit -le 0) { $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum @@ -1200,7 +1205,7 @@ function Invoke-SdkLibraryRegeneration { [int](nproc) } - $ThrottleLimit = [Math]::Max(1, [Math]::Min(8, $cpuCores - 2)) + $ThrottleLimit = [Math]::Max(4, [Math]::Min(8, [int][Math]::Ceiling($cpuCores * 1.5))) Write-Host "Using $ThrottleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray } else { Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index 20d19f51edb..489551f8bf0 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -396,11 +396,11 @@ If all libraries regenerate successfully, the script restores modified files: The library discovery and parallel regeneration logic lives in `RegenPreview.psm1` so it can be reused outside of local validation runs: -| Function | Description | -| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | -| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit`, `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | -| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | +| Function | Description | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | +| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit` (defaults to 1.5x the logical processors, clamped to 4-8), `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | +| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | ### Error Handling From d859ce8e3a43c54ba70320205dd1dc24a79fc9b4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:43:17 +0000 Subject: [PATCH 16/17] Increase default regen parallelism and track per-library durations Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../eng/scripts/RegenPreview.psm1 | 57 ++++++++++++------- .../eng/scripts/docs/RegenPreview.md | 2 +- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 index 22ace690768..118e58a3756 100644 --- a/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 +++ b/packages/http-client-csharp/eng/scripts/RegenPreview.psm1 @@ -1145,7 +1145,7 @@ function Invoke-SdkLibraryRegeneration { The libraries to regenerate, as returned by Get-SdkLibrariesToRegenerate. .PARAMETER ThrottleLimit - Optional. Number of concurrent regeneration jobs. Defaults to 1.5x the logical processors, clamped to 4-8, + Optional. Number of concurrent regeneration jobs. Defaults to 3x the logical processors, clamped to 4-12, since each job is dominated by child process and IO wait rather than CPU work. .PARAMETER NpmRegistry @@ -1193,9 +1193,9 @@ function Invoke-SdkLibraryRegeneration { # Determine parallel execution throttle limit. Each regeneration job spends most of its time # waiting on child processes (tsp-client spec sync, NuGet/npm restore, file IO), so the machine - # can run more jobs than it has cores without saturating the CPU. Oversubscribe by 1.5x the + # can run more jobs than it has cores without saturating the CPU. Oversubscribe by 3x the # logical processors, with a floor of 4 so low-core CI agents still get useful concurrency and a - # cap of 8 to avoid thrashing memory on those same agents. + # cap of 12 to keep memory usage on those same agents bounded. if ($ThrottleLimit -le 0) { $cpuCores = if ($IsWindows -or $PSVersionTable.PSVersion.Major -lt 6) { (Get-CimInstance -ClassName Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum @@ -1205,7 +1205,7 @@ function Invoke-SdkLibraryRegeneration { [int](nproc) } - $ThrottleLimit = [Math]::Max(4, [Math]::Min(8, [int][Math]::Ceiling($cpuCores * 1.5))) + $ThrottleLimit = [Math]::Max(4, [Math]::Min(12, $cpuCores * 3)) Write-Host "Using $ThrottleLimit concurrent jobs (detected $cpuCores logical processors)" -ForegroundColor Gray } else { Write-Host "Using $ThrottleLimit concurrent jobs" -ForegroundColor Gray @@ -1289,6 +1289,7 @@ function Invoke-SdkLibraryRegeneration { $extraArgs = $using:buildArgs Write-Host " -> Starting $($library.Library) ($($library.Service))" -ForegroundColor DarkGray + $libraryStopwatch = [System.Diagnostics.Stopwatch]::StartNew() # Determine build path (check for src subdirectory) $libraryPath = Join-Path $azureSdkPath $library.Path @@ -1325,6 +1326,8 @@ function Invoke-SdkLibraryRegeneration { } # Update progress counter + $libraryStopwatch.Stop() + $durationSeconds = [Math]::Round($libraryStopwatch.Elapsed.TotalSeconds, 1) $completedBag.Add(1) $currentCount = $completedBag.Count @@ -1332,18 +1335,19 @@ function Invoke-SdkLibraryRegeneration { $status = if ($result.Success) { "✓" } else { "✗" } $color = if ($result.Success) { "Green" } else { "White" } - $progressMsg = "[$currentCount/$total] $status $($library.Library)" + $progressMsg = "[$currentCount/$total] $status $($library.Library) ($($durationSeconds)s)" Write-Host $progressMsg -ForegroundColor $color # Return result with library metadata return @{ - Service = $library.Service - Library = $library.Library - Path = $library.Path - Generator = $library.Generator - Success = if ($result.ContainsKey('Success')) { $result.Success } else { $false } - Error = if ($result.ContainsKey('Error')) { $result.Error } else { "" } - Output = if ($result.ContainsKey('Output')) { $result.Output } else { "" } + Service = $library.Service + Library = $library.Library + Path = $library.Path + Generator = $library.Generator + DurationSeconds = $durationSeconds + Success = if ($result.ContainsKey('Success')) { $result.Success } else { $false } + Error = if ($result.ContainsKey('Error')) { $result.Error } else { "" } + Output = if ($result.ContainsKey('Output')) { $result.Output } else { "" } } }) $results += $batchResults @@ -1355,14 +1359,15 @@ function Invoke-SdkLibraryRegeneration { } foreach ($library in $remainingLibraries) { $results += @{ - Service = $library.Service - Library = $library.Library - Path = $library.Path - Generator = $library.Generator - Success = $false - Error = "Skipped because an earlier regeneration batch failed" - Output = "" - Skipped = $true + Service = $library.Service + Library = $library.Library + Path = $library.Path + Generator = $library.Generator + DurationSeconds = 0 + Success = $false + Error = "Skipped because an earlier regeneration batch failed" + Output = "" + Skipped = $true } } break @@ -1442,7 +1447,17 @@ function Write-RegenerationReport { if ($passed.Count -gt 0) { Write-Host "PASSED LIBRARIES:" -ForegroundColor Green foreach ($result in $passed) { - Write-Host " ✓ $($result.Library) ($($result.Service))" -ForegroundColor Green + $duration = if ($result.DurationSeconds) { " - $($result.DurationSeconds)s" } else { "" } + Write-Host " ✓ $($result.Library) ($($result.Service))$duration" -ForegroundColor Green + } + Write-Host "" + } + + $timed = @($Results | Where-Object { $_.DurationSeconds } | Sort-Object -Property DurationSeconds -Descending | Select-Object -First 5) + if ($timed.Count -gt 0) { + Write-Host "SLOWEST LIBRARIES:" -ForegroundColor Cyan + foreach ($result in $timed) { + Write-Host " $($result.Library) ($($result.Service)) - $($result.DurationSeconds)s" -ForegroundColor Gray } Write-Host "" } diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index 489551f8bf0..feb76a40693 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -399,7 +399,7 @@ The library discovery and parallel regeneration logic lives in `RegenPreview.psm | Function | Description | | ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | -| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit` (defaults to 1.5x the logical processors, clamped to 4-8), `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | +| `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit` (defaults to 3x the logical processors, clamped to 4-12), `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | | `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | ### Error Handling From ab011bcf11e6c3b21dd4e4c0dab8ff70b34e3aff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:57:13 +0000 Subject: [PATCH 17/17] Fix prettier formatting in RegenPreview.md Co-authored-by: jorgerangel-msft <102122018+jorgerangel-msft@users.noreply.github.com> --- .../http-client-csharp/eng/scripts/docs/RegenPreview.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md index feb76a40693..e52f3e75c5b 100644 --- a/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md +++ b/packages/http-client-csharp/eng/scripts/docs/RegenPreview.md @@ -396,11 +396,11 @@ If all libraries regenerate successfully, the script restores modified files: The library discovery and parallel regeneration logic lives in `RegenPreview.psm1` so it can be reused outside of local validation runs: -| Function | Description | -| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | +| Function | Description | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Get-SdkLibrariesToRegenerate` | Scans `sdk/` in azure-sdk-for-net and returns the libraries whose `tsp-location.yaml` references one of the TypeSpec C# emitter package json artifacts. Supports filtering by emitter (`-EmitterPackageJsonPaths`). | | `Invoke-SdkLibraryRegeneration` | Pre-installs tsp-client, pre-builds the code generation plugin, and then regenerates the given libraries in parallel with `dotnet build /t:GenerateCode`. Supports `-ThrottleLimit` (defaults to 3x the logical processors, clamped to 4-12), `-NpmRegistry`, `-AdditionalBuildArgs`, `-SerialServiceDirectories` (service directories that must be regenerated one library at a time), and `-StopOnFailure`. | -| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | +| `Write-RegenerationReport` | Prints the pass/fail summary and optionally writes the detailed JSON report. In CI mode, it also prints a human-readable JSON report. | ### Error Handling