From 41394e44fe3cf552b25c26c877878f7d4e0147a2 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 14 Apr 2026 15:00:31 -0700 Subject: [PATCH 1/7] Add buildcache channel for Build Caching Service runtime resolution Adds a new 'buildcache' channel that resolves .NET runtime binaries from the Build Caching Service (BCS) instead of VMR feeds. This provides per-commit granularity for performance regression bisection. Key changes: - New Job.cs properties: BuildCacheCommitSha, BuildCacheBranch, BuildCacheConfig - New BuildCacheClient.cs: HTTP client for BCS latestBuilds.json and artifact download/extraction with post-build overlay into published output - Startup.cs: 'buildcache' channel in version resolution that builds with real NuGet packages then overlays BCS runtime binaries (194 files) after publish - Agent CLI options: --build-cache-base-url, --build-cache-repo-name, --build-cache-disabled - Documentation in docs/dotnet_versions.md and docs/build_cache_requirements.md Usage: --application.channel buildcache [--application.buildCacheCommitSha ] Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/build_cache_requirements.md | 95 +++ docs/dotnet_versions.md | 53 +- src/Microsoft.Crank.Agent/BuildCacheClient.cs | 794 ++++++++++++++++++ src/Microsoft.Crank.Agent/Startup.cs | 104 ++- src/Microsoft.Crank.Models/Job.cs | 5 + 5 files changed, 1047 insertions(+), 4 deletions(-) create mode 100644 docs/build_cache_requirements.md create mode 100644 src/Microsoft.Crank.Agent/BuildCacheClient.cs diff --git a/docs/build_cache_requirements.md b/docs/build_cache_requirements.md new file mode 100644 index 000000000..3efac3690 --- /dev/null +++ b/docs/build_cache_requirements.md @@ -0,0 +1,95 @@ +# Build Cache Service — Requirements for Crank Integration + +## Context + +Crank (the .NET benchmarking tool) has been updated to support a new `buildcache` channel that downloads pre-built runtime binaries from the Build Cache Service (BCS) instead of resolving versions from VMR/NuGet feeds. This gives per-commit granularity for performance testing and regression bisection. + +The crank-side changes are complete. This document describes what's needed on the BCS/dotnet-performance-infra side to make the integration work end-to-end. + +--- + +## Requirement 1: Public Blob Access + +**Status:** Already in progress (per prior discussion). + +Crank's BCS client uses unauthenticated HTTP GET requests to download artifacts. The blobs in the `pvscmdupload` storage account's `$web` container need to be publicly readable. + +**URLs crank will hit:** + +``` +GET https://pvscmdupload.z22.web.core.windows.net/builds/{repoName}/latest/{branch}/latestBuilds.json +GET https://pvscmdupload.z22.web.core.windows.net/builds/{repoName}/buildArtifacts/{commitSha}/{configKey}/{artifactFile} +``` + +Where: +- `repoName` = `runtime` (initially; `aspnetcore` in the future) +- `branch` = e.g., `main`, `release/10.0` +- `configKey` = e.g., `coreclr_x64_linux`, `coreclr_arm64_windows` +- `artifactFile` = e.g., `BuildArtifacts_linux_x64_Release_coreclr.tar.gz` + +--- + +## Requirement 2: Commit Index File (Not Required) + +~~Originally proposed as a per-branch `commitIndex.json` mapping commits to timestamps.~~ + +**Decision:** Not needed. For the default case, `latestBuilds.json` provides the latest commit. For specific-commit runs (e.g., bisection), users will already know the SHAs — either from git history, GitHub, or a local tool that queries the GitHub API for the commit list. A separate index in BCS would be redundant. + +If automated bisection tooling is built in the future, it can query GitHub directly for ordered commit SHAs and then check BCS blob existence per-commit. + +--- + +## Requirement 3: latestBuilds.json Compatibility + +**Resolved.** The actual `latestBuilds.json` uses PascalCase (`CommitSha`, `CommitTime`), not snake_case. Crank's parser has been updated to accept both casings for forward compatibility. + +--- + +## Requirement 4: Artifact Layout Stability + +Crank extracts runtime artifacts using this path convention inside the archive: + +``` +microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net{X}.0/ → managed DLLs +microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/native/ → native libs +{rid}.Release/corehost/ → host binaries (dotnet, libhostfxr, libhostpolicy) +``` + +Where `{rid}` = `linux-x64`, `linux-arm64`, `win-x64`, etc. + +This layout was confirmed by inspecting `BuildArtifacts_linux_arm64_Release_coreclr.tar.gz`. **If this layout changes in future builds, the crank extraction will break.** Consider treating it as a stable contract or documenting it. + +--- + +## Nice-to-Have: Artifact Manifest + +A `manifest.json` per commit+config that describes the archive contents would make extraction more robust: + +``` +builds/{repoName}/buildArtifacts/{commitSha}/{configKey}/manifest.json +``` + +```json +{ + "runtimeVersion": "10.0.0-preview.4.26120.3", + "commitSha": "abc123...", + "rid": "linux-arm64", + "managedPath": "microsoft.netcore.app.runtime.linux-arm64/Release/runtimes/linux-arm64/lib/net10.0", + "nativePath": "microsoft.netcore.app.runtime.linux-arm64/Release/runtimes/linux-arm64/native", + "corehostPath": "linux-arm64.Release/corehost" +} +``` + +This isn't blocking — crank currently discovers paths by convention — but it would decouple crank from the internal archive layout and make future changes safe. + +--- + +## Summary + +| # | Requirement | Priority | Blocking? | +|---|-------------|----------|-----------| +| 1 | Public blob access | High | Yes — crank can't download without it | +| 2 | ~~Commit index~~ | N/A | Dropped — users provide SHAs directly or use GitHub | +| 3 | `latestBuilds.json` field names | N/A | Resolved — crank parser updated to handle PascalCase | +| 4 | Artifact layout stability | Medium | Not now, but breaking changes would break crank | +| 5 | Artifact manifest.json | Low | Nice-to-have for robustness | diff --git a/docs/dotnet_versions.md b/docs/dotnet_versions.md index 9c48d5305..1e919106c 100644 --- a/docs/dotnet_versions.md +++ b/docs/dotnet_versions.md @@ -55,9 +55,12 @@ When a TFM is configured, the agent will download the corresponding .NET SDK ver - `current`: only latest public versions, this is the default - `latest`: latest versions used by ASP.NET - `edge`: latest nightly builds available +- `buildcache`: runtime from the Build Cache Service (per-commit builds) The difference between `latest` and `edge` is that `latest` will pick runtimes and SDKs that are deemed compatible together. For instance a very recent .NET core runtime might be compatible with a less recent ASP.NET runtime. The `edge` is used to pick the absolute latest build for the select TFM. +The `buildcache` channel uses the Build Cache Service (BCS) from `dotnet-performance-infra` to resolve runtime versions by individual commit SHA rather than from VMR feeds. This provides much finer-grained control — every cached runtime commit is available, whereas VMR feeds may have multi-day gaps between ingested commits. SDK and ASP.NET Core versions are resolved from `latest` when using `buildcache`. + In order to benchmark and ASP.NET application using very recent runtimes of .NET 5, the `latest` channel is recommended: ``` @@ -115,4 +118,52 @@ The following command uses the `edge` channel but ASP.NET is fixed so it doesn't ``` > crank --config /crank/samples/hello/hello.benchmarks.yml --scenario hello --profile local --application.framework netcoreapp5.0 --application.channel edge --application.aspnetCoreVersion 5.0.0-preview.6.20279.12 -``` \ No newline at end of file +``` + +## Using the Build Cache channel + +The `buildcache` channel resolves the .NET runtime from the Build Cache Service (BCS), which caches pre-built runtime binaries for individual commits. This is useful for performance regression bisection where VMR feed gaps make it hard to pinpoint which commit caused a regression. + +### Basic usage (latest cached build on main) + +``` +> crank --config benchmarks.yml --scenario json --profile aspnet-perf-lin --application.channel buildcache +``` + +### Specific commit SHA + +``` +> crank --config benchmarks.yml --scenario json --profile aspnet-perf-lin --application.channel buildcache --application.buildCacheCommitSha a1b2c3d4e5f6... +``` + +If the commit is not found in the cache, crank will fail with an error rather than falling back. + +### Different branch + +``` +> crank --config benchmarks.yml --scenario json --profile aspnet-perf-lin --application.channel buildcache --application.buildCacheBranch release/10.0 +``` + +### Mixed channels (BCS runtime + pinned ASP.NET) + +``` +> crank --config benchmarks.yml --scenario json --profile aspnet-perf-lin --application.channel buildcache --application.aspNetCoreVersion 10.0.0-preview.3.26115.7 +``` + +### Build Cache properties + +| Property | Default | Description | +|----------|---------|-------------| +| `buildCacheCommitSha` | (empty) | Specific runtime commit SHA. If empty, uses the latest cached build for the branch. | +| `buildCacheBranch` | `main` | Branch to query for the latest build. | +| `buildCacheConfig` | (auto-detected) | BCS configuration key (e.g., `coreclr_x64_linux`). Auto-detected from agent platform. | + +### Agent configuration + +The agent supports these command-line options for BCS: + +| Option | Default | Description | +|--------|---------|-------------| +| `--build-cache-base-url` | `https://pvscmdupload.z22.web.core.windows.net` | Base URL for BCS blob storage. | +| `--build-cache-repo-name` | `runtime` | Repository name in BCS. | +| `--build-cache-disabled` | (not set) | Disables BCS integration on this agent. | \ No newline at end of file diff --git a/src/Microsoft.Crank.Agent/BuildCacheClient.cs b/src/Microsoft.Crank.Agent/BuildCacheClient.cs new file mode 100644 index 000000000..b6c53fac0 --- /dev/null +++ b/src/Microsoft.Crank.Agent/BuildCacheClient.cs @@ -0,0 +1,794 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net.Http; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Crank.Agent +{ + /// + /// Lightweight client for the Build Caching Service (BCS) in dotnet-performance-infra. + /// Downloads pre-built runtime artifacts from public Azure Blob Storage and overlays + /// them into a standard dotnet installation directory. + /// + internal static class BuildCacheClient + { + private static readonly HttpClient _httpClient = new HttpClient { Timeout = TimeSpan.FromMinutes(10) }; + + // Cache latestBuilds.json responses to avoid repeated downloads (keyed by baseUrl+branch) + private static readonly ConcurrentDictionary _latestBuildsCache = new(); + private static readonly TimeSpan _latestBuildsCacheDuration = TimeSpan.FromHours(1); + + // Cache of already-installed BCS commit SHAs to avoid re-extracting + private static readonly ConcurrentDictionary _installedBuildCacheRuntimes = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Maps the agent's platform (RID) to the BCS configuration key and artifact filename. + /// + private static readonly Dictionary _platformToBcsConfig = new(StringComparer.OrdinalIgnoreCase) + { + ["linux-x64"] = ("coreclr_x64_linux", "BuildArtifacts_linux_x64_Release_coreclr.tar.gz"), + ["linux-arm64"] = ("coreclr_arm64_linux", "BuildArtifacts_linux_arm64_Release_coreclr.tar.gz"), + ["linux-musl-x64"] = ("coreclr_muslx64_linux", "BuildArtifacts_linux_musl_x64_Release_coreclr.tar.gz"), + ["win-x64"] = ("coreclr_x64_windows", "BuildArtifacts_windows_x64_Release_coreclr.zip"), + ["win-arm64"] = ("coreclr_arm64_windows", "BuildArtifacts_windows_arm64_Release_coreclr.zip"), + ["win-x86"] = ("coreclr_x86_windows", "BuildArtifacts_windows_x86_Release_coreclr.zip"), + }; + + /// + /// Resolves the commit SHA to use from BCS. If a specific commit is provided, validates + /// it exists. Otherwise queries latestBuilds.json for the latest commit on the branch. + /// Returns the commit SHA and the runtime version string. + /// + public static async Task<(string commitSha, string runtimeVersion)> ResolveCommitAsync( + string baseUrl, + string repoName, + string branch, + string commitSha, + string buildCacheConfig, + CancellationToken cancellationToken = default) + { + var platformMoniker = GetPlatformMoniker(); + + if (!string.IsNullOrEmpty(buildCacheConfig)) + { + // Use explicit config key + } + else if (_platformToBcsConfig.TryGetValue(platformMoniker, out var mapped)) + { + buildCacheConfig = mapped.configKey; + } + else + { + throw new InvalidOperationException($"No Build Cache configuration mapping for platform '{platformMoniker}'. Specify buildCacheConfig explicitly."); + } + + if (string.IsNullOrEmpty(commitSha)) + { + // Query latestBuilds.json for the latest commit + var latestBuilds = await GetLatestBuildsAsync(baseUrl, repoName, branch, cancellationToken); + + // Try to get the config-specific entry, fall back to "all" + if (latestBuilds.Entries.TryGetValue(buildCacheConfig, out var configEntry) && !string.IsNullOrEmpty(configEntry.CommitSha)) + { + commitSha = configEntry.CommitSha; + Log.Info($"Build Cache: Using latest commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))} for config '{buildCacheConfig}' on branch '{branch}' (committed {configEntry.CommitTime})"); + } + else if (latestBuilds.Entries.TryGetValue("all", out var allEntry) && !string.IsNullOrEmpty(allEntry.CommitSha)) + { + commitSha = allEntry.CommitSha; + Log.Info($"Build Cache: Using latest commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))} for all configs on branch '{branch}' (committed {allEntry.CommitTime})"); + } + else + { + throw new InvalidOperationException($"Build Cache: No latest build found for branch '{branch}'. Check that BCS has builds for this branch."); + } + } + else + { + Log.Info($"Build Cache: Using specified commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))}"); + } + + // Try to determine a runtime version from the commit. For now, we return a placeholder + // that will be replaced after extraction by reading .version from the shared framework. + var runtimeVersion = $"buildcache-{commitSha.Substring(0, Math.Min(12, commitSha.Length))}"; + + return (commitSha, runtimeVersion); + } + + /// + /// Downloads and extracts BCS runtime artifacts to a temp directory without overlaying. + /// Returns the path to the extracted directory for later overlay into published output. + /// + public static async Task DownloadAndExtractAsync( + string baseUrl, + string repoName, + string commitSha, + string buildCacheConfig, + string targetFramework, + CancellationToken cancellationToken = default) + { + var platformMoniker = GetPlatformMoniker(); + + if (string.IsNullOrEmpty(buildCacheConfig)) + { + if (_platformToBcsConfig.TryGetValue(platformMoniker, out var mapped)) + { + buildCacheConfig = mapped.configKey; + } + else + { + throw new InvalidOperationException($"No Build Cache configuration mapping for platform '{platformMoniker}'."); + } + } + + string artifactFile; + if (_platformToBcsConfig.Values.Any(v => v.configKey == buildCacheConfig)) + { + artifactFile = _platformToBcsConfig.Values.First(v => v.configKey == buildCacheConfig).artifactFile; + } + else + { + throw new InvalidOperationException($"Unknown Build Cache configuration key: '{buildCacheConfig}'."); + } + + var artifactUrl = $"{baseUrl}/builds/{repoName}/buildArtifacts/{commitSha}/{buildCacheConfig}/{artifactFile}"; + + Log.Info($"Build Cache: Downloading {artifactFile} from {artifactUrl}"); + + var tempDir = Path.Combine(Path.GetTempPath(), "crank-buildcache", commitSha); + Directory.CreateDirectory(tempDir); + var tempArchive = Path.Combine(tempDir, artifactFile); + + if (!File.Exists(tempArchive)) + { + using var response = await _httpClient.GetAsync(artifactUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + throw new InvalidOperationException($"Build Cache: Artifact not found for commit {commitSha} with config '{buildCacheConfig}'."); + } + + response.EnsureSuccessStatusCode(); + + using var fileStream = File.Create(tempArchive); + await response.Content.CopyToAsync(fileStream, cancellationToken); + + Log.Info($"Build Cache: Downloaded {new FileInfo(tempArchive).Length / (1024 * 1024)} MB"); + } + else + { + Log.Info($"Build Cache: Using cached archive at {tempArchive}"); + } + + var extractDir = Path.Combine(tempDir, $"extracted-{buildCacheConfig}"); + if (Directory.Exists(extractDir)) + { + Directory.Delete(extractDir, true); + } + + Log.Info($"Build Cache: Extracting archive..."); + + if (artifactFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase)) + { + await ExtractTarGzAsync(tempArchive, extractDir, cancellationToken); + } + else if (artifactFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + ZipFile.ExtractToDirectory(tempArchive, extractDir); + } + else + { + throw new InvalidOperationException($"Unsupported archive format: {artifactFile}"); + } + + return extractDir; + } + + /// + /// Overlays BCS runtime binaries (managed + native) into a published output directory, + /// replacing NuGet-sourced runtime DLLs with BCS-built ones. + /// Returns the number of files overlaid. + /// + public static int OverlayPublishedOutput(string extractDir, string outputFolder) + { + var platformMoniker = GetPlatformMoniker(); + int filesCopied = 0; + + // Find the NuGet package directory for managed + native DLLs + var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{platformMoniker}"); + + if (nugetPackageDir != null) + { + var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", platformMoniker); + + if (Directory.Exists(runtimesDir)) + { + // Copy managed DLLs from lib/net{X}.0/ + var libDir = Path.Combine(runtimesDir, "lib"); + if (Directory.Exists(libDir)) + { + var managedDir = Directory.GetDirectories(libDir).FirstOrDefault(); + if (managedDir != null) + { + foreach (var file in Directory.GetFiles(managedDir, "*.dll")) + { + var destFile = Path.Combine(outputFolder, Path.GetFileName(file)); + if (File.Exists(destFile)) + { + File.Copy(file, destFile, overwrite: true); + filesCopied++; + } + } + } + } + + // Copy native libraries from native/ + var nativeDir = Path.Combine(runtimesDir, "native"); + if (Directory.Exists(nativeDir)) + { + foreach (var file in Directory.GetFiles(nativeDir)) + { + var fileName = Path.GetFileName(file); + if (fileName.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".dbg", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var destFile = Path.Combine(outputFolder, fileName); + if (File.Exists(destFile)) + { + File.Copy(file, destFile, overwrite: true); + filesCopied++; + } + } + } + } + } + + // Also overlay host binaries from {rid}.Release/corehost/ + var corehostDir = FindCorehostDirectory(extractDir, platformMoniker); + if (corehostDir != null) + { + var hostPolicyName = GetNativeLibName("hostpolicy"); + var hostPolicySrc = Path.Combine(corehostDir, hostPolicyName); + var hostPolicyDest = Path.Combine(outputFolder, hostPolicyName); + if (File.Exists(hostPolicySrc) && File.Exists(hostPolicyDest)) + { + File.Copy(hostPolicySrc, hostPolicyDest, overwrite: true); + filesCopied++; + } + } + + return filesCopied; + } + + /// + /// Downloads and extracts BCS runtime artifacts into a standard dotnet installation directory. + /// Overlays runtime binaries on top of an existing dotnet-install layout. + /// Returns the actual runtime version string read from the extracted artifacts. + /// + public static async Task InstallRuntimeFromBuildCacheAsync( + string baseUrl, + string repoName, + string commitSha, + string buildCacheConfig, + string dotnetHome, + string targetFramework, + CancellationToken cancellationToken = default) + { + if (_installedBuildCacheRuntimes.ContainsKey(commitSha)) + { + Log.Info($"Build Cache: Runtime for commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))} already installed, skipping."); + + // Read the version from the already-installed runtime + return ReadInstalledBuildCacheVersion(dotnetHome, targetFramework) ?? $"buildcache-{commitSha.Substring(0, 12)}"; + } + + var platformMoniker = GetPlatformMoniker(); + + if (string.IsNullOrEmpty(buildCacheConfig)) + { + if (_platformToBcsConfig.TryGetValue(platformMoniker, out var mapped)) + { + buildCacheConfig = mapped.configKey; + } + else + { + throw new InvalidOperationException($"No Build Cache configuration mapping for platform '{platformMoniker}'."); + } + } + + // Determine artifact filename + string artifactFile; + if (_platformToBcsConfig.Values.Any(v => v.configKey == buildCacheConfig)) + { + artifactFile = _platformToBcsConfig.Values.First(v => v.configKey == buildCacheConfig).artifactFile; + } + else + { + throw new InvalidOperationException($"Unknown Build Cache configuration key: '{buildCacheConfig}'."); + } + + // Construct the download URL + var artifactUrl = $"{baseUrl}/builds/{repoName}/buildArtifacts/{commitSha}/{buildCacheConfig}/{artifactFile}"; + + Log.Info($"Build Cache: Downloading {artifactFile} from {artifactUrl}"); + + // Download to a temp file + var tempDir = Path.Combine(Path.GetTempPath(), "crank-buildcache", commitSha); + Directory.CreateDirectory(tempDir); + var tempArchive = Path.Combine(tempDir, artifactFile); + + try + { + if (!File.Exists(tempArchive)) + { + using var response = await _httpClient.GetAsync(artifactUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + throw new InvalidOperationException($"Build Cache: Artifact not found for commit {commitSha} with config '{buildCacheConfig}'. The build may not exist in the cache."); + } + + response.EnsureSuccessStatusCode(); + + using var fileStream = File.Create(tempArchive); + await response.Content.CopyToAsync(fileStream, cancellationToken); + + Log.Info($"Build Cache: Downloaded {new FileInfo(tempArchive).Length / (1024 * 1024)} MB"); + } + else + { + Log.Info($"Build Cache: Using cached archive at {tempArchive}"); + } + + // Extract and overlay + var extractDir = Path.Combine(tempDir, "extracted"); + if (Directory.Exists(extractDir)) + { + Directory.Delete(extractDir, true); + } + + Log.Info($"Build Cache: Extracting archive..."); + + if (artifactFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase)) + { + await ExtractTarGzAsync(tempArchive, extractDir, cancellationToken); + } + else if (artifactFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + ZipFile.ExtractToDirectory(tempArchive, extractDir); + } + else + { + throw new InvalidOperationException($"Unsupported archive format: {artifactFile}"); + } + + // Overlay into dotnet home + var runtimeVersion = await OverlayRuntimeAsync(extractDir, dotnetHome, platformMoniker, targetFramework, commitSha, cancellationToken); + + _installedBuildCacheRuntimes.TryAdd(commitSha, 0); + + Log.Info($"Build Cache: Runtime {runtimeVersion} (commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))}) installed successfully."); + + return runtimeVersion; + } + catch (Exception ex) when (ex is not InvalidOperationException) + { + throw new InvalidOperationException($"Build Cache: Failed to install runtime from commit {commitSha}: {ex.Message}", ex); + } + } + + /// + /// Overlays extracted BCS artifacts into the dotnet home directory structure. + /// + private static async Task OverlayRuntimeAsync( + string extractDir, + string dotnetHome, + string platformMoniker, + string targetFramework, + string commitSha, + CancellationToken cancellationToken) + { + var versionPrefix = ExtractVersionPrefix(targetFramework); + var rid = platformMoniker; + + // The NuGet package layout inside the archive is at: + // microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/ + // lib/net{X}.0/ → managed DLLs + // native/ → native libraries + var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); + string managedDir = null; + string nativeDir = null; + + if (nugetPackageDir != null) + { + var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); + + if (Directory.Exists(runtimesDir)) + { + // Find the lib/net{X}.0 directory + var libDir = Path.Combine(runtimesDir, "lib"); + if (Directory.Exists(libDir)) + { + managedDir = Directory.GetDirectories(libDir) + .FirstOrDefault(d => Path.GetFileName(d).StartsWith($"net{versionPrefix}", StringComparison.OrdinalIgnoreCase)) + ?? Directory.GetDirectories(libDir).FirstOrDefault(); + } + + nativeDir = Path.Combine(runtimesDir, "native"); + if (!Directory.Exists(nativeDir)) + { + nativeDir = null; + } + } + } + + // Determine the runtime version from the managed DLLs directory or other metadata + var runtimeVersion = DetermineRuntimeVersion(extractDir, versionPrefix, commitSha); + + // Create the shared framework directory + var sharedFrameworkDir = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); + Directory.CreateDirectory(sharedFrameworkDir); + + int filesCopied = 0; + + // Copy managed DLLs + if (managedDir != null && Directory.Exists(managedDir)) + { + foreach (var file in Directory.GetFiles(managedDir, "*.dll")) + { + File.Copy(file, Path.Combine(sharedFrameworkDir, Path.GetFileName(file)), overwrite: true); + filesCopied++; + } + + Log.Info($"Build Cache: Copied {filesCopied} managed assemblies to shared framework."); + } + + // Copy native libraries + if (nativeDir != null && Directory.Exists(nativeDir)) + { + int nativeCount = 0; + + foreach (var file in Directory.GetFiles(nativeDir)) + { + var fileName = Path.GetFileName(file); + + // Skip debug symbols during overlay (keep it lean) + if (fileName.EndsWith(".dbg", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + File.Copy(file, Path.Combine(sharedFrameworkDir, fileName), overwrite: true); + nativeCount++; + } + + Log.Info($"Build Cache: Copied {nativeCount} native libraries to shared framework."); + filesCopied += nativeCount; + } + + // Also look for host binaries in the corehost directory + // Pattern: {rid}.Release/corehost/ or linux-arm64.Release/corehost/ + var corehostDir = FindCorehostDirectory(extractDir, platformMoniker); + + if (corehostDir != null) + { + // Copy libhostpolicy to shared framework + CopyFileIfExists(corehostDir, sharedFrameworkDir, GetNativeLibName("hostpolicy")); + + // Copy libhostfxr to host/fxr/{version}/ + var hostFxrDir = Path.Combine(dotnetHome, "host", "fxr", runtimeVersion); + Directory.CreateDirectory(hostFxrDir); + CopyFileIfExists(corehostDir, hostFxrDir, GetNativeLibName("hostfxr")); + + Log.Info($"Build Cache: Copied host binaries."); + } + + // Write a .version file with the commit SHA for traceability + var versionFilePath = Path.Combine(sharedFrameworkDir, ".version"); + await File.WriteAllTextAsync(versionFilePath, $"{commitSha}\n{runtimeVersion}\n", cancellationToken); + + if (filesCopied == 0) + { + throw new InvalidOperationException($"Build Cache: No runtime files found to extract. The archive may not contain the expected layout for platform '{platformMoniker}'."); + } + + return runtimeVersion; + } + + /// + /// Determines the runtime version from extracted artifacts. + /// + private static string DetermineRuntimeVersion(string extractDir, string versionPrefix, string commitSha) + { + // Look for .version file in the shared framework subdirectory of the archive + var versionFiles = Directory.GetFiles(extractDir, ".version", SearchOption.AllDirectories); + + foreach (var versionFile in versionFiles) + { + try + { + var lines = File.ReadAllLines(versionFile); + // The .version file typically has: line 0 = commit hash, line 1 = version string + if (lines.Length >= 2 && lines[1].StartsWith(versionPrefix, StringComparison.OrdinalIgnoreCase)) + { + return lines[1].Trim(); + } + } + catch + { + // Continue searching + } + } + + // Fallback: construct a version string from the prefix and commit + return $"{versionPrefix}.0-buildcache.{commitSha.Substring(0, Math.Min(8, commitSha.Length))}"; + } + + /// + /// Reads the runtime version from an already-installed BCS runtime. + /// + private static string ReadInstalledBuildCacheVersion(string dotnetHome, string targetFramework) + { + var versionPrefix = ExtractVersionPrefix(targetFramework); + var sharedDir = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App"); + + if (!Directory.Exists(sharedDir)) + { + return null; + } + + // Find directories matching the version prefix that have a .version file with a commit SHA + foreach (var dir in Directory.GetDirectories(sharedDir).OrderByDescending(d => d)) + { + var dirName = Path.GetFileName(dir); + if (dirName.StartsWith(versionPrefix, StringComparison.OrdinalIgnoreCase)) + { + var versionFile = Path.Combine(dir, ".version"); + if (File.Exists(versionFile)) + { + var lines = File.ReadAllLines(versionFile); + if (lines.Length >= 2) + { + return lines[1].Trim(); + } + } + } + } + + return null; + } + + private static async Task GetLatestBuildsAsync( + string baseUrl, string repoName, string branch, CancellationToken cancellationToken) + { + var cacheKey = $"{baseUrl}|{repoName}/{branch}"; + + if (_latestBuildsCache.TryGetValue(cacheKey, out var cached) && + DateTimeOffset.UtcNow - cached.fetchedAt < _latestBuildsCacheDuration) + { + return cached.data; + } + + var url = $"{baseUrl}/builds/{repoName}/latest/{branch}/latestBuilds.json"; + Log.Info($"Build Cache: Fetching latest builds from {url}"); + + using var response = await _httpClient.GetAsync(url, cancellationToken); + + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + throw new InvalidOperationException($"Build Cache: No latest builds found for branch '{branch}' in repo '{repoName}'. URL: {url}"); + } + + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var latestBuilds = ParseLatestBuilds(json); + + _latestBuildsCache[cacheKey] = (DateTimeOffset.UtcNow, latestBuilds); + + return latestBuilds; + } + + /// + /// Parses the latestBuilds.json format from BCS. The JSON has dynamic keys for each + /// build configuration, with "branch_name" as a special key. + /// + private static LatestBuildsResponse ParseLatestBuilds(string json) + { + var result = new LatestBuildsResponse(); + + using var doc = JsonDocument.Parse(json); + + foreach (var property in doc.RootElement.EnumerateObject()) + { + if (property.Name == "branch_name") + { + result.BranchName = property.Value.GetString(); + continue; + } + + if (property.Value.ValueKind == JsonValueKind.Object) + { + var entry = new LatestBuildEntry + { + CommitSha = property.Value.TryGetProperty("CommitSha", out var sha) ? sha.GetString() + : property.Value.TryGetProperty("commit_sha", out sha) ? sha.GetString() + : null, + CommitTime = property.Value.TryGetProperty("CommitTime", out var time) ? time.GetString() + : property.Value.TryGetProperty("commit_time", out time) ? time.GetString() + : null, + }; + + result.Entries[property.Name] = entry; + } + } + + return result; + } + + private static string FindDirectory(string root, string directoryName) + { + if (!Directory.Exists(root)) + { + return null; + } + + // Check direct children first + foreach (var dir in Directory.GetDirectories(root)) + { + if (Path.GetFileName(dir).Equals(directoryName, StringComparison.OrdinalIgnoreCase)) + { + return dir; + } + } + + return null; + } + + private static string FindCorehostDirectory(string extractDir, string platformMoniker) + { + // BCS layout: {rid}.Release/corehost/ (e.g., linux-arm64.Release/corehost/) + // Map RID to the directory name format used in BCS artifacts + var ridDirName = $"{platformMoniker}.Release"; + var corehostPath = Path.Combine(extractDir, ridDirName, "corehost"); + + if (Directory.Exists(corehostPath)) + { + return corehostPath; + } + + // Also try the raw format without dots + var altCorehostPath = Path.Combine(extractDir, "corehost"); + if (Directory.Exists(altCorehostPath)) + { + return altCorehostPath; + } + + return null; + } + + private static void CopyFileIfExists(string sourceDir, string destDir, string fileName) + { + var sourcePath = Path.Combine(sourceDir, fileName); + if (File.Exists(sourcePath)) + { + File.Copy(sourcePath, Path.Combine(destDir, fileName), overwrite: true); + } + } + + private static string GetNativeLibName(string baseName) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return $"{baseName}.dll"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return $"lib{baseName}.dylib"; + } + else + { + return $"lib{baseName}.so"; + } + } + + private static string GetPlatformMoniker() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "win-arm64" + : RuntimeInformation.ProcessArchitecture == Architecture.X86 ? "win-x86" + : "win-x64"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "osx-arm64" : "osx-x64"; + } + else + { + return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "linux-arm64" : "linux-x64"; + } + } + + private static string ExtractVersionPrefix(string targetFramework) + { + if (string.IsNullOrWhiteSpace(targetFramework)) + { + throw new InvalidOperationException("Target framework must be specified."); + } + + // "net10.0" → "10.0", "net9.0" → "9.0" + if (targetFramework.StartsWith("net", StringComparison.OrdinalIgnoreCase) && + targetFramework.Length > 3 && + char.IsDigit(targetFramework[3])) + { + return targetFramework.Substring(3); + } + + // "netcoreapp3.1" → "3.1" + if (targetFramework.StartsWith("netcoreapp", StringComparison.OrdinalIgnoreCase) && + targetFramework.Length > "netcoreapp".Length) + { + return targetFramework.Substring("netcoreapp".Length); + } + + throw new InvalidOperationException( + $"Unsupported target framework '{targetFramework}' for Build Cache runtime version inference."); + } + + private static async Task ExtractTarGzAsync(string archivePath, string outputDir, CancellationToken cancellationToken) + { + Directory.CreateDirectory(outputDir); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // On Windows, use tar (available since Windows 10 1803) + var result = await ProcessUtil.RunAsync("tar", $"-xzf \"{archivePath}\" -C \"{outputDir}\"", + throwOnError: false, cancellationToken: cancellationToken); + + if (result.ExitCode != 0) + { + throw new InvalidOperationException($"Failed to extract tar.gz: {result.StandardError}"); + } + } + else + { + var result = await ProcessUtil.RunAsync("/usr/bin/env", $"tar -xzf \"{archivePath}\" -C \"{outputDir}\"", + throwOnError: false, cancellationToken: cancellationToken); + + if (result.ExitCode != 0) + { + throw new InvalidOperationException($"Failed to extract tar.gz: {result.StandardError}"); + } + } + } + + internal class LatestBuildsResponse + { + public string BranchName { get; set; } + public Dictionary Entries { get; set; } = new(StringComparer.OrdinalIgnoreCase); + } + + internal class LatestBuildEntry + { + public string CommitSha { get; set; } + public string CommitTime { get; set; } + } + } +} diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index 8168a2e88..4300949f1 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -109,6 +109,11 @@ public class Startup "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2" ]; + // Build Cache Service configuration + private static string _buildCacheBaseUrl = "https://pvscmdupload.z22.web.core.windows.net"; + private static string _buildCacheRepoName = "runtime"; + private static bool _buildCacheEnabled = true; + // Cached lists of SDKs and runtimes already installed private static readonly HashSet _installedAspNetRuntimes = new(StringComparer.OrdinalIgnoreCase); private static readonly HashSet _installedDotnetRuntimes = new(StringComparer.OrdinalIgnoreCase); @@ -270,6 +275,10 @@ public static int Main(string[] args) _certSniAuth = app.Option("--cert-sni", "Enable subject name / issuer based authentication (SNI).", CommandOptionType.NoValue); _managedIdentityClientId = app.Option("--mi-client-id", "Client ID of the user-assigned managed identity to use for authentication.", CommandOptionType.SingleValue); + var buildCacheBaseUrlOption = app.Option("--build-cache-base-url", $"Base URL for Build Cache Service blob storage. Default is '{_buildCacheBaseUrl}'.", CommandOptionType.SingleValue); + var buildCacheRepoNameOption = app.Option("--build-cache-repo-name", $"Repository name for Build Cache Service. Default is '{_buildCacheRepoName}'.", CommandOptionType.SingleValue); + var buildCacheDisabledOption = app.Option("--build-cache-disabled", "Disable Build Cache Service integration.", CommandOptionType.NoValue); + app.OnExecute(() => { var logConf = new LoggerConfiguration() @@ -278,6 +287,21 @@ public static int Main(string[] args) .Enrich.FromLogContext() .WriteTo.Console(theme: AnsiConsoleTheme.Code); + if (buildCacheBaseUrlOption.HasValue()) + { + _buildCacheBaseUrl = buildCacheBaseUrlOption.Value(); + } + + if (buildCacheRepoNameOption.HasValue()) + { + _buildCacheRepoName = buildCacheRepoNameOption.Value(); + } + + if (buildCacheDisabledOption.HasValue()) + { + _buildCacheEnabled = false; + } + if (_runAsService.HasValue() && OperatingSystem != OperatingSystem.Windows) { throw new PlatformNotSupportedException($"--service is only available on Windows"); @@ -2933,23 +2957,71 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str runtimeVersion = channel; } + // For buildcache channel, SDK/ASP.NET/Desktop use "latest" since BCS only has runtime + var nonRuntimeChannel = String.Equals(channel, "buildcache", StringComparison.OrdinalIgnoreCase) ? "latest" : channel; + if (String.IsNullOrEmpty(desktopVersion)) { - desktopVersion = channel; + desktopVersion = nonRuntimeChannel; } if (String.IsNullOrEmpty(aspNetCoreVersion)) { - aspNetCoreVersion = channel; + aspNetCoreVersion = nonRuntimeChannel; } if (String.IsNullOrEmpty(sdkVersion)) { - sdkVersion = channel; + sdkVersion = nonRuntimeChannel; } runtimeVersion = await ResolveRuntimeVersion(buildToolsPath, targetFramework, runtimeVersion); + // Build Cache Service: if the runtime version is "BuildCache", prepare BCS artifacts + // and resolve a real "Latest" runtime version for the NuGet build. + var useBuildCache = String.Equals(runtimeVersion, "BuildCache", StringComparison.OrdinalIgnoreCase); + string buildCacheCommitSha = null; + string buildCacheExtractDir = null; + + if (useBuildCache) + { + if (!_buildCacheEnabled) + { + job.Error = "Build Cache channel was requested but Build Cache Service is disabled on this agent (--build-cache-disabled)."; + return null; + } + + try + { + var branch = !string.IsNullOrEmpty(job.BuildCacheBranch) ? job.BuildCacheBranch : "main"; + var commitSha = job.BuildCacheCommitSha; + var buildCacheConfig = job.BuildCacheConfig; + + // Resolve which commit to use + var resolved = await BuildCacheClient.ResolveCommitAsync( + _buildCacheBaseUrl, _buildCacheRepoName, branch, commitSha, buildCacheConfig); + + buildCacheCommitSha = resolved.commitSha; + + // Download and extract the BCS artifacts to a temp directory + buildCacheExtractDir = await BuildCacheClient.DownloadAndExtractAsync( + _buildCacheBaseUrl, _buildCacheRepoName, buildCacheCommitSha, buildCacheConfig, + targetFramework, cancellationToken); + + var shortSha = buildCacheCommitSha.Substring(0, Math.Min(8, buildCacheCommitSha.Length)); + Log.Info($"Build Cache: Artifacts for commit {shortSha} ready for post-build overlay"); + + // Resolve a REAL runtime version from feeds for the NuGet build + runtimeVersion = await ResolveRuntimeVersion(buildToolsPath, targetFramework, "Latest"); + Log.Info($"Runtime for build: {runtimeVersion} (Latest from feeds, will be overlaid with BCS commit {shortSha})"); + } + catch (Exception ex) + { + job.Error = $"Build Cache: {ex.Message}"; + return null; + } + } + sdkVersion = await ResolveSdkVersion(sdkVersion, targetFramework); aspNetCoreVersion = await ResolveAspNetCoreVersion(aspNetCoreVersion, targetFramework); @@ -3434,6 +3506,25 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => Log.Info($"Application published successfully in {job.BuildTime.TotalMilliseconds} ms"); + // Build Cache: overlay BCS runtime binaries into the published output + // This replaces the NuGet-sourced runtime DLLs with BCS-built ones from the specific commit + if (useBuildCache && buildCacheExtractDir != null) + { + try + { + var shortSha = buildCacheCommitSha.Substring(0, Math.Min(8, buildCacheCommitSha.Length)); + var overlayCount = BuildCacheClient.OverlayPublishedOutput(buildCacheExtractDir, outputFolder); + Log.Info($"Build Cache: Overlaid {overlayCount} runtime files from commit {shortSha} into published output"); + + // Update the reported runtime version to reflect the BCS commit (for metadata/reporting only) + job.RuntimeVersion = $"{runtimeVersion}+buildcache.{shortSha}"; + } + catch (Exception ex) + { + Log.Info($"Build Cache: Warning - overlay failed: {ex.Message}. Published app will use feed-sourced runtime."); + } + } + PatchRuntimeConfig(job, outputFolder, aspNetCoreVersion, runtimeVersion); } @@ -4582,6 +4673,13 @@ private static async Task ResolveRuntimeVersion(string buildToolsPath, s break; } } + else if (String.Equals(runtimeVersion, "BuildCache", StringComparison.OrdinalIgnoreCase)) + { + // BuildCache channel: version resolution is deferred to InstallRuntimeFromBuildCacheAsync + // because it needs to download artifacts. We return a placeholder here. + runtimeVersion = "BuildCache"; + Log.Info($"Runtime: will be resolved from Build Cache Service"); + } else { // Custom version diff --git a/src/Microsoft.Crank.Models/Job.cs b/src/Microsoft.Crank.Models/Job.cs index 1128cd0f0..299387380 100644 --- a/src/Microsoft.Crank.Models/Job.cs +++ b/src/Microsoft.Crank.Models/Job.cs @@ -71,6 +71,11 @@ public class Job public string UseMonoRuntime { get; set; } = ""; public bool NoGlobalJson { get; set; } + // Build Cache Service properties for per-commit runtime resolution + public string BuildCacheCommitSha { get; set; } = ""; + public string BuildCacheBranch { get; set; } = ""; + public string BuildCacheConfig { get; set; } = ""; + // Delay from the process started to the console receiving "Application started" public TimeSpan StartupMainMethod { get; set; } public TimeSpan BuildTime { get; set; } From 1ecc2393f084611c1d179558ad5f5fb1cca695ad Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 1 May 2026 11:58:11 -0700 Subject: [PATCH 2/7] Address PR review: dead code, FDD overlay, hardening, and tests Changes: * BuildCacheClient: rewrite to drop ~250 LOC of dead code from an earlier abandoned design (synthesizing a new shared-framework dir with a synthetic version). The FDD overlay path is restored as a new public API. * New OverlayDotnetHome(extractDir, dotnetHome, runtimeVersion): overlays BCS bits into dotnetHome/shared/Microsoft.NETCore.App/{runtimeVersion}/ and host/fxr/{runtimeVersion}/ + the dotnet host. Wired into Startup.cs after publish so framework-dependent jobs actually run against BCS bits instead of silently using the feed runtime. * OverlayPublishedOutput: copy ALL managed/native runtime files unconditionally (previously skipped any file not already in destination, which would silently drop new DLLs introduced by the BCS commit). Also copies hostfxr/hostpolicy/dotnet for self-contained. * Hardening: - URL-encode repoName/commitSha/buildCacheConfig/branch with Uri.EscapeDataString. - Atomic download (.partial -> rename) and Content-Length validation so truncated archives are not reused after a failed run. - Retry transient HTTP failures via ProcessUtil.RetryOnExceptionAsync. - Replace shelling out to tar with System.Formats.Tar.TarFile. - Wrap synchronous archive extraction in Task.Run. - Per-(commit,config) SemaphoreSlim + per-call unique extract dir to avoid races between concurrent jobs. - Drop unused targetFramework parameter from DownloadAndExtractAsync. - All commit-SHA Substring uses go through ShortSha (Math.Min length guard). - ParseLatestBuilds: case-insensitive branch_name/BranchName handling and skip non-object metadata properties safely. * Startup.cs: - Validate user-supplied BuildCacheCommitSha length (>= 8 chars) up front instead of crashing later. - Stop mutating runtimeVersion before PatchRuntimeConfig - use feed-resolved version so runtimeconfig.json points to a really-installed shared framework dir. Suffix +buildcache. is now applied to job.RuntimeVersion only, after PatchRuntimeConfig has run. - Treat 0-file overlay as fatal (job.Error + return null) so silent failures do not produce wrong-runtime benchmarks. - Promote overlay-failure log from Info to fatal job error. * docs/dotnet_versions.md: add trailing newline. * test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs: 17 new tests covering ParseLatestBuilds (PascalCase / snake_case / mixed / missing fields / case-insensitive lookup / non-object values), GetPlatformMoniker, ShortSha, GetNativeLibName, OverlayPublishedOutput unconditional copy, pdb/dbg skip, OverlayDotnetHome shared-framework precondition, and full shared-framework + hostfxr overlay. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/dotnet_versions.md | 2 +- src/Microsoft.Crank.Agent/BuildCacheClient.cs | 845 +++++++----------- src/Microsoft.Crank.Agent/Startup.cs | 63 +- .../BuildCacheClientTests.cs | 406 +++++++++ 4 files changed, 794 insertions(+), 522 deletions(-) create mode 100644 test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs diff --git a/docs/dotnet_versions.md b/docs/dotnet_versions.md index 1e919106c..70e21181c 100644 --- a/docs/dotnet_versions.md +++ b/docs/dotnet_versions.md @@ -166,4 +166,4 @@ The agent supports these command-line options for BCS: |--------|---------|-------------| | `--build-cache-base-url` | `https://pvscmdupload.z22.web.core.windows.net` | Base URL for BCS blob storage. | | `--build-cache-repo-name` | `runtime` | Repository name in BCS. | -| `--build-cache-disabled` | (not set) | Disables BCS integration on this agent. | \ No newline at end of file +| `--build-cache-disabled` | (not set) | Disables BCS integration on this agent. | diff --git a/src/Microsoft.Crank.Agent/BuildCacheClient.cs b/src/Microsoft.Crank.Agent/BuildCacheClient.cs index b6c53fac0..46ba0fc37 100644 --- a/src/Microsoft.Crank.Agent/BuildCacheClient.cs +++ b/src/Microsoft.Crank.Agent/BuildCacheClient.cs @@ -5,13 +5,13 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Formats.Tar; using System.IO; using System.IO.Compression; using System.Linq; using System.Net.Http; using System.Runtime.InteropServices; using System.Text.Json; -using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; @@ -20,38 +20,43 @@ namespace Microsoft.Crank.Agent /// /// Lightweight client for the Build Caching Service (BCS) in dotnet-performance-infra. /// Downloads pre-built runtime artifacts from public Azure Blob Storage and overlays - /// them into a standard dotnet installation directory. + /// them onto the agent's installed shared framework and/or the published app output so + /// benchmarks run against the BCS runtime instead of the feed-installed one. /// internal static class BuildCacheClient { - private static readonly HttpClient _httpClient = new HttpClient { Timeout = TimeSpan.FromMinutes(10) }; + private const int DownloadRetryCount = 3; + private static readonly TimeSpan _httpTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan _latestBuildsCacheDuration = TimeSpan.FromHours(1); + + private static readonly HttpClient _httpClient = new HttpClient { Timeout = _httpTimeout }; - // Cache latestBuilds.json responses to avoid repeated downloads (keyed by baseUrl+branch) + // Cache latestBuilds.json responses to avoid repeated downloads (keyed by baseUrl|repo|branch). private static readonly ConcurrentDictionary _latestBuildsCache = new(); - private static readonly TimeSpan _latestBuildsCacheDuration = TimeSpan.FromHours(1); - // Cache of already-installed BCS commit SHAs to avoid re-extracting - private static readonly ConcurrentDictionary _installedBuildCacheRuntimes = new(StringComparer.OrdinalIgnoreCase); + // Per-(commit,config) async locks so concurrent jobs serialize their downloads/extracts. + private static readonly ConcurrentDictionary _extractLocks = new(); /// /// Maps the agent's platform (RID) to the BCS configuration key and artifact filename. /// - private static readonly Dictionary _platformToBcsConfig = new(StringComparer.OrdinalIgnoreCase) - { - ["linux-x64"] = ("coreclr_x64_linux", "BuildArtifacts_linux_x64_Release_coreclr.tar.gz"), - ["linux-arm64"] = ("coreclr_arm64_linux", "BuildArtifacts_linux_arm64_Release_coreclr.tar.gz"), - ["linux-musl-x64"] = ("coreclr_muslx64_linux", "BuildArtifacts_linux_musl_x64_Release_coreclr.tar.gz"), - ["win-x64"] = ("coreclr_x64_windows", "BuildArtifacts_windows_x64_Release_coreclr.zip"), - ["win-arm64"] = ("coreclr_arm64_windows", "BuildArtifacts_windows_arm64_Release_coreclr.zip"), - ["win-x86"] = ("coreclr_x86_windows", "BuildArtifacts_windows_x86_Release_coreclr.zip"), - }; + internal static readonly IReadOnlyDictionary PlatformToBcsConfig = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["linux-x64"] = ("coreclr_x64_linux", "BuildArtifacts_linux_x64_Release_coreclr.tar.gz"), + ["linux-arm64"] = ("coreclr_arm64_linux", "BuildArtifacts_linux_arm64_Release_coreclr.tar.gz"), + ["linux-musl-x64"] = ("coreclr_muslx64_linux", "BuildArtifacts_linux_musl_x64_Release_coreclr.tar.gz"), + ["win-x64"] = ("coreclr_x64_windows", "BuildArtifacts_windows_x64_Release_coreclr.zip"), + ["win-arm64"] = ("coreclr_arm64_windows", "BuildArtifacts_windows_arm64_Release_coreclr.zip"), + ["win-x86"] = ("coreclr_x86_windows", "BuildArtifacts_windows_x86_Release_coreclr.zip"), + }; /// - /// Resolves the commit SHA to use from BCS. If a specific commit is provided, validates - /// it exists. Otherwise queries latestBuilds.json for the latest commit on the branch. - /// Returns the commit SHA and the runtime version string. + /// Resolves the commit SHA to use from BCS. If a specific commit is provided, returns it + /// (after platform-config inference). Otherwise queries latestBuilds.json for the latest + /// commit on the branch. /// - public static async Task<(string commitSha, string runtimeVersion)> ResolveCommitAsync( + public static async Task<(string commitSha, string buildCacheConfig)> ResolveCommitAsync( string baseUrl, string repoName, string branch, @@ -59,587 +64,428 @@ internal static class BuildCacheClient string buildCacheConfig, CancellationToken cancellationToken = default) { - var platformMoniker = GetPlatformMoniker(); - - if (!string.IsNullOrEmpty(buildCacheConfig)) - { - // Use explicit config key - } - else if (_platformToBcsConfig.TryGetValue(platformMoniker, out var mapped)) - { - buildCacheConfig = mapped.configKey; - } - else - { - throw new InvalidOperationException($"No Build Cache configuration mapping for platform '{platformMoniker}'. Specify buildCacheConfig explicitly."); - } + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); if (string.IsNullOrEmpty(commitSha)) { - // Query latestBuilds.json for the latest commit var latestBuilds = await GetLatestBuildsAsync(baseUrl, repoName, branch, cancellationToken); - // Try to get the config-specific entry, fall back to "all" if (latestBuilds.Entries.TryGetValue(buildCacheConfig, out var configEntry) && !string.IsNullOrEmpty(configEntry.CommitSha)) { commitSha = configEntry.CommitSha; - Log.Info($"Build Cache: Using latest commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))} for config '{buildCacheConfig}' on branch '{branch}' (committed {configEntry.CommitTime})"); + Log.Info($"Build Cache: Using latest commit {ShortSha(commitSha)} for config '{buildCacheConfig}' on branch '{branch}' (committed {configEntry.CommitTime})"); } else if (latestBuilds.Entries.TryGetValue("all", out var allEntry) && !string.IsNullOrEmpty(allEntry.CommitSha)) { commitSha = allEntry.CommitSha; - Log.Info($"Build Cache: Using latest commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))} for all configs on branch '{branch}' (committed {allEntry.CommitTime})"); + Log.Info($"Build Cache: Using latest commit {ShortSha(commitSha)} for all configs on branch '{branch}' (committed {allEntry.CommitTime})"); } else { - throw new InvalidOperationException($"Build Cache: No latest build found for branch '{branch}'. Check that BCS has builds for this branch."); + throw new InvalidOperationException( + $"Build Cache: No latest build found for branch '{branch}' (config '{buildCacheConfig}'). Check that BCS has builds for this branch."); } } else { - Log.Info($"Build Cache: Using specified commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))}"); + Log.Info($"Build Cache: Using specified commit {ShortSha(commitSha)}"); } - // Try to determine a runtime version from the commit. For now, we return a placeholder - // that will be replaced after extraction by reading .version from the shared framework. - var runtimeVersion = $"buildcache-{commitSha.Substring(0, Math.Min(12, commitSha.Length))}"; - - return (commitSha, runtimeVersion); + return (commitSha, buildCacheConfig); } /// - /// Downloads and extracts BCS runtime artifacts to a temp directory without overlaying. - /// Returns the path to the extracted directory for later overlay into published output. + /// Downloads and extracts BCS runtime artifacts to a per-job temp directory. The caller + /// is responsible for invoking the overlay methods on the returned directory. /// public static async Task DownloadAndExtractAsync( string baseUrl, string repoName, string commitSha, string buildCacheConfig, - string targetFramework, CancellationToken cancellationToken = default) { - var platformMoniker = GetPlatformMoniker(); - - if (string.IsNullOrEmpty(buildCacheConfig)) + if (string.IsNullOrEmpty(commitSha)) { - if (_platformToBcsConfig.TryGetValue(platformMoniker, out var mapped)) - { - buildCacheConfig = mapped.configKey; - } - else - { - throw new InvalidOperationException($"No Build Cache configuration mapping for platform '{platformMoniker}'."); - } + throw new ArgumentException("commitSha must be provided.", nameof(commitSha)); } - string artifactFile; - if (_platformToBcsConfig.Values.Any(v => v.configKey == buildCacheConfig)) - { - artifactFile = _platformToBcsConfig.Values.First(v => v.configKey == buildCacheConfig).artifactFile; - } - else - { - throw new InvalidOperationException($"Unknown Build Cache configuration key: '{buildCacheConfig}'."); - } + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); + var artifactFile = GetArtifactFile(buildCacheConfig); + var normalizedBaseUrl = (baseUrl ?? string.Empty).TrimEnd('/'); - var artifactUrl = $"{baseUrl}/builds/{repoName}/buildArtifacts/{commitSha}/{buildCacheConfig}/{artifactFile}"; + var artifactUrl = + $"{normalizedBaseUrl}/builds/{Uri.EscapeDataString(repoName)}/buildArtifacts/" + + $"{Uri.EscapeDataString(commitSha)}/{Uri.EscapeDataString(buildCacheConfig)}/{Uri.EscapeDataString(artifactFile)}"; - Log.Info($"Build Cache: Downloading {artifactFile} from {artifactUrl}"); + var rootCacheDir = Path.Combine(Path.GetTempPath(), "crank-buildcache"); + Directory.CreateDirectory(rootCacheDir); - var tempDir = Path.Combine(Path.GetTempPath(), "crank-buildcache", commitSha); - Directory.CreateDirectory(tempDir); - var tempArchive = Path.Combine(tempDir, artifactFile); + var safeConfig = SanitizeForPath(buildCacheConfig); - if (!File.Exists(tempArchive)) + // Per-(commit,config) lock so two concurrent jobs don't race on the same directory. + var lockKey = $"{commitSha}|{safeConfig}"; + var gate = _extractLocks.GetOrAdd(lockKey, _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken); + try { - using var response = await _httpClient.GetAsync(artifactUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + var commitDir = Path.Combine(rootCacheDir, commitSha); + Directory.CreateDirectory(commitDir); - if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + var archivePath = Path.Combine(commitDir, $"{safeConfig}-{artifactFile}"); + + if (!File.Exists(archivePath)) { - throw new InvalidOperationException($"Build Cache: Artifact not found for commit {commitSha} with config '{buildCacheConfig}'."); + Log.Info($"Build Cache: Downloading {artifactFile} from {artifactUrl}"); + await DownloadWithRetryAsync(artifactUrl, archivePath, cancellationToken); + Log.Info($"Build Cache: Downloaded {new FileInfo(archivePath).Length / (1024 * 1024)} MB"); + } + else + { + Log.Info($"Build Cache: Using cached archive at {archivePath}"); } - response.EnsureSuccessStatusCode(); - - using var fileStream = File.Create(tempArchive); - await response.Content.CopyToAsync(fileStream, cancellationToken); - - Log.Info($"Build Cache: Downloaded {new FileInfo(tempArchive).Length / (1024 * 1024)} MB"); - } - else - { - Log.Info($"Build Cache: Using cached archive at {tempArchive}"); - } - - var extractDir = Path.Combine(tempDir, $"extracted-{buildCacheConfig}"); - if (Directory.Exists(extractDir)) - { - Directory.Delete(extractDir, true); - } + var extractDir = Path.Combine(commitDir, $"extracted-{safeConfig}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(extractDir); - Log.Info($"Build Cache: Extracting archive..."); + Log.Info($"Build Cache: Extracting archive to {extractDir} ..."); + await ExtractArchiveAsync(archivePath, extractDir, cancellationToken); - if (artifactFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase)) - { - await ExtractTarGzAsync(tempArchive, extractDir, cancellationToken); + return extractDir; } - else if (artifactFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) - { - ZipFile.ExtractToDirectory(tempArchive, extractDir); - } - else + finally { - throw new InvalidOperationException($"Unsupported archive format: {artifactFile}"); + gate.Release(); } - - return extractDir; } /// - /// Overlays BCS runtime binaries (managed + native) into a published output directory, - /// replacing NuGet-sourced runtime DLLs with BCS-built ones. - /// Returns the number of files overlaid. + /// Overlays BCS runtime binaries (managed + native + host binaries) into a published + /// output directory. Used for self-contained publishes where the runtime is bundled in + /// the publish output. /// + /// Number of files overlaid. public static int OverlayPublishedOutput(string extractDir, string outputFolder) { - var platformMoniker = GetPlatformMoniker(); + var rid = GetPlatformMoniker(); int filesCopied = 0; - // Find the NuGet package directory for managed + native DLLs - var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{platformMoniker}"); - + var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); if (nugetPackageDir != null) { - var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", platformMoniker); - + var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); if (Directory.Exists(runtimesDir)) { - // Copy managed DLLs from lib/net{X}.0/ - var libDir = Path.Combine(runtimesDir, "lib"); - if (Directory.Exists(libDir)) - { - var managedDir = Directory.GetDirectories(libDir).FirstOrDefault(); - if (managedDir != null) - { - foreach (var file in Directory.GetFiles(managedDir, "*.dll")) - { - var destFile = Path.Combine(outputFolder, Path.GetFileName(file)); - if (File.Exists(destFile)) - { - File.Copy(file, destFile, overwrite: true); - filesCopied++; - } - } - } - } - - // Copy native libraries from native/ - var nativeDir = Path.Combine(runtimesDir, "native"); - if (Directory.Exists(nativeDir)) - { - foreach (var file in Directory.GetFiles(nativeDir)) - { - var fileName = Path.GetFileName(file); - if (fileName.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase) || - fileName.EndsWith(".dbg", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var destFile = Path.Combine(outputFolder, fileName); - if (File.Exists(destFile)) - { - File.Copy(file, destFile, overwrite: true); - filesCopied++; - } - } - } + filesCopied += CopyManaged(runtimesDir, outputFolder); + filesCopied += CopyNative(runtimesDir, outputFolder); } } - // Also overlay host binaries from {rid}.Release/corehost/ - var corehostDir = FindCorehostDirectory(extractDir, platformMoniker); + var corehostDir = FindCorehostDirectory(extractDir, rid); if (corehostDir != null) { - var hostPolicyName = GetNativeLibName("hostpolicy"); - var hostPolicySrc = Path.Combine(corehostDir, hostPolicyName); - var hostPolicyDest = Path.Combine(outputFolder, hostPolicyName); - if (File.Exists(hostPolicySrc) && File.Exists(hostPolicyDest)) - { - File.Copy(hostPolicySrc, hostPolicyDest, overwrite: true); - filesCopied++; - } + // For self-contained, all three host binaries live alongside the app. + filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetNativeLibName("hostpolicy")); + filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetNativeLibName("hostfxr")); + filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetDotnetExecutableName()); } return filesCopied; } /// - /// Downloads and extracts BCS runtime artifacts into a standard dotnet installation directory. - /// Overlays runtime binaries on top of an existing dotnet-install layout. - /// Returns the actual runtime version string read from the extracted artifacts. + /// Overlays BCS runtime binaries into the agent's installed dotnet home so framework- + /// dependent apps that load the runtime from shared/Microsoft.NETCore.App/{version}/ + /// get the BCS bits at runtime. /// - public static async Task InstallRuntimeFromBuildCacheAsync( - string baseUrl, - string repoName, - string commitSha, - string buildCacheConfig, - string dotnetHome, - string targetFramework, - CancellationToken cancellationToken = default) + /// Number of files overlaid. + public static int OverlayDotnetHome(string extractDir, string dotnetHome, string runtimeVersion) { - if (_installedBuildCacheRuntimes.ContainsKey(commitSha)) + if (string.IsNullOrEmpty(runtimeVersion)) { - Log.Info($"Build Cache: Runtime for commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))} already installed, skipping."); - - // Read the version from the already-installed runtime - return ReadInstalledBuildCacheVersion(dotnetHome, targetFramework) ?? $"buildcache-{commitSha.Substring(0, 12)}"; + throw new ArgumentException("runtimeVersion must be provided.", nameof(runtimeVersion)); } - var platformMoniker = GetPlatformMoniker(); + var rid = GetPlatformMoniker(); + int filesCopied = 0; - if (string.IsNullOrEmpty(buildCacheConfig)) + var sharedFrameworkDir = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); + if (!Directory.Exists(sharedFrameworkDir)) { - if (_platformToBcsConfig.TryGetValue(platformMoniker, out var mapped)) - { - buildCacheConfig = mapped.configKey; - } - else - { - throw new InvalidOperationException($"No Build Cache configuration mapping for platform '{platformMoniker}'."); - } + throw new InvalidOperationException( + $"Build Cache: Expected shared framework directory does not exist: '{sharedFrameworkDir}'. " + + "The feed-resolved runtime must be installed before overlaying."); } - // Determine artifact filename - string artifactFile; - if (_platformToBcsConfig.Values.Any(v => v.configKey == buildCacheConfig)) - { - artifactFile = _platformToBcsConfig.Values.First(v => v.configKey == buildCacheConfig).artifactFile; - } - else + var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); + if (nugetPackageDir != null) { - throw new InvalidOperationException($"Unknown Build Cache configuration key: '{buildCacheConfig}'."); + var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); + if (Directory.Exists(runtimesDir)) + { + filesCopied += CopyManaged(runtimesDir, sharedFrameworkDir); + filesCopied += CopyNative(runtimesDir, sharedFrameworkDir); + } } - // Construct the download URL - var artifactUrl = $"{baseUrl}/builds/{repoName}/buildArtifacts/{commitSha}/{buildCacheConfig}/{artifactFile}"; - - Log.Info($"Build Cache: Downloading {artifactFile} from {artifactUrl}"); - - // Download to a temp file - var tempDir = Path.Combine(Path.GetTempPath(), "crank-buildcache", commitSha); - Directory.CreateDirectory(tempDir); - var tempArchive = Path.Combine(tempDir, artifactFile); - - try + var corehostDir = FindCorehostDirectory(extractDir, rid); + if (corehostDir != null) { - if (!File.Exists(tempArchive)) - { - using var response = await _httpClient.GetAsync(artifactUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + // hostpolicy lives in the shared framework dir. + filesCopied += CopyHostBinaryIfPresent(corehostDir, sharedFrameworkDir, GetNativeLibName("hostpolicy")); - if (response.StatusCode == System.Net.HttpStatusCode.NotFound) - { - throw new InvalidOperationException($"Build Cache: Artifact not found for commit {commitSha} with config '{buildCacheConfig}'. The build may not exist in the cache."); - } - - response.EnsureSuccessStatusCode(); - - using var fileStream = File.Create(tempArchive); - await response.Content.CopyToAsync(fileStream, cancellationToken); - - Log.Info($"Build Cache: Downloaded {new FileInfo(tempArchive).Length / (1024 * 1024)} MB"); - } - else - { - Log.Info($"Build Cache: Using cached archive at {tempArchive}"); - } - - // Extract and overlay - var extractDir = Path.Combine(tempDir, "extracted"); - if (Directory.Exists(extractDir)) + // hostfxr lives at host/fxr/{version}/. + var hostFxrDir = Path.Combine(dotnetHome, "host", "fxr", runtimeVersion); + if (Directory.Exists(hostFxrDir)) { - Directory.Delete(extractDir, true); + filesCopied += CopyHostBinaryIfPresent(corehostDir, hostFxrDir, GetNativeLibName("hostfxr")); } - Log.Info($"Build Cache: Extracting archive..."); - - if (artifactFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase)) - { - await ExtractTarGzAsync(tempArchive, extractDir, cancellationToken); - } - else if (artifactFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) - { - ZipFile.ExtractToDirectory(tempArchive, extractDir); - } - else - { - throw new InvalidOperationException($"Unsupported archive format: {artifactFile}"); - } + // The dotnet host lives at the dotnetHome root. + filesCopied += CopyHostBinaryIfPresent(corehostDir, dotnetHome, GetDotnetExecutableName()); + } - // Overlay into dotnet home - var runtimeVersion = await OverlayRuntimeAsync(extractDir, dotnetHome, platformMoniker, targetFramework, commitSha, cancellationToken); + return filesCopied; + } - _installedBuildCacheRuntimes.TryAdd(commitSha, 0); + // --- HTTP / latestBuilds.json ------------------------------------------------- - Log.Info($"Build Cache: Runtime {runtimeVersion} (commit {commitSha.Substring(0, Math.Min(8, commitSha.Length))}) installed successfully."); + private static async Task GetLatestBuildsAsync( + string baseUrl, string repoName, string branch, CancellationToken cancellationToken) + { + var normalizedBaseUrl = (baseUrl ?? string.Empty).TrimEnd('/'); + var cacheKey = $"{normalizedBaseUrl}|{repoName}|{branch}"; - return runtimeVersion; - } - catch (Exception ex) when (ex is not InvalidOperationException) + if (_latestBuildsCache.TryGetValue(cacheKey, out var cached) && + DateTimeOffset.UtcNow - cached.fetchedAt < _latestBuildsCacheDuration) { - throw new InvalidOperationException($"Build Cache: Failed to install runtime from commit {commitSha}: {ex.Message}", ex); + return cached.data; } - } - /// - /// Overlays extracted BCS artifacts into the dotnet home directory structure. - /// - private static async Task OverlayRuntimeAsync( - string extractDir, - string dotnetHome, - string platformMoniker, - string targetFramework, - string commitSha, - CancellationToken cancellationToken) - { - var versionPrefix = ExtractVersionPrefix(targetFramework); - var rid = platformMoniker; + // Branch may contain slashes (e.g., "release/10.0"). Escape each segment but keep + // the slash semantics so the URL still resolves correctly on the server. + var escapedBranch = string.Join("/", branch.Split('/').Select(Uri.EscapeDataString)); + var url = $"{normalizedBaseUrl}/builds/{Uri.EscapeDataString(repoName)}/latest/{escapedBranch}/latestBuilds.json"; - // The NuGet package layout inside the archive is at: - // microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/ - // lib/net{X}.0/ → managed DLLs - // native/ → native libraries - var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); - string managedDir = null; - string nativeDir = null; + Log.Info($"Build Cache: Fetching latest builds from {url}"); - if (nugetPackageDir != null) + string json = null; + await ProcessUtil.RetryOnExceptionAsync(DownloadRetryCount, async () => { - var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); + using var response = await _httpClient.GetAsync(url, cancellationToken); - if (Directory.Exists(runtimesDir)) + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) { - // Find the lib/net{X}.0 directory - var libDir = Path.Combine(runtimesDir, "lib"); - if (Directory.Exists(libDir)) - { - managedDir = Directory.GetDirectories(libDir) - .FirstOrDefault(d => Path.GetFileName(d).StartsWith($"net{versionPrefix}", StringComparison.OrdinalIgnoreCase)) - ?? Directory.GetDirectories(libDir).FirstOrDefault(); - } - - nativeDir = Path.Combine(runtimesDir, "native"); - if (!Directory.Exists(nativeDir)) - { - nativeDir = null; - } + throw new InvalidOperationException( + $"Build Cache: No latest builds found for branch '{branch}' in repo '{repoName}'. URL: {url}"); } - } - // Determine the runtime version from the managed DLLs directory or other metadata - var runtimeVersion = DetermineRuntimeVersion(extractDir, versionPrefix, commitSha); + response.EnsureSuccessStatusCode(); + json = await response.Content.ReadAsStringAsync(cancellationToken); + }); - // Create the shared framework directory - var sharedFrameworkDir = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); - Directory.CreateDirectory(sharedFrameworkDir); + var latestBuilds = ParseLatestBuilds(json); + _latestBuildsCache[cacheKey] = (DateTimeOffset.UtcNow, latestBuilds); + return latestBuilds; + } - int filesCopied = 0; + private static async Task DownloadWithRetryAsync(string url, string destination, CancellationToken cancellationToken) + { + var partial = destination + ".partial"; - // Copy managed DLLs - if (managedDir != null && Directory.Exists(managedDir)) + await ProcessUtil.RetryOnExceptionAsync(DownloadRetryCount, async () => { - foreach (var file in Directory.GetFiles(managedDir, "*.dll")) + if (File.Exists(partial)) { - File.Copy(file, Path.Combine(sharedFrameworkDir, Path.GetFileName(file)), overwrite: true); - filesCopied++; + File.Delete(partial); } - Log.Info($"Build Cache: Copied {filesCopied} managed assemblies to shared framework."); - } - - // Copy native libraries - if (nativeDir != null && Directory.Exists(nativeDir)) - { - int nativeCount = 0; - - foreach (var file in Directory.GetFiles(nativeDir)) + using (var response = await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken)) { - var fileName = Path.GetFileName(file); - - // Skip debug symbols during overlay (keep it lean) - if (fileName.EndsWith(".dbg", StringComparison.OrdinalIgnoreCase) || - fileName.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase)) + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) { - continue; + // Don't retry 404s; they aren't transient. + throw new InvalidOperationException( + $"Build Cache: Artifact not found at {url}. The build may not exist in the cache."); } - File.Copy(file, Path.Combine(sharedFrameworkDir, fileName), overwrite: true); - nativeCount++; - } - - Log.Info($"Build Cache: Copied {nativeCount} native libraries to shared framework."); - filesCopied += nativeCount; - } - - // Also look for host binaries in the corehost directory - // Pattern: {rid}.Release/corehost/ or linux-arm64.Release/corehost/ - var corehostDir = FindCorehostDirectory(extractDir, platformMoniker); - - if (corehostDir != null) - { - // Copy libhostpolicy to shared framework - CopyFileIfExists(corehostDir, sharedFrameworkDir, GetNativeLibName("hostpolicy")); + response.EnsureSuccessStatusCode(); - // Copy libhostfxr to host/fxr/{version}/ - var hostFxrDir = Path.Combine(dotnetHome, "host", "fxr", runtimeVersion); - Directory.CreateDirectory(hostFxrDir); - CopyFileIfExists(corehostDir, hostFxrDir, GetNativeLibName("hostfxr")); + var expectedLength = response.Content.Headers.ContentLength; - Log.Info($"Build Cache: Copied host binaries."); - } + using (var fileStream = File.Create(partial)) + { + await response.Content.CopyToAsync(fileStream, cancellationToken); + } - // Write a .version file with the commit SHA for traceability - var versionFilePath = Path.Combine(sharedFrameworkDir, ".version"); - await File.WriteAllTextAsync(versionFilePath, $"{commitSha}\n{runtimeVersion}\n", cancellationToken); + if (expectedLength.HasValue) + { + var actual = new FileInfo(partial).Length; + if (actual != expectedLength.Value) + { + throw new InvalidOperationException( + $"Build Cache: Download size mismatch (expected {expectedLength.Value}, got {actual}). URL: {url}"); + } + } + } - if (filesCopied == 0) - { - throw new InvalidOperationException($"Build Cache: No runtime files found to extract. The archive may not contain the expected layout for platform '{platformMoniker}'."); - } + if (File.Exists(destination)) + { + File.Delete(destination); + } - return runtimeVersion; + File.Move(partial, destination); + }); } /// - /// Determines the runtime version from extracted artifacts. + /// Parses the latestBuilds.json format from BCS. The JSON has dynamic keys for each + /// build configuration plus a "branch_name" / "BranchName" string property. /// - private static string DetermineRuntimeVersion(string extractDir, string versionPrefix, string commitSha) + internal static LatestBuildsResponse ParseLatestBuilds(string json) { - // Look for .version file in the shared framework subdirectory of the archive - var versionFiles = Directory.GetFiles(extractDir, ".version", SearchOption.AllDirectories); + var result = new LatestBuildsResponse(); + + using var doc = JsonDocument.Parse(json); - foreach (var versionFile in versionFiles) + foreach (var property in doc.RootElement.EnumerateObject()) { - try + if (property.Name.Equals("branch_name", StringComparison.OrdinalIgnoreCase) || + property.Name.Equals("BranchName", StringComparison.Ordinal)) { - var lines = File.ReadAllLines(versionFile); - // The .version file typically has: line 0 = commit hash, line 1 = version string - if (lines.Length >= 2 && lines[1].StartsWith(versionPrefix, StringComparison.OrdinalIgnoreCase)) + if (property.Value.ValueKind == JsonValueKind.String) { - return lines[1].Trim(); + result.BranchName = property.Value.GetString(); } + continue; } - catch + + if (property.Value.ValueKind == JsonValueKind.Object) { - // Continue searching + var entry = new LatestBuildEntry + { + CommitSha = TryGetStringPropertyAnyCase(property.Value, "CommitSha", "commit_sha"), + CommitTime = TryGetStringPropertyAnyCase(property.Value, "CommitTime", "commit_time"), + }; + + result.Entries[property.Name] = entry; } } - // Fallback: construct a version string from the prefix and commit - return $"{versionPrefix}.0-buildcache.{commitSha.Substring(0, Math.Min(8, commitSha.Length))}"; + return result; } - /// - /// Reads the runtime version from an already-installed BCS runtime. - /// - private static string ReadInstalledBuildCacheVersion(string dotnetHome, string targetFramework) + private static string TryGetStringPropertyAnyCase(JsonElement element, params string[] names) { - var versionPrefix = ExtractVersionPrefix(targetFramework); - var sharedDir = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App"); - - if (!Directory.Exists(sharedDir)) - { - return null; - } - - // Find directories matching the version prefix that have a .version file with a commit SHA - foreach (var dir in Directory.GetDirectories(sharedDir).OrderByDescending(d => d)) + foreach (var name in names) { - var dirName = Path.GetFileName(dir); - if (dirName.StartsWith(versionPrefix, StringComparison.OrdinalIgnoreCase)) + if (element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String) { - var versionFile = Path.Combine(dir, ".version"); - if (File.Exists(versionFile)) - { - var lines = File.ReadAllLines(versionFile); - if (lines.Length >= 2) - { - return lines[1].Trim(); - } - } + return value.GetString(); } } return null; } - private static async Task GetLatestBuildsAsync( - string baseUrl, string repoName, string branch, CancellationToken cancellationToken) + // --- Extraction --------------------------------------------------------------- + + private static Task ExtractArchiveAsync(string archivePath, string outputDir, CancellationToken cancellationToken) { - var cacheKey = $"{baseUrl}|{repoName}/{branch}"; + Directory.CreateDirectory(outputDir); - if (_latestBuildsCache.TryGetValue(cacheKey, out var cached) && - DateTimeOffset.UtcNow - cached.fetchedAt < _latestBuildsCacheDuration) + if (archivePath.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase)) { - return cached.data; + return ExtractTarGzAsync(archivePath, outputDir, cancellationToken); } - var url = $"{baseUrl}/builds/{repoName}/latest/{branch}/latestBuilds.json"; - Log.Info($"Build Cache: Fetching latest builds from {url}"); + if (archivePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + return Task.Run(() => ZipFile.ExtractToDirectory(archivePath, outputDir, overwriteFiles: true), cancellationToken); + } + + throw new InvalidOperationException($"Unsupported archive format: {archivePath}"); + } - using var response = await _httpClient.GetAsync(url, cancellationToken); + private static async Task ExtractTarGzAsync(string archivePath, string outputDir, CancellationToken cancellationToken) + { + await using var fs = File.OpenRead(archivePath); + await using var gz = new GZipStream(fs, CompressionMode.Decompress); + await TarFile.ExtractToDirectoryAsync(gz, outputDir, overwriteFiles: true, cancellationToken: cancellationToken); + } - if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + // --- Overlay helpers ---------------------------------------------------------- + + private static int CopyManaged(string runtimesDir, string destinationDir) + { + int copied = 0; + var libDir = Path.Combine(runtimesDir, "lib"); + if (!Directory.Exists(libDir)) { - throw new InvalidOperationException($"Build Cache: No latest builds found for branch '{branch}' in repo '{repoName}'. URL: {url}"); + return 0; } - response.EnsureSuccessStatusCode(); + // Pick the highest-versioned net{X}.0 directory (the archive should only ship one). + var managedDir = Directory.GetDirectories(libDir) + .OrderByDescending(d => Path.GetFileName(d), StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); - var json = await response.Content.ReadAsStringAsync(cancellationToken); - var latestBuilds = ParseLatestBuilds(json); + if (managedDir == null) + { + return 0; + } - _latestBuildsCache[cacheKey] = (DateTimeOffset.UtcNow, latestBuilds); + Directory.CreateDirectory(destinationDir); - return latestBuilds; + foreach (var file in Directory.GetFiles(managedDir, "*.dll")) + { + var dest = Path.Combine(destinationDir, Path.GetFileName(file)); + File.Copy(file, dest, overwrite: true); + copied++; + } + + return copied; } - /// - /// Parses the latestBuilds.json format from BCS. The JSON has dynamic keys for each - /// build configuration, with "branch_name" as a special key. - /// - private static LatestBuildsResponse ParseLatestBuilds(string json) + private static int CopyNative(string runtimesDir, string destinationDir) { - var result = new LatestBuildsResponse(); + int copied = 0; + var nativeDir = Path.Combine(runtimesDir, "native"); + if (!Directory.Exists(nativeDir)) + { + return 0; + } - using var doc = JsonDocument.Parse(json); + Directory.CreateDirectory(destinationDir); - foreach (var property in doc.RootElement.EnumerateObject()) + foreach (var file in Directory.GetFiles(nativeDir)) { - if (property.Name == "branch_name") + var fileName = Path.GetFileName(file); + if (fileName.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".dbg", StringComparison.OrdinalIgnoreCase)) { - result.BranchName = property.Value.GetString(); continue; } - if (property.Value.ValueKind == JsonValueKind.Object) - { - var entry = new LatestBuildEntry - { - CommitSha = property.Value.TryGetProperty("CommitSha", out var sha) ? sha.GetString() - : property.Value.TryGetProperty("commit_sha", out sha) ? sha.GetString() - : null, - CommitTime = property.Value.TryGetProperty("CommitTime", out var time) ? time.GetString() - : property.Value.TryGetProperty("commit_time", out time) ? time.GetString() - : null, - }; + var dest = Path.Combine(destinationDir, fileName); + File.Copy(file, dest, overwrite: true); + copied++; + } - result.Entries[property.Name] = entry; - } + return copied; + } + + private static int CopyHostBinaryIfPresent(string sourceDir, string destDir, string fileName) + { + var src = Path.Combine(sourceDir, fileName); + if (!File.Exists(src)) + { + return 0; } - return result; + Directory.CreateDirectory(destDir); + File.Copy(src, Path.Combine(destDir, fileName), overwrite: true); + return 1; } private static string FindDirectory(string root, string directoryName) @@ -649,7 +495,6 @@ private static string FindDirectory(string root, string directoryName) return null; } - // Check direct children first foreach (var dir in Directory.GetDirectories(root)) { if (Path.GetFileName(dir).Equals(directoryName, StringComparison.OrdinalIgnoreCase)) @@ -661,124 +506,112 @@ private static string FindDirectory(string root, string directoryName) return null; } - private static string FindCorehostDirectory(string extractDir, string platformMoniker) + private static string FindCorehostDirectory(string extractDir, string rid) { - // BCS layout: {rid}.Release/corehost/ (e.g., linux-arm64.Release/corehost/) - // Map RID to the directory name format used in BCS artifacts - var ridDirName = $"{platformMoniker}.Release"; - var corehostPath = Path.Combine(extractDir, ridDirName, "corehost"); - - if (Directory.Exists(corehostPath)) + var primary = Path.Combine(extractDir, $"{rid}.Release", "corehost"); + if (Directory.Exists(primary)) { - return corehostPath; + return primary; } - // Also try the raw format without dots - var altCorehostPath = Path.Combine(extractDir, "corehost"); - if (Directory.Exists(altCorehostPath)) + var alternate = Path.Combine(extractDir, "corehost"); + if (Directory.Exists(alternate)) { - return altCorehostPath; + return alternate; } return null; } - private static void CopyFileIfExists(string sourceDir, string destDir, string fileName) - { - var sourcePath = Path.Combine(sourceDir, fileName); - if (File.Exists(sourcePath)) - { - File.Copy(sourcePath, Path.Combine(destDir, fileName), overwrite: true); - } - } + // --- Platform / RID mapping --------------------------------------------------- - private static string GetNativeLibName(string baseName) + private static string ResolveBuildCacheConfig(string buildCacheConfig) { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (!string.IsNullOrEmpty(buildCacheConfig)) { - return $"{baseName}.dll"; + return buildCacheConfig; } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + + var rid = GetPlatformMoniker(); + if (PlatformToBcsConfig.TryGetValue(rid, out var mapped)) { - return $"lib{baseName}.dylib"; + return mapped.configKey; } - else + + throw new InvalidOperationException( + $"No Build Cache configuration mapping for platform '{rid}'. Specify buildCacheConfig explicitly."); + } + + private static string GetArtifactFile(string buildCacheConfig) + { + var match = PlatformToBcsConfig.Values.FirstOrDefault(v => + string.Equals(v.configKey, buildCacheConfig, StringComparison.OrdinalIgnoreCase)); + + if (match.artifactFile == null) { - return $"lib{baseName}.so"; + throw new InvalidOperationException( + $"Unknown Build Cache configuration key: '{buildCacheConfig}'."); } + + return match.artifactFile; } - private static string GetPlatformMoniker() + internal static string GetNativeLibName(string baseName) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "win-arm64" - : RuntimeInformation.ProcessArchitecture == Architecture.X86 ? "win-x86" - : "win-x64"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "osx-arm64" : "osx-x64"; + return $"{baseName}.dll"; } - else + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "linux-arm64" : "linux-x64"; + return $"lib{baseName}.dylib"; } + + return $"lib{baseName}.so"; } - private static string ExtractVersionPrefix(string targetFramework) - { - if (string.IsNullOrWhiteSpace(targetFramework)) - { - throw new InvalidOperationException("Target framework must be specified."); - } + private static string GetDotnetExecutableName() + => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet"; - // "net10.0" → "10.0", "net9.0" → "9.0" - if (targetFramework.StartsWith("net", StringComparison.OrdinalIgnoreCase) && - targetFramework.Length > 3 && - char.IsDigit(targetFramework[3])) + internal static string GetPlatformMoniker() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - return targetFramework.Substring(3); + return RuntimeInformation.ProcessArchitecture switch + { + Architecture.Arm64 => "win-arm64", + Architecture.X86 => "win-x86", + _ => "win-x64", + }; } - // "netcoreapp3.1" → "3.1" - if (targetFramework.StartsWith("netcoreapp", StringComparison.OrdinalIgnoreCase) && - targetFramework.Length > "netcoreapp".Length) + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - return targetFramework.Substring("netcoreapp".Length); + return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "osx-arm64" : "osx-x64"; } - throw new InvalidOperationException( - $"Unsupported target framework '{targetFramework}' for Build Cache runtime version inference."); + return RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "linux-arm64" : "linux-x64"; } - private static async Task ExtractTarGzAsync(string archivePath, string outputDir, CancellationToken cancellationToken) + private static string SanitizeForPath(string value) { - Directory.CreateDirectory(outputDir); - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (string.IsNullOrEmpty(value)) { - // On Windows, use tar (available since Windows 10 1803) - var result = await ProcessUtil.RunAsync("tar", $"-xzf \"{archivePath}\" -C \"{outputDir}\"", - throwOnError: false, cancellationToken: cancellationToken); - - if (result.ExitCode != 0) - { - throw new InvalidOperationException($"Failed to extract tar.gz: {result.StandardError}"); - } + return "default"; } - else - { - var result = await ProcessUtil.RunAsync("/usr/bin/env", $"tar -xzf \"{archivePath}\" -C \"{outputDir}\"", - throwOnError: false, cancellationToken: cancellationToken); - if (result.ExitCode != 0) - { - throw new InvalidOperationException($"Failed to extract tar.gz: {result.StandardError}"); - } - } + var invalid = Path.GetInvalidFileNameChars(); + return string.Concat(value.Select(c => invalid.Contains(c) ? '_' : c)); } + internal static string ShortSha(string commitSha) + => string.IsNullOrEmpty(commitSha) + ? string.Empty + : commitSha.Substring(0, Math.Min(8, commitSha.Length)); + + // --- DTOs --------------------------------------------------------------------- + internal class LatestBuildsResponse { public string BranchName { get; set; } diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index 4300949f1..40c50b620 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -2983,6 +2983,7 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str string buildCacheCommitSha = null; string buildCacheExtractDir = null; + string buildCacheConfigResolved = null; if (useBuildCache) { if (!_buildCacheEnabled) @@ -2991,27 +2992,38 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str return null; } + // Validate user-supplied commit SHA early so we can fail with a clear message instead + // of throwing later from a Substring call. + if (!string.IsNullOrEmpty(job.BuildCacheCommitSha) && job.BuildCacheCommitSha.Length < 8) + { + job.Error = $"Build Cache: 'buildCacheCommitSha' must be at least 8 characters long (got '{job.BuildCacheCommitSha}')."; + return null; + } + try { var branch = !string.IsNullOrEmpty(job.BuildCacheBranch) ? job.BuildCacheBranch : "main"; var commitSha = job.BuildCacheCommitSha; var buildCacheConfig = job.BuildCacheConfig; - // Resolve which commit to use + // Resolve which commit and config to use var resolved = await BuildCacheClient.ResolveCommitAsync( - _buildCacheBaseUrl, _buildCacheRepoName, branch, commitSha, buildCacheConfig); + _buildCacheBaseUrl, _buildCacheRepoName, branch, commitSha, buildCacheConfig, cancellationToken); buildCacheCommitSha = resolved.commitSha; + buildCacheConfigResolved = resolved.buildCacheConfig; - // Download and extract the BCS artifacts to a temp directory + // Download and extract the BCS artifacts to a per-job temp directory buildCacheExtractDir = await BuildCacheClient.DownloadAndExtractAsync( - _buildCacheBaseUrl, _buildCacheRepoName, buildCacheCommitSha, buildCacheConfig, - targetFramework, cancellationToken); + _buildCacheBaseUrl, _buildCacheRepoName, buildCacheCommitSha, buildCacheConfigResolved, + cancellationToken); - var shortSha = buildCacheCommitSha.Substring(0, Math.Min(8, buildCacheCommitSha.Length)); + var shortSha = BuildCacheClient.ShortSha(buildCacheCommitSha); Log.Info($"Build Cache: Artifacts for commit {shortSha} ready for post-build overlay"); - // Resolve a REAL runtime version from feeds for the NuGet build + // Resolve a REAL runtime version from feeds for the NuGet build. We deliberately keep + // runtimeVersion pointing at this feed-resolved version so PatchRuntimeConfig and the + // dotnet-install steps agree; the BCS bits are overlaid on top of that exact version. runtimeVersion = await ResolveRuntimeVersion(buildToolsPath, targetFramework, "Latest"); Log.Info($"Runtime for build: {runtimeVersion} (Latest from feeds, will be overlaid with BCS commit {shortSha})"); } @@ -3506,23 +3518,44 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => Log.Info($"Application published successfully in {job.BuildTime.TotalMilliseconds} ms"); - // Build Cache: overlay BCS runtime binaries into the published output - // This replaces the NuGet-sourced runtime DLLs with BCS-built ones from the specific commit + // Build Cache: overlay BCS runtime binaries onto the just-published app and the + // agent's installed shared framework. We overlay BOTH targets so that: + // - self-contained jobs (which load the runtime from outputFolder) see BCS bits, and + // - framework-dependent jobs (which load the runtime from dotnetHome/shared/...) see BCS bits. + // PatchRuntimeConfig still runs with the feed-resolved runtimeVersion so runtimeconfig.json + // points to a real installed directory; the BCS overlay replaces the binaries inside it. if (useBuildCache && buildCacheExtractDir != null) { + var shortSha = BuildCacheClient.ShortSha(buildCacheCommitSha); + int totalOverlaid; + try { - var shortSha = buildCacheCommitSha.Substring(0, Math.Min(8, buildCacheCommitSha.Length)); - var overlayCount = BuildCacheClient.OverlayPublishedOutput(buildCacheExtractDir, outputFolder); - Log.Info($"Build Cache: Overlaid {overlayCount} runtime files from commit {shortSha} into published output"); + var publishedOverlay = BuildCacheClient.OverlayPublishedOutput(buildCacheExtractDir, outputFolder); + var dotnetHomeOverlay = BuildCacheClient.OverlayDotnetHome(buildCacheExtractDir, dotnetHome, runtimeVersion); + totalOverlaid = publishedOverlay + dotnetHomeOverlay; - // Update the reported runtime version to reflect the BCS commit (for metadata/reporting only) - job.RuntimeVersion = $"{runtimeVersion}+buildcache.{shortSha}"; + Log.Info($"Build Cache: Overlaid {publishedOverlay} files into published output and " + + $"{dotnetHomeOverlay} files into dotnet home (commit {shortSha})"); } catch (Exception ex) { - Log.Info($"Build Cache: Warning - overlay failed: {ex.Message}. Published app will use feed-sourced runtime."); + job.Error = $"Build Cache: overlay failed: {ex.Message}"; + return null; } + + if (totalOverlaid == 0) + { + // Silently skipping the overlay would mean the benchmark runs against the FEED runtime + // instead of the BCS commit, which is the worst possible outcome for bisection. Fail. + job.Error = $"Build Cache: overlay copied 0 files for commit {shortSha}. " + + "The archive layout may have changed or the platform is not supported."; + return null; + } + + // Record the BCS commit alongside the runtime version for reporting. We append rather than + // replace so PatchRuntimeConfig still sees a valid feed-resolved version below. + job.RuntimeVersion = $"{runtimeVersion}+buildcache.{shortSha}"; } PatchRuntimeConfig(job, outputFolder, aspNetCoreVersion, runtimeVersion); diff --git a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs new file mode 100644 index 000000000..bf2e784e3 --- /dev/null +++ b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs @@ -0,0 +1,406 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Microsoft.Crank.Agent; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Crank.UnitTests +{ + public class BuildCacheClientTests : IDisposable + { + private readonly ITestOutputHelper _output; + private readonly string _testDir; + + public BuildCacheClientTests(ITestOutputHelper output) + { + _output = output; + _testDir = Path.Combine(Path.GetTempPath(), "crank_buildcache_tests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_testDir); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_testDir)) + { + Directory.Delete(_testDir, true); + } + } + catch + { + // Ignore cleanup errors + } + } + + // ------------------------------------------------------------------- + // ParseLatestBuilds + // ------------------------------------------------------------------- + + [Fact] + public void ParseLatestBuilds_PascalCase_ParsesCommitShaAndTime() + { + const string json = """ + { + "BranchName": "main", + "coreclr_x64_linux": { + "CommitSha": "abc123def456", + "CommitTime": "2025-01-01T00:00:00Z" + } + } + """; + + var result = BuildCacheClient.ParseLatestBuilds(json); + + Assert.Equal("main", result.BranchName); + Assert.True(result.Entries.ContainsKey("coreclr_x64_linux")); + Assert.Equal("abc123def456", result.Entries["coreclr_x64_linux"].CommitSha); + Assert.Equal("2025-01-01T00:00:00Z", result.Entries["coreclr_x64_linux"].CommitTime); + } + + [Fact] + public void ParseLatestBuilds_SnakeCase_ParsesCommitShaAndTime() + { + const string json = """ + { + "branch_name": "release/10.0", + "coreclr_arm64_linux": { + "commit_sha": "deadbeef", + "commit_time": "2025-02-02T00:00:00Z" + } + } + """; + + var result = BuildCacheClient.ParseLatestBuilds(json); + + Assert.Equal("release/10.0", result.BranchName); + Assert.True(result.Entries.ContainsKey("coreclr_arm64_linux")); + Assert.Equal("deadbeef", result.Entries["coreclr_arm64_linux"].CommitSha); + Assert.Equal("2025-02-02T00:00:00Z", result.Entries["coreclr_arm64_linux"].CommitTime); + } + + [Fact] + public void ParseLatestBuilds_MixedCasing_ParsesAllConfigs() + { + const string json = """ + { + "branch_name": "main", + "coreclr_x64_windows": { "CommitSha": "win123", "CommitTime": "2025-03-03" }, + "coreclr_x64_linux": { "commit_sha": "lnx456", "commit_time": "2025-04-04" } + } + """; + + var result = BuildCacheClient.ParseLatestBuilds(json); + + Assert.Equal(2, result.Entries.Count); + Assert.Equal("win123", result.Entries["coreclr_x64_windows"].CommitSha); + Assert.Equal("lnx456", result.Entries["coreclr_x64_linux"].CommitSha); + } + + [Fact] + public void ParseLatestBuilds_MissingFields_ReturnsNullsWithoutThrowing() + { + const string json = """ + { + "branch_name": "main", + "coreclr_x64_linux": { "CommitSha": "abc" }, + "empty_config": {} + } + """; + + var result = BuildCacheClient.ParseLatestBuilds(json); + + Assert.Equal("abc", result.Entries["coreclr_x64_linux"].CommitSha); + Assert.Null(result.Entries["coreclr_x64_linux"].CommitTime); + Assert.Null(result.Entries["empty_config"].CommitSha); + Assert.Null(result.Entries["empty_config"].CommitTime); + } + + [Fact] + public void ParseLatestBuilds_EntriesLookupIsCaseInsensitive() + { + const string json = """ + { "branch_name": "main", "CoreCLR_X64_Linux": { "CommitSha": "abc" } } + """; + + var result = BuildCacheClient.ParseLatestBuilds(json); + + Assert.True(result.Entries.ContainsKey("coreclr_x64_linux")); + Assert.True(result.Entries.ContainsKey("CORECLR_X64_LINUX")); + } + + [Fact] + public void ParseLatestBuilds_NonObjectValues_AreSkipped() + { + // Real-world payloads sometimes carry non-object metadata that must be ignored. + const string json = """ + { + "branch_name": "main", + "schemaVersion": 2, + "lastUpdated": "2025-01-01", + "coreclr_x64_linux": { "CommitSha": "abc" } + } + """; + + var result = BuildCacheClient.ParseLatestBuilds(json); + + Assert.Single(result.Entries); + Assert.True(result.Entries.ContainsKey("coreclr_x64_linux")); + } + + // ------------------------------------------------------------------- + // GetPlatformMoniker + // ------------------------------------------------------------------- + + [Fact] + public void GetPlatformMoniker_ReturnsKnownRid() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + + var validRids = new[] + { + "linux-x64", "linux-arm64", + "win-x64", "win-arm64", "win-x86", + "osx-x64", "osx-arm64", + }; + + Assert.Contains(rid, validRids); + } + + [Fact] + public void PlatformToBcsConfig_ContainsAllSupportedRids() + { + // Sanity: agents typically run on these RIDs; ensure the table covers them. + Assert.True(BuildCacheClient.PlatformToBcsConfig.ContainsKey("linux-x64")); + Assert.True(BuildCacheClient.PlatformToBcsConfig.ContainsKey("linux-arm64")); + Assert.True(BuildCacheClient.PlatformToBcsConfig.ContainsKey("win-x64")); + } + + // ------------------------------------------------------------------- + // ShortSha + // ------------------------------------------------------------------- + + [Fact] + public void ShortSha_LongInput_ReturnsFirstEight() + { + Assert.Equal("abcdef12", BuildCacheClient.ShortSha("abcdef1234567890")); + } + + [Fact] + public void ShortSha_ShortInput_ReturnsAsIs() + { + Assert.Equal("abc", BuildCacheClient.ShortSha("abc")); + } + + [Fact] + public void ShortSha_NullOrEmpty_ReturnsEmpty() + { + Assert.Equal(string.Empty, BuildCacheClient.ShortSha(null)); + Assert.Equal(string.Empty, BuildCacheClient.ShortSha("")); + } + + // ------------------------------------------------------------------- + // GetNativeLibName + // ------------------------------------------------------------------- + + [Fact] + public void GetNativeLibName_MatchesHostPlatform() + { + var name = BuildCacheClient.GetNativeLibName("hostpolicy"); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Assert.Equal("hostpolicy.dll", name); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + Assert.Equal("libhostpolicy.dylib", name); + } + else + { + Assert.Equal("libhostpolicy.so", name); + } + } + + // ------------------------------------------------------------------- + // OverlayPublishedOutput / OverlayDotnetHome + // ------------------------------------------------------------------- + + [Fact] + public void OverlayPublishedOutput_CopiesAllRuntimeFilesUnconditionally() + { + // Build a fake BCS extract layout for the host RID. + var rid = BuildCacheClient.GetPlatformMoniker(); + var (extractDir, _, expectedManagedNames, expectedNativeNames) = BuildFakeBcsArchive(rid, includeHost: true); + + var outputFolder = Path.Combine(_testDir, "published"); + Directory.CreateDirectory(outputFolder); + + // Note: outputFolder is intentionally EMPTY — the overlay must still copy + // managed/native runtime files (regression: earlier behavior skipped missing dest). + var copied = BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder); + + Assert.True(copied >= expectedManagedNames.Count + expectedNativeNames.Count, + $"Expected at least {expectedManagedNames.Count + expectedNativeNames.Count} files; got {copied}"); + + foreach (var dll in expectedManagedNames) + { + Assert.True(File.Exists(Path.Combine(outputFolder, dll)), $"Missing managed file {dll}"); + } + foreach (var native in expectedNativeNames) + { + Assert.True(File.Exists(Path.Combine(outputFolder, native)), $"Missing native file {native}"); + } + + // hostpolicy must have been copied alongside the app for self-contained. + Assert.True(File.Exists(Path.Combine(outputFolder, BuildCacheClient.GetNativeLibName("hostpolicy")))); + } + + [Fact] + public void OverlayPublishedOutput_NoMatchingPlatformLayout_ReturnsZero() + { + // Empty extract directory ⇒ overlay finds nothing ⇒ returns 0 (caller fails the job). + var extractDir = Path.Combine(_testDir, "empty-extract"); + Directory.CreateDirectory(extractDir); + + var outputFolder = Path.Combine(_testDir, "published"); + Directory.CreateDirectory(outputFolder); + + var copied = BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder); + + Assert.Equal(0, copied); + } + + [Fact] + public void OverlayPublishedOutput_SkipsPdbAndDbgFiles() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var (extractDir, runtimesDir, _, _) = BuildFakeBcsArchive(rid, includeHost: false); + + // Add a .pdb and .dbg in the native dir. + var nativeDir = Path.Combine(runtimesDir, "native"); + File.WriteAllText(Path.Combine(nativeDir, "coreclr.pdb"), "pdb"); + File.WriteAllText(Path.Combine(nativeDir, "libcoreclr.dbg"), "dbg"); + + var outputFolder = Path.Combine(_testDir, "published"); + Directory.CreateDirectory(outputFolder); + + BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder); + + Assert.False(File.Exists(Path.Combine(outputFolder, "coreclr.pdb"))); + Assert.False(File.Exists(Path.Combine(outputFolder, "libcoreclr.dbg"))); + } + + [Fact] + public void OverlayDotnetHome_RequiresExistingSharedFrameworkDir() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var (extractDir, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true); + + var dotnetHome = Path.Combine(_testDir, "dotnetHome-missing"); + Directory.CreateDirectory(dotnetHome); + + // shared/Microsoft.NETCore.App/{version} does NOT exist ⇒ should throw with a clear message. + var ex = Assert.Throws( + () => BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, "10.0.0-preview.1")); + + Assert.Contains("shared framework", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void OverlayDotnetHome_OverlaysSharedFrameworkAndHostFxr() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var (extractDir, _, expectedManagedNames, expectedNativeNames) = BuildFakeBcsArchive(rid, includeHost: true); + + const string runtimeVersion = "10.0.0-preview.1"; + var dotnetHome = Path.Combine(_testDir, "dotnetHome"); + var sharedFw = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); + var hostFxr = Path.Combine(dotnetHome, "host", "fxr", runtimeVersion); + Directory.CreateDirectory(sharedFw); + Directory.CreateDirectory(hostFxr); + + var copied = BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, runtimeVersion); + + Assert.True(copied > 0); + foreach (var dll in expectedManagedNames) + { + Assert.True(File.Exists(Path.Combine(sharedFw, dll)), $"Missing managed in shared FW: {dll}"); + } + foreach (var native in expectedNativeNames) + { + Assert.True(File.Exists(Path.Combine(sharedFw, native)), $"Missing native in shared FW: {native}"); + } + + Assert.True(File.Exists(Path.Combine(hostFxr, BuildCacheClient.GetNativeLibName("hostfxr")))); + } + + // ------------------------------------------------------------------- + // Fake BCS archive helpers + // ------------------------------------------------------------------- + + /// + /// Builds an on-disk fake of an extracted BCS archive matching the layout the agent expects: + /// microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net10.0/*.dll, + /// microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/native/*, and + /// optionally {rid}.Release/corehost/*. + /// + private (string extractDir, string runtimesDir, System.Collections.Generic.List managed, System.Collections.Generic.List native) + BuildFakeBcsArchive(string rid, bool includeHost) + { + var extractDir = Path.Combine(_testDir, "extracted-" + Guid.NewGuid().ToString("N")); + var nugetPkg = Path.Combine(extractDir, $"microsoft.netcore.app.runtime.{rid}"); + var runtimesDir = Path.Combine(nugetPkg, "Release", "runtimes", rid); + var libDir = Path.Combine(runtimesDir, "lib", "net10.0"); + var nativeDir = Path.Combine(runtimesDir, "native"); + Directory.CreateDirectory(libDir); + Directory.CreateDirectory(nativeDir); + + var managed = new System.Collections.Generic.List + { + "System.Private.CoreLib.dll", + "System.Runtime.dll", + "System.Console.dll", + }; + foreach (var dll in managed) + { + File.WriteAllText(Path.Combine(libDir, dll), "fake managed " + dll); + } + + var native = new System.Collections.Generic.List(); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + native.AddRange(new[] { "coreclr.dll", "clrjit.dll" }); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + native.AddRange(new[] { "libcoreclr.dylib", "libclrjit.dylib" }); + } + else + { + native.AddRange(new[] { "libcoreclr.so", "libclrjit.so" }); + } + foreach (var n in native) + { + File.WriteAllText(Path.Combine(nativeDir, n), "fake native " + n); + } + + if (includeHost) + { + var hostDir = Path.Combine(extractDir, $"{rid}.Release", "corehost"); + Directory.CreateDirectory(hostDir); + File.WriteAllText(Path.Combine(hostDir, BuildCacheClient.GetNativeLibName("hostpolicy")), "hostpolicy"); + File.WriteAllText(Path.Combine(hostDir, BuildCacheClient.GetNativeLibName("hostfxr")), "hostfxr"); + File.WriteAllText(Path.Combine(hostDir, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet"), "dotnet"); + } + + return (extractDir, runtimesDir, managed, native); + } + } +} From 56657e2a7fd6eaf78921a64b491e1d28d0e1d14f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 7 May 2026 14:14:18 -0700 Subject: [PATCH 3/7] Address PR review (round 2): metadata reporting + early dotnet-home overlay Live-tested against an agent with --application.channel buildcache. The first end-to-end run was reporting the FEED commit hash in .NET Runtime Version because: 1. The .version file in shared/Microsoft.NETCore.App/{ver}/ was never rewritten after the BCS overlay, so any consumer of that file (notably the agent's own BenchmarksNetCoreAppVersion measurement metadata) reported the feed-installed commit. 2. Even with .version rewritten, the metadata-capture block runs immediately after the feed runtime install -- before the post-publish overlay -- so the feed commit was captured into Job.Measurements before BCS bits were in place. Fixes: * OverlayDotnetHome now accepts an optional commitSha parameter and rewrites shared/Microsoft.NETCore.App/{ver}/.version with the BCS commit so anything reading that file gets the correct hash. * Startup.cs splits the BCS overlay into two stages: - dotnet-home overlay runs RIGHT AFTER feed runtime install (before the BenchmarksNetCoreAppVersion metadata is captured), so the metadata picks up the BCS commit. Treat 0 files as fatal. - published-output overlay runs after publish (only path where outputFolder exists). For self-contained, 0 files is fatal; for FDD it's expected. * Two new tests: - OverlayDotnetHome_WithCommitSha_RewritesVersionFile - OverlayDotnetHome_WithoutCommitSha_LeavesVersionFileUntouched Live test result with --application.channel buildcache --application.framework net11.0 against a Linux agent: | .NET Runtime Version | 11.0.0-preview.5.26256.117+603403d9cb49 | | Requests/sec | 7,593,977 | The +603403d9cb49 suffix is the actual BCS commit, and 212+212 files were overlaid (dotnet home + published output). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Crank.Agent/BuildCacheClient.cs | 14 ++++- src/Microsoft.Crank.Agent/Startup.cs | 60 +++++++++++++------ .../BuildCacheClientTests.cs | 42 +++++++++++++ 3 files changed, 95 insertions(+), 21 deletions(-) diff --git a/src/Microsoft.Crank.Agent/BuildCacheClient.cs b/src/Microsoft.Crank.Agent/BuildCacheClient.cs index 46ba0fc37..c01e89267 100644 --- a/src/Microsoft.Crank.Agent/BuildCacheClient.cs +++ b/src/Microsoft.Crank.Agent/BuildCacheClient.cs @@ -196,10 +196,11 @@ public static int OverlayPublishedOutput(string extractDir, string outputFolder) /// /// Overlays BCS runtime binaries into the agent's installed dotnet home so framework- /// dependent apps that load the runtime from shared/Microsoft.NETCore.App/{version}/ - /// get the BCS bits at runtime. + /// get the BCS bits at runtime. Also rewrites the .version file so reporting code + /// that reads it (e.g., GetDependencies) picks up the BCS commit instead of the feed commit. /// /// Number of files overlaid. - public static int OverlayDotnetHome(string extractDir, string dotnetHome, string runtimeVersion) + public static int OverlayDotnetHome(string extractDir, string dotnetHome, string runtimeVersion, string commitSha = null) { if (string.IsNullOrEmpty(runtimeVersion)) { @@ -245,6 +246,15 @@ public static int OverlayDotnetHome(string extractDir, string dotnetHome, string filesCopied += CopyHostBinaryIfPresent(corehostDir, dotnetHome, GetDotnetExecutableName()); } + // Rewrite the .version file so anything that reads it (notably the agent's own + // GetDependencies / BenchmarksNetCoreAppVersion measurement) reports the BCS commit + // instead of the feed-installed commit. Format: "\n\n". + if (!string.IsNullOrEmpty(commitSha)) + { + var versionFile = Path.Combine(sharedFrameworkDir, ".version"); + File.WriteAllText(versionFile, $"{commitSha}\n{runtimeVersion}\n"); + } + return filesCopied; } diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index 40c50b620..ac904b1e5 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -3290,6 +3290,32 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => var dotnetDir = dotnetHome; + // Build Cache: overlay BCS bits into the freshly-installed shared framework BEFORE any + // metadata capture below reads the .version file. Doing it here means the Microsoft.NETCore.App + // metadata measurement records the BCS commit and the agent's GetDependencies pass picks up + // the BCS-built assemblies' AssemblyInformationalVersion. The published-output overlay still + // happens after publish (only that folder exists by then). + if (useBuildCache && buildCacheExtractDir != null) + { + try + { + var dotnetHomeOverlay = BuildCacheClient.OverlayDotnetHome( + buildCacheExtractDir, dotnetDir, runtimeVersion, buildCacheCommitSha); + Log.Info($"Build Cache: Overlaid {dotnetHomeOverlay} files into dotnet home (commit {BuildCacheClient.ShortSha(buildCacheCommitSha)})"); + + if (dotnetHomeOverlay == 0) + { + job.Error = $"Build Cache: dotnet-home overlay copied 0 files for commit {buildCacheCommitSha}."; + return null; + } + } + catch (Exception ex) + { + job.Error = $"Build Cache: dotnet-home overlay failed: {ex.Message}"; + return null; + } + } + // Updating Job to reflect actual versions used job.AspNetCoreVersion = aspNetCoreVersion; job.RuntimeVersion = runtimeVersion; @@ -3518,38 +3544,34 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => Log.Info($"Application published successfully in {job.BuildTime.TotalMilliseconds} ms"); - // Build Cache: overlay BCS runtime binaries onto the just-published app and the - // agent's installed shared framework. We overlay BOTH targets so that: - // - self-contained jobs (which load the runtime from outputFolder) see BCS bits, and - // - framework-dependent jobs (which load the runtime from dotnetHome/shared/...) see BCS bits. - // PatchRuntimeConfig still runs with the feed-resolved runtimeVersion so runtimeconfig.json - // points to a real installed directory; the BCS overlay replaces the binaries inside it. + // Build Cache: overlay BCS runtime binaries onto the just-published app. The agent's + // installed shared framework was already overlaid earlier (right after install) so the + // .NET runtime metadata and FDD execution see BCS bits; here we cover the SCD case where + // the runtime ships in the publish output. PatchRuntimeConfig still runs with the + // feed-resolved runtimeVersion so runtimeconfig.json points to a real installed dir. if (useBuildCache && buildCacheExtractDir != null) { var shortSha = BuildCacheClient.ShortSha(buildCacheCommitSha); - int totalOverlaid; + int publishedOverlay; try { - var publishedOverlay = BuildCacheClient.OverlayPublishedOutput(buildCacheExtractDir, outputFolder); - var dotnetHomeOverlay = BuildCacheClient.OverlayDotnetHome(buildCacheExtractDir, dotnetHome, runtimeVersion); - totalOverlaid = publishedOverlay + dotnetHomeOverlay; - - Log.Info($"Build Cache: Overlaid {publishedOverlay} files into published output and " + - $"{dotnetHomeOverlay} files into dotnet home (commit {shortSha})"); + publishedOverlay = BuildCacheClient.OverlayPublishedOutput(buildCacheExtractDir, outputFolder); + Log.Info($"Build Cache: Overlaid {publishedOverlay} files into published output (commit {shortSha})"); } catch (Exception ex) { - job.Error = $"Build Cache: overlay failed: {ex.Message}"; + job.Error = $"Build Cache: published-output overlay failed: {ex.Message}"; return null; } - if (totalOverlaid == 0) + // For self-contained publishes the published output must contain runtime binaries. + // For framework-dependent publishes 0 is acceptable here because the dotnet-home + // overlay above already placed the BCS bits in the shared framework directory. + if (job.SelfContained && publishedOverlay == 0) { - // Silently skipping the overlay would mean the benchmark runs against the FEED runtime - // instead of the BCS commit, which is the worst possible outcome for bisection. Fail. - job.Error = $"Build Cache: overlay copied 0 files for commit {shortSha}. " + - "The archive layout may have changed or the platform is not supported."; + job.Error = $"Build Cache: published-output overlay copied 0 files for self-contained " + + $"commit {shortSha}. The archive layout may have changed or the platform is not supported."; return null; } diff --git a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs index bf2e784e3..e5baac882 100644 --- a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs +++ b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs @@ -341,6 +341,48 @@ public void OverlayDotnetHome_OverlaysSharedFrameworkAndHostFxr() Assert.True(File.Exists(Path.Combine(hostFxr, BuildCacheClient.GetNativeLibName("hostfxr")))); } + [Fact] + public void OverlayDotnetHome_WithCommitSha_RewritesVersionFile() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var (extractDir, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true); + + const string runtimeVersion = "11.0.0-preview.5.26256.117"; + const string commitSha = "603403d9cb49d3d1c35b56bcff024ce99a8c5c3a"; + var dotnetHome = Path.Combine(_testDir, "dotnetHome-version"); + var sharedFw = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); + Directory.CreateDirectory(sharedFw); + + // Simulate dotnet-install having already written a .version file with the FEED commit. + File.WriteAllText(Path.Combine(sharedFw, ".version"), "feedfeedfeed\n" + runtimeVersion + "\n"); + + BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, runtimeVersion, commitSha); + + var versionFileContents = File.ReadAllText(Path.Combine(sharedFw, ".version")); + var lines = versionFileContents.Split('\n', StringSplitOptions.RemoveEmptyEntries); + Assert.Equal(commitSha, lines[0]); + Assert.Equal(runtimeVersion, lines[1]); + } + + [Fact] + public void OverlayDotnetHome_WithoutCommitSha_LeavesVersionFileUntouched() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var (extractDir, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true); + + const string runtimeVersion = "11.0.0-preview.5.26256.117"; + var dotnetHome = Path.Combine(_testDir, "dotnetHome-noversion"); + var sharedFw = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); + Directory.CreateDirectory(sharedFw); + + const string original = "feedfeedfeed\n" + "11.0.0-preview.5.26256.117\n"; + File.WriteAllText(Path.Combine(sharedFw, ".version"), original); + + BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, runtimeVersion); + + Assert.Equal(original, File.ReadAllText(Path.Combine(sharedFw, ".version"))); + } + // ------------------------------------------------------------------- // Fake BCS archive helpers // ------------------------------------------------------------------- From 19f909e47859a3d6f8b184e809964d9219175e85 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 27 May 2026 14:16:58 -0700 Subject: [PATCH 4/7] Address PR review (round 3): isolated dotnet home, BuildKey, SDK-bound apphost, and more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-3 fixes informed by a rubber-duck critique + a second live test. Per-job ISOLATED dotnet home (replaces in-place overlay of shared dotnetHome) The biggest design change: stop mutating the global dotnetHome. After feed install, build a per-job dotnet home by copying the relevant subtrees (shared/Microsoft.NETCore.App/{ver}, shared/Microsoft.AspNetCore.App/{ver}, host/fxr/{ver}, dotnet[.exe]) and overlay BCS bits into the copy. The global dotnetHome stays clean. This fixes three real bugs at once: - cross-job pollution (a later non-buildcache job would have read the BCS commit from .version), - concurrent buildcache jobs racing on the same shared framework dir, - mid-overlay failures leaving a permanently-corrupted shared framework that _installedDotnetRuntimes would continue to trust. The per-job home is exposed via JobContext.BuildCacheDotnetHome and used by StartProcess so framework-dependent jobs actually load BCS bits at runtime. Cleanup happens at job-end. Build cache fields in BuildKeyData Job.BuildKeyData now carries BuildCacheCommitSha / BuildCacheBranch / BuildCacheConfig so reuseBuild + NoBuild don't silently reuse a build pinned to a different BCS commit. SDK-bound apphost is no longer clobbered A live test exposed this regression: the BCS archive ships the raw, unbound apphost. Overlaying it on top of the SDK-bound published binary left the apphost with the placeholder SHA-256 binding and the app refused to start with "This executable is not bound to a managed DLL to execute. The binding value is: 'c3ab8ff1...'". Fixed by NOT overlaying apphost. CoreCLR JIT, GC, managed BCL, hostfxr, hostpolicy still all come from BCS, so the perf relevant code is correct. (Doc comment explains how a future enhancement could rebind a BCS apphost via Microsoft.NET.HostModel.HostWriter.) BCS-config RID for overlay discovery Stop using the host's GetPlatformMoniker for overlay discovery. Resolve the RID from the buildCacheConfig instead so an explicit musl/cross-arch override finds the right runtime pack inside the archive. Numeric-aware managed lib dir selection SelectHighestManagedDir parses net{major}.0 numerically so the BCS archive could ship multiple TFMs without lexically picking net9.0 over net10.0. Non-retryable 404 New BuildCacheNotFoundException sentinel + RetryTransientAsync helper so 404 responses fail fast instead of being retried 3 times. Strict SHA validation ValidateCommitSha requires 8-40 hex chars; rejects "../../../etc/passwd", non-hex, and silly inputs early instead of letting them propagate into URLs and temp paths. Cleanup CleanupExtractDir wipes per-call extraction dirs after overlay; the archive in the parent commit dir is intentionally kept so subsequent jobs for the same commit can skip the download. Per-job dotnet home is also deleted at job end. Executable bits EnsureExecutable preserves +x on native files and the dotnet host on Unix-like systems (File.Copy can drop the bit if the destination didn't have it). Tests (36 BuildCacheClient tests, all passing) New coverage for: - SelectHighestManagedDir picks net11.0 over net10.0 over net9.0, - GetRidForConfig for every supported config, - ValidateCommitSha (accepts/rejects cases incl. path-traversal-like inputs), - CreateBuildCacheDotnetHome: mirrors global, overlays BCS, rewrites .version, and asserts the global home is NOT touched, - Two concurrent CreateBuildCacheDotnetHome invocations produce isolated homes, - OverlayPublishedOutput preserves a pre-existing SDK-bound apphost. Live test result --application.framework net11.0 --application.channel buildcache against a Linux agent: | .NET Runtime Version | 11.0.0-preview.6.26277.104+38d408d22a64 | | Requests/sec | 8,456,046 | Per-job dotnet home created at: /tmp/crank-buildcache/home-38d408d2-coreclr_x64_linux-aaa3385bb0ae4bdca01c7c6b4a7ca2bc Global dotnetHome .version preserved: b520ee7cc01690f70d1431951f554c4e0666a69a (the feed commit, not the BCS commit) — proving the isolation works. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Crank.Agent/BuildCacheClient.cs | 434 +++++++++++++--- src/Microsoft.Crank.Agent/JobContext.cs | 15 + src/Microsoft.Crank.Agent/Startup.cs | 95 ++-- src/Microsoft.Crank.Models/Job.cs | 8 +- .../BuildCacheClientTests.cs | 464 +++++++++++------- 5 files changed, 739 insertions(+), 277 deletions(-) diff --git a/src/Microsoft.Crank.Agent/BuildCacheClient.cs b/src/Microsoft.Crank.Agent/BuildCacheClient.cs index c01e89267..08d35f0ed 100644 --- a/src/Microsoft.Crank.Agent/BuildCacheClient.cs +++ b/src/Microsoft.Crank.Agent/BuildCacheClient.cs @@ -12,6 +12,7 @@ using System.Net.Http; using System.Runtime.InteropServices; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -19,9 +20,9 @@ namespace Microsoft.Crank.Agent { /// /// Lightweight client for the Build Caching Service (BCS) in dotnet-performance-infra. - /// Downloads pre-built runtime artifacts from public Azure Blob Storage and overlays - /// them onto the agent's installed shared framework and/or the published app output so - /// benchmarks run against the BCS runtime instead of the feed-installed one. + /// Downloads pre-built runtime artifacts from public Azure Blob Storage and assembles a + /// per-job dotnet home (or overlays a self-contained published app) so the benchmark runs + /// against the BCS runtime instead of the feed-installed one. /// internal static class BuildCacheClient { @@ -37,20 +38,52 @@ internal static class BuildCacheClient // Per-(commit,config) async locks so concurrent jobs serialize their downloads/extracts. private static readonly ConcurrentDictionary _extractLocks = new(); + // Hex SHA-1 (full or short), 8-40 chars. BCS commits are runtime repo commits. + private static readonly Regex _shaRegex = new("^[0-9a-fA-F]{8,40}$", RegexOptions.Compiled); + /// /// Maps the agent's platform (RID) to the BCS configuration key and artifact filename. + /// The reverse direction (config → RID) is used to decide which RID-shaped subtree to + /// look for inside the archive when the user supplies an explicit config override. /// - internal static readonly IReadOnlyDictionary PlatformToBcsConfig = - new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["linux-x64"] = ("coreclr_x64_linux", "BuildArtifacts_linux_x64_Release_coreclr.tar.gz"), - ["linux-arm64"] = ("coreclr_arm64_linux", "BuildArtifacts_linux_arm64_Release_coreclr.tar.gz"), - ["linux-musl-x64"] = ("coreclr_muslx64_linux", "BuildArtifacts_linux_musl_x64_Release_coreclr.tar.gz"), - ["win-x64"] = ("coreclr_x64_windows", "BuildArtifacts_windows_x64_Release_coreclr.zip"), - ["win-arm64"] = ("coreclr_arm64_windows", "BuildArtifacts_windows_arm64_Release_coreclr.zip"), - ["win-x86"] = ("coreclr_x86_windows", "BuildArtifacts_windows_x86_Release_coreclr.zip"), + internal static readonly IReadOnlyDictionary PlatformToBcsConfig = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["linux-x64"] = ("coreclr_x64_linux", "BuildArtifacts_linux_x64_Release_coreclr.tar.gz", "linux-x64"), + ["linux-arm64"] = ("coreclr_arm64_linux", "BuildArtifacts_linux_arm64_Release_coreclr.tar.gz", "linux-arm64"), + ["linux-musl-x64"] = ("coreclr_muslx64_linux", "BuildArtifacts_linux_musl_x64_Release_coreclr.tar.gz", "linux-musl-x64"), + ["win-x64"] = ("coreclr_x64_windows", "BuildArtifacts_windows_x64_Release_coreclr.zip", "win-x64"), + ["win-arm64"] = ("coreclr_arm64_windows", "BuildArtifacts_windows_arm64_Release_coreclr.zip", "win-arm64"), + ["win-x86"] = ("coreclr_x86_windows", "BuildArtifacts_windows_x86_Release_coreclr.zip", "win-x86"), }; + /// + /// Sentinel thrown for HTTP responses that are definitively not retryable (e.g. 404). + /// Distinguishes "the build doesn't exist" from "transient network blip". + /// + public class BuildCacheNotFoundException : InvalidOperationException + { + public BuildCacheNotFoundException(string message) : base(message) { } + } + + /// + /// Validates a user-supplied commit SHA. Accepts 8-40 lowercase/uppercase hex chars. + /// + internal static void ValidateCommitSha(string commitSha) + { + if (string.IsNullOrEmpty(commitSha)) + { + return; + } + + if (!_shaRegex.IsMatch(commitSha)) + { + throw new ArgumentException( + $"'{commitSha}' is not a valid commit SHA. Expected 8-40 hex characters.", + nameof(commitSha)); + } + } + /// /// Resolves the commit SHA to use from BCS. If a specific commit is provided, returns it /// (after platform-config inference). Otherwise queries latestBuilds.json for the latest @@ -64,6 +97,7 @@ internal static class BuildCacheClient string buildCacheConfig, CancellationToken cancellationToken = default) { + ValidateCommitSha(commitSha); buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); if (string.IsNullOrEmpty(commitSha)) @@ -95,8 +129,8 @@ internal static class BuildCacheClient } /// - /// Downloads and extracts BCS runtime artifacts to a per-job temp directory. The caller - /// is responsible for invoking the overlay methods on the returned directory. + /// Downloads and extracts BCS runtime artifacts to a per-call temp directory. The caller + /// owns the returned directory and is responsible for deleting it when done. /// public static async Task DownloadAndExtractAsync( string baseUrl, @@ -105,6 +139,7 @@ public static async Task DownloadAndExtractAsync( string buildCacheConfig, CancellationToken cancellationToken = default) { + ValidateCommitSha(commitSha); if (string.IsNullOrEmpty(commitSha)) { throw new ArgumentException("commitSha must be provided.", nameof(commitSha)); @@ -123,7 +158,7 @@ public static async Task DownloadAndExtractAsync( var safeConfig = SanitizeForPath(buildCacheConfig); - // Per-(commit,config) lock so two concurrent jobs don't race on the same directory. + // Per-(commit,config) lock so two concurrent jobs don't race on the same archive download. var lockKey = $"{commitSha}|{safeConfig}"; var gate = _extractLocks.GetOrAdd(lockKey, _ => new SemaphoreSlim(1, 1)); await gate.WaitAsync(cancellationToken); @@ -145,6 +180,8 @@ public static async Task DownloadAndExtractAsync( Log.Info($"Build Cache: Using cached archive at {archivePath}"); } + // Per-call unique extract dir so two concurrent jobs for the same (commit,config) + // never delete each other's working tree. var extractDir = Path.Combine(commitDir, $"extracted-{safeConfig}-{Guid.NewGuid():N}"); Directory.CreateDirectory(extractDir); @@ -160,99 +197,196 @@ public static async Task DownloadAndExtractAsync( } /// - /// Overlays BCS runtime binaries (managed + native + host binaries) into a published - /// output directory. Used for self-contained publishes where the runtime is bundled in - /// the publish output. + /// Deletes a previously-extracted directory. Safe to call multiple times. Archives in the + /// parent commit dir are intentionally NOT deleted so subsequent jobs for the same commit + /// can reuse the download. /// - /// Number of files overlaid. - public static int OverlayPublishedOutput(string extractDir, string outputFolder) + public static void CleanupExtractDir(string extractDir) { - var rid = GetPlatformMoniker(); - int filesCopied = 0; + if (string.IsNullOrEmpty(extractDir)) + { + return; + } - var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); - if (nugetPackageDir != null) + try { - var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); - if (Directory.Exists(runtimesDir)) + if (Directory.Exists(extractDir)) { - filesCopied += CopyManaged(runtimesDir, outputFolder); - filesCopied += CopyNative(runtimesDir, outputFolder); + Directory.Delete(extractDir, recursive: true); } } - - var corehostDir = FindCorehostDirectory(extractDir, rid); - if (corehostDir != null) + catch (Exception ex) { - // For self-contained, all three host binaries live alongside the app. - filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetNativeLibName("hostpolicy")); - filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetNativeLibName("hostfxr")); - filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetDotnetExecutableName()); + Log.Info($"Build Cache: Failed to clean up extracted dir '{extractDir}': {ex.Message}"); } - - return filesCopied; } /// - /// Overlays BCS runtime binaries into the agent's installed dotnet home so framework- - /// dependent apps that load the runtime from shared/Microsoft.NETCore.App/{version}/ - /// get the BCS bits at runtime. Also rewrites the .version file so reporting code - /// that reads it (e.g., GetDependencies) picks up the BCS commit instead of the feed commit. + /// Builds a per-job dotnet home that mirrors the relevant subtrees of the global dotnet + /// home (runtime, asp.net, host) and overlays BCS bits on top. The global dotnet home is + /// NOT modified, so concurrent jobs and subsequent non-buildcache jobs are unaffected. /// - /// Number of files overlaid. - public static int OverlayDotnetHome(string extractDir, string dotnetHome, string runtimeVersion, string commitSha = null) + /// Absolute path to the per-job dotnet home root. Caller owns it. + public static string CreateBuildCacheDotnetHome( + string globalDotnetHome, + string extractDir, + string runtimeVersion, + string aspNetCoreVersion, + string commitSha, + string buildCacheConfig) { if (string.IsNullOrEmpty(runtimeVersion)) { throw new ArgumentException("runtimeVersion must be provided.", nameof(runtimeVersion)); } - var rid = GetPlatformMoniker(); - int filesCopied = 0; + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); + var rid = GetRidForConfig(buildCacheConfig); - var sharedFrameworkDir = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); - if (!Directory.Exists(sharedFrameworkDir)) + // Per-job, never reused across jobs to avoid pollution. + var bcsHomeRoot = Path.Combine( + Path.GetTempPath(), + "crank-buildcache", + $"home-{ShortSha(commitSha)}-{SanitizeForPath(buildCacheConfig)}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(bcsHomeRoot); + + // 1. Copy the dotnet host binary. + var dotnetExeName = GetDotnetExecutableName(); + var srcDotnet = Path.Combine(globalDotnetHome, dotnetExeName); + if (File.Exists(srcDotnet)) { - throw new InvalidOperationException( - $"Build Cache: Expected shared framework directory does not exist: '{sharedFrameworkDir}'. " + - "The feed-resolved runtime must be installed before overlaying."); + var dstDotnet = Path.Combine(bcsHomeRoot, dotnetExeName); + File.Copy(srcDotnet, dstDotnet, overwrite: true); + EnsureExecutable(dstDotnet); } + // 2. Mirror host/fxr/{runtimeVersion} (small dir, ~1 file). + var srcHostFxr = Path.Combine(globalDotnetHome, "host", "fxr", runtimeVersion); + var dstHostFxr = Path.Combine(bcsHomeRoot, "host", "fxr", runtimeVersion); + if (Directory.Exists(srcHostFxr)) + { + CopyDirectory(srcHostFxr, dstHostFxr); + } + + // 3. Mirror shared/Microsoft.NETCore.App/{runtimeVersion}. + var srcNetCoreApp = Path.Combine(globalDotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); + var dstNetCoreApp = Path.Combine(bcsHomeRoot, "shared", "Microsoft.NETCore.App", runtimeVersion); + if (Directory.Exists(srcNetCoreApp)) + { + CopyDirectory(srcNetCoreApp, dstNetCoreApp); + } + + // 4. Mirror shared/Microsoft.AspNetCore.App/{aspNetCoreVersion}. + if (!string.IsNullOrEmpty(aspNetCoreVersion)) + { + var srcAspNet = Path.Combine(globalDotnetHome, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); + var dstAspNet = Path.Combine(bcsHomeRoot, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); + if (Directory.Exists(srcAspNet)) + { + CopyDirectory(srcAspNet, dstAspNet); + } + } + + // 5. Overlay BCS managed + native into the per-job NETCore.App. + int filesOverlaid = 0; var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); if (nugetPackageDir != null) { var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); if (Directory.Exists(runtimesDir)) { - filesCopied += CopyManaged(runtimesDir, sharedFrameworkDir); - filesCopied += CopyNative(runtimesDir, sharedFrameworkDir); + filesOverlaid += CopyManaged(runtimesDir, dstNetCoreApp); + filesOverlaid += CopyNative(runtimesDir, dstNetCoreApp); } } + // 6. Overlay BCS host binaries. var corehostDir = FindCorehostDirectory(extractDir, rid); if (corehostDir != null) { - // hostpolicy lives in the shared framework dir. - filesCopied += CopyHostBinaryIfPresent(corehostDir, sharedFrameworkDir, GetNativeLibName("hostpolicy")); + filesOverlaid += CopyHostBinaryIfPresent(corehostDir, dstNetCoreApp, GetNativeLibName("hostpolicy")); - // hostfxr lives at host/fxr/{version}/. - var hostFxrDir = Path.Combine(dotnetHome, "host", "fxr", runtimeVersion); - if (Directory.Exists(hostFxrDir)) + if (Directory.Exists(dstHostFxr)) { - filesCopied += CopyHostBinaryIfPresent(corehostDir, hostFxrDir, GetNativeLibName("hostfxr")); + filesOverlaid += CopyHostBinaryIfPresent(corehostDir, dstHostFxr, GetNativeLibName("hostfxr")); } - // The dotnet host lives at the dotnetHome root. - filesCopied += CopyHostBinaryIfPresent(corehostDir, dotnetHome, GetDotnetExecutableName()); + var dstDotnetHost = Path.Combine(bcsHomeRoot, dotnetExeName); + var copied = CopyHostBinaryIfPresent(corehostDir, bcsHomeRoot, dotnetExeName); + if (copied > 0) + { + EnsureExecutable(dstDotnetHost); + } + filesOverlaid += copied; } - // Rewrite the .version file so anything that reads it (notably the agent's own - // GetDependencies / BenchmarksNetCoreAppVersion measurement) reports the BCS commit - // instead of the feed-installed commit. Format: "\n\n". - if (!string.IsNullOrEmpty(commitSha)) + if (filesOverlaid == 0) { - var versionFile = Path.Combine(sharedFrameworkDir, ".version"); - File.WriteAllText(versionFile, $"{commitSha}\n{runtimeVersion}\n"); + // The per-job home would be just a copy of the feed runtime with no BCS bits. + // Tear it down and let the caller fail the job loudly. + try { Directory.Delete(bcsHomeRoot, recursive: true); } catch { } + throw new InvalidOperationException( + $"Build Cache: overlay copied 0 files for commit {ShortSha(commitSha)} (config '{buildCacheConfig}', rid '{rid}'). " + + "The archive layout may have changed or the platform is not supported."); + } + + // 7. Rewrite .version so any consumer (the agent's own BenchmarksNetCoreAppVersion + // measurement, GetDependencies, etc.) reports the BCS commit. + File.WriteAllText( + Path.Combine(dstNetCoreApp, ".version"), + $"{commitSha}\n{runtimeVersion}\n"); + + Log.Info($"Build Cache: Per-job dotnet home built at {bcsHomeRoot} ({filesOverlaid} BCS files overlaid)"); + return bcsHomeRoot; + } + + /// + /// Overlays BCS runtime binaries (managed + native + apphost) into a self-contained + /// published output directory. For SCD the runtime ships next to the app, so this is the + /// only way to make the benchmark actually run BCS bits. The BCS apphost is renamed to + /// match the published app's executable name (the SDK renames apphost → AssemblyName). + /// + /// Number of files overlaid. + public static int OverlayPublishedOutput( + string extractDir, + string outputFolder, + string buildCacheConfig, + string assemblyName) + { + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); + var rid = GetRidForConfig(buildCacheConfig); + int filesCopied = 0; + + var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); + if (nugetPackageDir != null) + { + var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); + if (Directory.Exists(runtimesDir)) + { + filesCopied += CopyManaged(runtimesDir, outputFolder); + filesCopied += CopyNative(runtimesDir, outputFolder); + } + } + + var corehostDir = FindCorehostDirectory(extractDir, rid); + if (corehostDir != null) + { + filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetNativeLibName("hostpolicy")); + + // Intentionally NOT replacing the SDK-bound apphost. The BCS archive ships the raw, + // unbound apphost (the binary has a placeholder SHA-256 hash where the managed DLL + // path is encoded). The SDK's publish step normally invokes HostWriter.CreateAppHost + // to bake the managed entry-point path into that binary. Overlaying the raw BCS + // apphost on top of the SDK-bound one leaves the executable unable to locate its + // managed DLL and the app fails to start with: + // + // "This executable is not bound to a managed DLL to execute. The binding value + // is: ''" + // + // The perf-relevant runtime code (CoreCLR JIT, GC, managed BCL, hostfxr, hostpolicy) + // is still overlaid above. To overlay apphost as well, BCS would need to ship a + // pre-bound apphost per project, or the agent would need to invoke the apphost + // binder against the BCS apphost using the published app's binding metadata. } return filesCopied; @@ -280,13 +414,15 @@ private static async Task GetLatestBuildsAsync( Log.Info($"Build Cache: Fetching latest builds from {url}"); string json = null; - await ProcessUtil.RetryOnExceptionAsync(DownloadRetryCount, async () => + + // 404s are not transient; pre-check before entering the retry loop. + await RetryTransientAsync(DownloadRetryCount, async () => { using var response = await _httpClient.GetAsync(url, cancellationToken); if (response.StatusCode == System.Net.HttpStatusCode.NotFound) { - throw new InvalidOperationException( + throw new BuildCacheNotFoundException( $"Build Cache: No latest builds found for branch '{branch}' in repo '{repoName}'. URL: {url}"); } @@ -303,7 +439,7 @@ private static async Task DownloadWithRetryAsync(string url, string destination, { var partial = destination + ".partial"; - await ProcessUtil.RetryOnExceptionAsync(DownloadRetryCount, async () => + await RetryTransientAsync(DownloadRetryCount, async () => { if (File.Exists(partial)) { @@ -314,8 +450,8 @@ await ProcessUtil.RetryOnExceptionAsync(DownloadRetryCount, async () => { if (response.StatusCode == System.Net.HttpStatusCode.NotFound) { - // Don't retry 404s; they aren't transient. - throw new InvalidOperationException( + // Definitively not transient; do not retry. + throw new BuildCacheNotFoundException( $"Build Cache: Artifact not found at {url}. The build may not exist in the cache."); } @@ -348,6 +484,38 @@ await ProcessUtil.RetryOnExceptionAsync(DownloadRetryCount, async () => }); } + /// + /// Like + /// but rethrows immediately without retrying. + /// + private static async Task RetryTransientAsync(int retries, Func operation) + { + var attempts = 0; + while (true) + { + try + { + attempts++; + await operation(); + return; + } + catch (BuildCacheNotFoundException) + { + // Non-retryable: fail fast. + throw; + } + catch (Exception ex) + { + if (attempts > retries) + { + throw; + } + + Log.Info($"Build Cache: Attempt {attempts} failed: {ex.Message}"); + } + } + } + /// /// Parses the latestBuilds.json format from BCS. The JSON has dynamic keys for each /// build configuration plus a "branch_name" / "BranchName" string property. @@ -426,19 +594,49 @@ private static async Task ExtractTarGzAsync(string archivePath, string outputDir // --- Overlay helpers ---------------------------------------------------------- - private static int CopyManaged(string runtimesDir, string destinationDir) + /// + /// Selects the highest net{X}.0 directory under lib/ using a numeric comparison. + /// Lexicographic ordering puts "net9.0" above "net10.0", which would silently overlay + /// the wrong managed assemblies if BCS ever ships multiple TFMs. + /// + internal static string SelectHighestManagedDir(string libDir) { - int copied = 0; - var libDir = Path.Combine(runtimesDir, "lib"); if (!Directory.Exists(libDir)) { - return 0; + return null; + } + + (int major, int minor, string path) Parse(string dir) + { + var name = Path.GetFileName(dir); + if (name.StartsWith("net", StringComparison.OrdinalIgnoreCase)) + { + var rest = name.Substring(3); + var dot = rest.IndexOf('.'); + if (dot > 0 && + int.TryParse(rest.Substring(0, dot), out var maj) && + int.TryParse(rest.Substring(dot + 1), out var min)) + { + return (maj, min, dir); + } + } + + return (-1, -1, dir); } - // Pick the highest-versioned net{X}.0 directory (the archive should only ship one). - var managedDir = Directory.GetDirectories(libDir) - .OrderByDescending(d => Path.GetFileName(d), StringComparer.OrdinalIgnoreCase) + return Directory.GetDirectories(libDir) + .Select(Parse) + .OrderByDescending(t => t.major) + .ThenByDescending(t => t.minor) + .Select(t => t.path) .FirstOrDefault(); + } + + private static int CopyManaged(string runtimesDir, string destinationDir) + { + int copied = 0; + var libDir = Path.Combine(runtimesDir, "lib"); + var managedDir = SelectHighestManagedDir(libDir); if (managedDir == null) { @@ -479,6 +677,7 @@ private static int CopyNative(string runtimesDir, string destinationDir) var dest = Path.Combine(destinationDir, fileName); File.Copy(file, dest, overwrite: true); + EnsureExecutable(dest); copied++; } @@ -487,17 +686,71 @@ private static int CopyNative(string runtimesDir, string destinationDir) private static int CopyHostBinaryIfPresent(string sourceDir, string destDir, string fileName) { - var src = Path.Combine(sourceDir, fileName); - if (!File.Exists(src)) + var sourcePath = Path.Combine(sourceDir, fileName); + if (!File.Exists(sourcePath)) { return 0; } Directory.CreateDirectory(destDir); - File.Copy(src, Path.Combine(destDir, fileName), overwrite: true); + var destPath = Path.Combine(destDir, fileName); + File.Copy(sourcePath, destPath, overwrite: true); + EnsureExecutable(destPath); return 1; } + /// + /// Recursively copies a directory tree. Used to materialize the per-job dotnet home + /// from the global feed-installed one. Native files and the dotnet host are chmod'd + /// executable on Unix-like systems. + /// + internal static void CopyDirectory(string sourceDir, string destDir) + { + Directory.CreateDirectory(destDir); + + foreach (var file in Directory.EnumerateFiles(sourceDir)) + { + var destFile = Path.Combine(destDir, Path.GetFileName(file)); + File.Copy(file, destFile, overwrite: true); + EnsureExecutable(destFile); + } + + foreach (var subDir in Directory.EnumerateDirectories(sourceDir)) + { + CopyDirectory(subDir, Path.Combine(destDir, Path.GetFileName(subDir))); + } + } + + /// + /// On Unix-like systems, ensures the destination file has the user-execute bit set. + /// Native libs don't strictly require +x, but the dotnet host and apphost do, and the + /// File.Copy + overwrite path can drop the bit if the destination didn't have it. + /// + private static void EnsureExecutable(string path) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return; + } + + try + { + var current = File.GetUnixFileMode(path); + var withExec = current + | UnixFileMode.UserExecute + | UnixFileMode.GroupExecute + | UnixFileMode.OtherExecute; + if (current != withExec) + { + File.SetUnixFileMode(path, withExec); + } + } + catch + { + // Best-effort; some filesystems (FAT, network shares) don't support mode bits. + } + } + private static string FindDirectory(string root, string directoryName) { if (!Directory.Exists(root)) @@ -566,6 +819,25 @@ private static string GetArtifactFile(string buildCacheConfig) return match.artifactFile; } + /// + /// Maps a BCS config key back to its RID. Use this for overlay path discovery so an + /// explicit musl/cross-arch override actually finds the right runtime pack inside the + /// archive instead of falling back to the host's detected RID. + /// + internal static string GetRidForConfig(string buildCacheConfig) + { + var match = PlatformToBcsConfig.Values.FirstOrDefault(v => + string.Equals(v.configKey, buildCacheConfig, StringComparison.OrdinalIgnoreCase)); + + if (match.rid == null) + { + throw new InvalidOperationException( + $"Unknown Build Cache configuration key: '{buildCacheConfig}'."); + } + + return match.rid; + } + internal static string GetNativeLibName(string baseName) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) diff --git a/src/Microsoft.Crank.Agent/JobContext.cs b/src/Microsoft.Crank.Agent/JobContext.cs index a358b0e42..6959636bb 100644 --- a/src/Microsoft.Crank.Agent/JobContext.cs +++ b/src/Microsoft.Crank.Agent/JobContext.cs @@ -32,6 +32,21 @@ public class JobContext public string DockerImage { get; set; } public string DockerContainerId { get; set; } + /// + /// Per-job isolated dotnet home used for buildcache runs. When non-null, the agent should + /// run the application using this root (so the BCS-overlaid runtime is loaded), and the + /// global dotnet home is left untouched for concurrent jobs. The directory is owned by + /// the job and deleted when the job completes. + /// + public string BuildCacheDotnetHome { get; set; } + + /// + /// Temporary directory holding the extracted BCS archive for this job. Deleted at the + /// end of the job; the underlying archive in the parent commit directory is kept so + /// subsequent jobs for the same commit can reuse it without re-downloading. + /// + public string BuildCacheExtractDir { get; set; } + public ulong EventPipeSessionId { get; set; } public Task EventPipeTask { get; set; } public bool EventPipeTerminated { get; set; } diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index ac904b1e5..be2dbf116 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -1039,7 +1039,7 @@ private static async Task ProcessJobs(string hostname, string dockerHostname, Ca { buildAndRunTask = Task.Run(async () => { - benchmarksDir = await CloneRestoreAndBuild(tempDir, job, _dotnethome, cts.Token); + benchmarksDir = await CloneRestoreAndBuild(tempDir, job, _dotnethome, context, cts.Token); if (benchmarksDir == null) { @@ -1051,7 +1051,10 @@ private static async Task ProcessJobs(string hostname, string dockerHostname, Ca { try { - process = await StartProcess(hostname, Path.Combine(tempDir, benchmarksDir), job, _dotnethome, context); + // For buildcache jobs the per-job isolated dotnet home holds the + // BCS-overlaid runtime; run against it instead of the global home. + var runtimeDotnetHome = context.BuildCacheDotnetHome ?? _dotnethome; + process = await StartProcess(hostname, Path.Combine(tempDir, benchmarksDir), job, runtimeDotnetHome, context); Log.Info($"Process started: {job.ProcessId}"); @@ -1934,6 +1937,22 @@ async Task DeleteJobAsync() await TryDeleteDirAsync(tempDir); } + // Build Cache: clean up per-job extracted artifacts and the isolated + // dotnet home so concurrent / future jobs do not see stale state and + // /tmp does not accumulate multi-GB extracts. + if (_cleanup && !job.NoClean) + { + BuildCacheClient.CleanupExtractDir(context.BuildCacheExtractDir); + + if (!string.IsNullOrEmpty(context.BuildCacheDotnetHome)) + { + await TryDeleteDirAsync(context.BuildCacheDotnetHome); + } + + context.BuildCacheExtractDir = null; + context.BuildCacheDotnetHome = null; + } + // Delete temporary attachment files // NB: Attachments are already deleted once they are copied, unless the job fails // to reach that point. @@ -2856,7 +2875,7 @@ private static async Task DockerCleanUpAsync(string containerId, string imageNam } } - private static async Task CloneRestoreAndBuild(string path, Job job, string dotnetHome, CancellationToken cancellationToken = default) + private static async Task CloneRestoreAndBuild(string path, Job job, string dotnetHome, JobContext jobContext = null, CancellationToken cancellationToken = default) { var reuseFolder = await RetrieveSourcesAsync(job, path); @@ -3290,28 +3309,40 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => var dotnetDir = dotnetHome; - // Build Cache: overlay BCS bits into the freshly-installed shared framework BEFORE any - // metadata capture below reads the .version file. Doing it here means the Microsoft.NETCore.App - // metadata measurement records the BCS commit and the agent's GetDependencies pass picks up - // the BCS-built assemblies' AssemblyInformationalVersion. The published-output overlay still - // happens after publish (only that folder exists by then). + // Build Cache: build a per-job dotnet home that contains BCS-overlaid runtime + asp.net + // + host. We DO NOT mutate the global dotnet home — concurrent jobs and subsequent + // non-buildcache jobs must remain unaffected. The publish step continues to use the + // global dotnetDir (it has the SDK); only the runtime-resolution paths (metadata + // reading, crossgen/symbols emit, StartProcess) point at the per-job home. + var runtimeHomeDir = dotnetDir; + if (useBuildCache && buildCacheExtractDir != null) { try { - var dotnetHomeOverlay = BuildCacheClient.OverlayDotnetHome( - buildCacheExtractDir, dotnetDir, runtimeVersion, buildCacheCommitSha); - Log.Info($"Build Cache: Overlaid {dotnetHomeOverlay} files into dotnet home (commit {BuildCacheClient.ShortSha(buildCacheCommitSha)})"); + var bcsHome = BuildCacheClient.CreateBuildCacheDotnetHome( + dotnetDir, + buildCacheExtractDir, + runtimeVersion, + aspNetCoreVersion, + buildCacheCommitSha, + job.BuildCacheConfig); + + runtimeHomeDir = bcsHome; - if (dotnetHomeOverlay == 0) + // Stash on the JobContext so StartProcess uses this isolated home (FDD) and + // so the cleanup pass at job end can delete it. + if (jobContext != null) { - job.Error = $"Build Cache: dotnet-home overlay copied 0 files for commit {buildCacheCommitSha}."; - return null; + jobContext.BuildCacheDotnetHome = bcsHome; + jobContext.BuildCacheExtractDir = buildCacheExtractDir; } + + Log.Info($"Build Cache: Isolated dotnet home: {bcsHome}"); } catch (Exception ex) { - job.Error = $"Build Cache: dotnet-home overlay failed: {ex.Message}"; + job.Error = $"Build Cache: failed to build isolated dotnet home: {ex.Message}"; return null; } } @@ -3349,7 +3380,7 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => { try { - var aspNetCoreVersionFileName = Path.Combine(dotnetDir, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion, ".version"); + var aspNetCoreVersionFileName = Path.Combine(runtimeHomeDir, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion, ".version"); (_, var aspnetCoreCommitHash) = await ParseLatestVersionFile(aspNetCoreVersionFileName); job.Metadata.Enqueue(new MeasurementMetadata @@ -3382,7 +3413,7 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => { try { - var netCoreAppVersionFileName = Path.Combine(dotnetDir, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version"); + var netCoreAppVersionFileName = Path.Combine(runtimeHomeDir, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version"); (_, var netCoreAppCommitHash) = await ParseLatestVersionFile(netCoreAppVersionFileName); job.Metadata.Enqueue(new MeasurementMetadata @@ -3544,11 +3575,11 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => Log.Info($"Application published successfully in {job.BuildTime.TotalMilliseconds} ms"); - // Build Cache: overlay BCS runtime binaries onto the just-published app. The agent's - // installed shared framework was already overlaid earlier (right after install) so the - // .NET runtime metadata and FDD execution see BCS bits; here we cover the SCD case where - // the runtime ships in the publish output. PatchRuntimeConfig still runs with the - // feed-resolved runtimeVersion so runtimeconfig.json points to a real installed dir. + // Build Cache: overlay BCS runtime binaries onto the just-published app. The + // per-job dotnet home was built earlier (used for FDD execution + metadata); + // here we cover the SCD case where the runtime ships in the publish output. + // PatchRuntimeConfig still runs with the feed-resolved runtimeVersion so + // runtimeconfig.json points to a real installed shared-framework dir. if (useBuildCache && buildCacheExtractDir != null) { var shortSha = BuildCacheClient.ShortSha(buildCacheCommitSha); @@ -3556,7 +3587,15 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => int publishedOverlay; try { - publishedOverlay = BuildCacheClient.OverlayPublishedOutput(buildCacheExtractDir, outputFolder); + var publishProjectFileName = Path.Combine(benchmarkedApp, FormatPathSeparators(job.Project)); + var assemblyName = GetAssemblyName(job, publishProjectFileName); + + publishedOverlay = BuildCacheClient.OverlayPublishedOutput( + buildCacheExtractDir, + outputFolder, + job.BuildCacheConfig, + assemblyName); + Log.Info($"Build Cache: Overlaid {publishedOverlay} files into published output (commit {shortSha})"); } catch (Exception ex) @@ -3565,9 +3604,9 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => return null; } - // For self-contained publishes the published output must contain runtime binaries. - // For framework-dependent publishes 0 is acceptable here because the dotnet-home - // overlay above already placed the BCS bits in the shared framework directory. + // For self-contained publishes the published output must contain runtime binaries + // (managed + native + apphost). For framework-dependent publishes 0 is acceptable + // here because the per-job dotnet home already provides BCS bits at runtime. if (job.SelfContained && publishedOverlay == 0) { job.Error = $"Build Cache: published-output overlay copied 0 files for self-contained " + @@ -3678,7 +3717,7 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => { var crossgenFolder = job.SelfContained ? outputFolder - : Path.Combine(dotnetDir, "shared", "Microsoft.NETCore.App", runtimeVersion) + : Path.Combine(runtimeHomeDir, "shared", "Microsoft.NETCore.App", runtimeVersion) ; var crossgenFilename = Path.Combine(crossgenFolder, "crossgen"); @@ -3706,7 +3745,7 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => var symbolsFolder = job.SelfContained ? outputFolder - : Path.Combine(dotnetDir, "shared", "Microsoft.NETCore.App", runtimeVersion) + : Path.Combine(runtimeHomeDir, "shared", "Microsoft.NETCore.App", runtimeVersion) ; // dotnet symbol --symbols --output mySymbols /usr/share/dotnet/shared/Microsoft.NETCore.App/2.1.0/lib*.so diff --git a/src/Microsoft.Crank.Models/Job.cs b/src/Microsoft.Crank.Models/Job.cs index 299387380..ee1537cb4 100644 --- a/src/Microsoft.Crank.Models/Job.cs +++ b/src/Microsoft.Crank.Models/Job.cs @@ -404,7 +404,10 @@ public BuildKeyData GetBuildKeyData() DockerPull = DockerPull, DockerFile = DockerFile, DockerImageName = DockerImageName, - DockerContextDirectory = DockerContextDirectory + DockerContextDirectory = DockerContextDirectory, + BuildCacheCommitSha = BuildCacheCommitSha, + BuildCacheBranch = BuildCacheBranch, + BuildCacheConfig = BuildCacheConfig }; } @@ -513,5 +516,8 @@ public class BuildKeyData public string DockerFile { get; set; } public string DockerImageName { get; set; } public string DockerContextDirectory { get; set; } + public string BuildCacheCommitSha { get; set; } + public string BuildCacheBranch { get; set; } + public string BuildCacheConfig { get; set; } } } diff --git a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs index e5baac882..21cfe42bd 100644 --- a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs +++ b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -59,9 +60,7 @@ public void ParseLatestBuilds_PascalCase_ParsesCommitShaAndTime() var result = BuildCacheClient.ParseLatestBuilds(json); Assert.Equal("main", result.BranchName); - Assert.True(result.Entries.ContainsKey("coreclr_x64_linux")); Assert.Equal("abc123def456", result.Entries["coreclr_x64_linux"].CommitSha); - Assert.Equal("2025-01-01T00:00:00Z", result.Entries["coreclr_x64_linux"].CommitTime); } [Fact] @@ -80,9 +79,7 @@ public void ParseLatestBuilds_SnakeCase_ParsesCommitShaAndTime() var result = BuildCacheClient.ParseLatestBuilds(json); Assert.Equal("release/10.0", result.BranchName); - Assert.True(result.Entries.ContainsKey("coreclr_arm64_linux")); Assert.Equal("deadbeef", result.Entries["coreclr_arm64_linux"].CommitSha); - Assert.Equal("2025-02-02T00:00:00Z", result.Entries["coreclr_arm64_linux"].CommitTime); } [Fact] @@ -104,58 +101,74 @@ public void ParseLatestBuilds_MixedCasing_ParsesAllConfigs() } [Fact] - public void ParseLatestBuilds_MissingFields_ReturnsNullsWithoutThrowing() + public void ParseLatestBuilds_NonObjectValues_AreSkipped() { const string json = """ { "branch_name": "main", - "coreclr_x64_linux": { "CommitSha": "abc" }, - "empty_config": {} + "schemaVersion": 2, + "lastUpdated": "2025-01-01", + "coreclr_x64_linux": { "CommitSha": "abc" } } """; var result = BuildCacheClient.ParseLatestBuilds(json); - Assert.Equal("abc", result.Entries["coreclr_x64_linux"].CommitSha); - Assert.Null(result.Entries["coreclr_x64_linux"].CommitTime); - Assert.Null(result.Entries["empty_config"].CommitSha); - Assert.Null(result.Entries["empty_config"].CommitTime); + Assert.Single(result.Entries); + Assert.True(result.Entries.ContainsKey("coreclr_x64_linux")); } - [Fact] - public void ParseLatestBuilds_EntriesLookupIsCaseInsensitive() - { - const string json = """ - { "branch_name": "main", "CoreCLR_X64_Linux": { "CommitSha": "abc" } } - """; + // ------------------------------------------------------------------- + // ValidateCommitSha + // ------------------------------------------------------------------- - var result = BuildCacheClient.ParseLatestBuilds(json); + [Theory] + [InlineData("")] + [InlineData(null)] + [InlineData("abcdef12")] // min length + [InlineData("ABCDEF12")] // upper hex + [InlineData("603403d9cb49d3d1c35b56bcff024ce99a8c5c3a")] // full 40 + public void ValidateCommitSha_AcceptsValid(string sha) + { + BuildCacheClient.ValidateCommitSha(sha); + } - Assert.True(result.Entries.ContainsKey("coreclr_x64_linux")); - Assert.True(result.Entries.ContainsKey("CORECLR_X64_LINUX")); + [Theory] + [InlineData("abc")] // too short + [InlineData("ghijklmn")] // non-hex + [InlineData("abcd 1234")] // contains space + [InlineData("../../../etc/passwd")] // path traversal attempt + [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")] // 41 chars, too long + public void ValidateCommitSha_RejectsInvalid(string sha) + { + Assert.Throws(() => BuildCacheClient.ValidateCommitSha(sha)); } + // ------------------------------------------------------------------- + // ShortSha + // ------------------------------------------------------------------- + [Fact] - public void ParseLatestBuilds_NonObjectValues_AreSkipped() + public void ShortSha_LongInput_ReturnsFirstEight() { - // Real-world payloads sometimes carry non-object metadata that must be ignored. - const string json = """ - { - "branch_name": "main", - "schemaVersion": 2, - "lastUpdated": "2025-01-01", - "coreclr_x64_linux": { "CommitSha": "abc" } - } - """; + Assert.Equal("abcdef12", BuildCacheClient.ShortSha("abcdef1234567890")); + } - var result = BuildCacheClient.ParseLatestBuilds(json); + [Fact] + public void ShortSha_ShortInput_ReturnsAsIs() + { + Assert.Equal("abc", BuildCacheClient.ShortSha("abc")); + } - Assert.Single(result.Entries); - Assert.True(result.Entries.ContainsKey("coreclr_x64_linux")); + [Fact] + public void ShortSha_NullOrEmpty_ReturnsEmpty() + { + Assert.Equal(string.Empty, BuildCacheClient.ShortSha(null)); + Assert.Equal(string.Empty, BuildCacheClient.ShortSha("")); } // ------------------------------------------------------------------- - // GetPlatformMoniker + // Platform / RID mapping // ------------------------------------------------------------------- [Fact] @@ -173,238 +186,337 @@ public void GetPlatformMoniker_ReturnsKnownRid() Assert.Contains(rid, validRids); } + [Theory] + [InlineData("coreclr_x64_linux", "linux-x64")] + [InlineData("coreclr_arm64_linux", "linux-arm64")] + [InlineData("coreclr_muslx64_linux", "linux-musl-x64")] + [InlineData("coreclr_x64_windows", "win-x64")] + [InlineData("coreclr_arm64_windows", "win-arm64")] + [InlineData("coreclr_x86_windows", "win-x86")] + public void GetRidForConfig_ReturnsMatchingRid(string configKey, string expectedRid) + { + Assert.Equal(expectedRid, BuildCacheClient.GetRidForConfig(configKey)); + } + [Fact] - public void PlatformToBcsConfig_ContainsAllSupportedRids() + public void GetRidForConfig_UnknownConfig_Throws() { - // Sanity: agents typically run on these RIDs; ensure the table covers them. - Assert.True(BuildCacheClient.PlatformToBcsConfig.ContainsKey("linux-x64")); - Assert.True(BuildCacheClient.PlatformToBcsConfig.ContainsKey("linux-arm64")); - Assert.True(BuildCacheClient.PlatformToBcsConfig.ContainsKey("win-x64")); + Assert.Throws(() => BuildCacheClient.GetRidForConfig("totally_unknown")); } // ------------------------------------------------------------------- - // ShortSha + // SelectHighestManagedDir (numeric-aware) // ------------------------------------------------------------------- [Fact] - public void ShortSha_LongInput_ReturnsFirstEight() + public void SelectHighestManagedDir_NumericOrderNotLexicographic() { - Assert.Equal("abcdef12", BuildCacheClient.ShortSha("abcdef1234567890")); - } + var libDir = Path.Combine(_testDir, "lib"); + Directory.CreateDirectory(Path.Combine(libDir, "net8.0")); + Directory.CreateDirectory(Path.Combine(libDir, "net9.0")); + Directory.CreateDirectory(Path.Combine(libDir, "net10.0")); + Directory.CreateDirectory(Path.Combine(libDir, "net11.0")); - [Fact] - public void ShortSha_ShortInput_ReturnsAsIs() - { - Assert.Equal("abc", BuildCacheClient.ShortSha("abc")); + // Lexicographic: net9.0 > net8.0 > net11.0 > net10.0 (wrong). + // Numeric: net11.0 > net10.0 > net9.0 > net8.0 (correct). + var selected = BuildCacheClient.SelectHighestManagedDir(libDir); + + Assert.Equal("net11.0", Path.GetFileName(selected)); } [Fact] - public void ShortSha_NullOrEmpty_ReturnsEmpty() + public void SelectHighestManagedDir_NoDirs_ReturnsNull() { - Assert.Equal(string.Empty, BuildCacheClient.ShortSha(null)); - Assert.Equal(string.Empty, BuildCacheClient.ShortSha("")); - } + var libDir = Path.Combine(_testDir, "empty-lib"); + Directory.CreateDirectory(libDir); - // ------------------------------------------------------------------- - // GetNativeLibName - // ------------------------------------------------------------------- + Assert.Null(BuildCacheClient.SelectHighestManagedDir(libDir)); + } [Fact] - public void GetNativeLibName_MatchesHostPlatform() + public void SelectHighestManagedDir_MissingDir_ReturnsNull() { - var name = BuildCacheClient.GetNativeLibName("hostpolicy"); - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - Assert.Equal("hostpolicy.dll", name); - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - Assert.Equal("libhostpolicy.dylib", name); - } - else - { - Assert.Equal("libhostpolicy.so", name); - } + Assert.Null(BuildCacheClient.SelectHighestManagedDir(Path.Combine(_testDir, "does-not-exist"))); } // ------------------------------------------------------------------- - // OverlayPublishedOutput / OverlayDotnetHome + // OverlayPublishedOutput // ------------------------------------------------------------------- [Fact] - public void OverlayPublishedOutput_CopiesAllRuntimeFilesUnconditionally() + public void OverlayPublishedOutput_CopiesRuntimeFilesAndHostpolicyButNotApphost() { - // Build a fake BCS extract layout for the host RID. + // The BCS archive ships an unbound apphost (the SDK normally binds the published + // managed DLL path into the executable during publish). Overlaying the raw BCS apphost + // on top of the SDK-bound one breaks the published app, so we deliberately skip it. var rid = BuildCacheClient.GetPlatformMoniker(); - var (extractDir, _, expectedManagedNames, expectedNativeNames) = BuildFakeBcsArchive(rid, includeHost: true); + var configKey = ConfigKeyForRid(rid); + var (extractDir, _, managed, native) = BuildFakeBcsArchive(rid, includeHost: true, includeApphost: true); var outputFolder = Path.Combine(_testDir, "published"); Directory.CreateDirectory(outputFolder); - // Note: outputFolder is intentionally EMPTY — the overlay must still copy - // managed/native runtime files (regression: earlier behavior skipped missing dest). - var copied = BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder); + // Pre-existing SDK-bound apphost that must NOT be overwritten. + var apphostName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "MyApp.exe" : "MyApp"; + File.WriteAllText(Path.Combine(outputFolder, apphostName), "SDK_BOUND_APPHOST"); + + var copied = BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder, configKey, "MyApp"); - Assert.True(copied >= expectedManagedNames.Count + expectedNativeNames.Count, - $"Expected at least {expectedManagedNames.Count + expectedNativeNames.Count} files; got {copied}"); + // managed + native + hostpolicy (no apphost contribution) + Assert.True(copied >= managed.Count + native.Count + 1); - foreach (var dll in expectedManagedNames) + foreach (var dll in managed) { Assert.True(File.Exists(Path.Combine(outputFolder, dll)), $"Missing managed file {dll}"); } - foreach (var native in expectedNativeNames) + foreach (var n in native) { - Assert.True(File.Exists(Path.Combine(outputFolder, native)), $"Missing native file {native}"); + Assert.True(File.Exists(Path.Combine(outputFolder, n)), $"Missing native file {n}"); } - // hostpolicy must have been copied alongside the app for self-contained. Assert.True(File.Exists(Path.Combine(outputFolder, BuildCacheClient.GetNativeLibName("hostpolicy")))); + + // SDK-bound apphost preserved. + Assert.Equal("SDK_BOUND_APPHOST", File.ReadAllText(Path.Combine(outputFolder, apphostName))); } [Fact] - public void OverlayPublishedOutput_NoMatchingPlatformLayout_ReturnsZero() + public void OverlayPublishedOutput_EmptyExtract_ReturnsZero() { - // Empty extract directory ⇒ overlay finds nothing ⇒ returns 0 (caller fails the job). - var extractDir = Path.Combine(_testDir, "empty-extract"); + var rid = BuildCacheClient.GetPlatformMoniker(); + var configKey = ConfigKeyForRid(rid); + + var extractDir = Path.Combine(_testDir, "empty"); Directory.CreateDirectory(extractDir); - var outputFolder = Path.Combine(_testDir, "published"); + var outputFolder = Path.Combine(_testDir, "output"); Directory.CreateDirectory(outputFolder); - var copied = BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder); - + var copied = BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder, configKey, "MyApp"); Assert.Equal(0, copied); } [Fact] - public void OverlayPublishedOutput_SkipsPdbAndDbgFiles() + public void OverlayPublishedOutput_SkipsPdbAndDbg() { var rid = BuildCacheClient.GetPlatformMoniker(); - var (extractDir, runtimesDir, _, _) = BuildFakeBcsArchive(rid, includeHost: false); + var configKey = ConfigKeyForRid(rid); + var (extractDir, runtimesDir, _, _) = BuildFakeBcsArchive(rid, includeHost: false, includeApphost: false); - // Add a .pdb and .dbg in the native dir. var nativeDir = Path.Combine(runtimesDir, "native"); File.WriteAllText(Path.Combine(nativeDir, "coreclr.pdb"), "pdb"); File.WriteAllText(Path.Combine(nativeDir, "libcoreclr.dbg"), "dbg"); - var outputFolder = Path.Combine(_testDir, "published"); + var outputFolder = Path.Combine(_testDir, "published-pdb"); Directory.CreateDirectory(outputFolder); - BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder); + BuildCacheClient.OverlayPublishedOutput(extractDir, outputFolder, configKey, "MyApp"); Assert.False(File.Exists(Path.Combine(outputFolder, "coreclr.pdb"))); Assert.False(File.Exists(Path.Combine(outputFolder, "libcoreclr.dbg"))); } + // ------------------------------------------------------------------- + // CreateBuildCacheDotnetHome — the heart of round 3 + // ------------------------------------------------------------------- + [Fact] - public void OverlayDotnetHome_RequiresExistingSharedFrameworkDir() + public void CreateBuildCacheDotnetHome_MirrorsGlobalAndOverlaysBcs() { var rid = BuildCacheClient.GetPlatformMoniker(); - var (extractDir, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true); + var configKey = ConfigKeyForRid(rid); + var (extractDir, _, managed, native) = BuildFakeBcsArchive(rid, includeHost: true, includeApphost: false); - var dotnetHome = Path.Combine(_testDir, "dotnetHome-missing"); - Directory.CreateDirectory(dotnetHome); + const string runtimeVersion = "11.0.0-preview.5.26256.117"; + const string aspNetCoreVersion = "11.0.0-preview.5.26256.117"; + var globalHome = BuildFakeGlobalDotnetHome(runtimeVersion, aspNetCoreVersion); + var commitSha = "603403d9cb49d3d1c35b56bcff024ce99a8c5c3a"; - // shared/Microsoft.NETCore.App/{version} does NOT exist ⇒ should throw with a clear message. - var ex = Assert.Throws( - () => BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, "10.0.0-preview.1")); + var bcsHome = BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir, runtimeVersion, aspNetCoreVersion, commitSha, configKey); - Assert.Contains("shared framework", ex.Message, StringComparison.OrdinalIgnoreCase); - } + try + { + // 1. Global dotnet home must NOT be touched (no cross-job pollution). + var globalVersion = File.ReadAllText(Path.Combine(globalHome, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version")); + Assert.Contains("FEED_COMMIT", globalVersion); + Assert.DoesNotContain(commitSha, globalVersion); - [Fact] - public void OverlayDotnetHome_OverlaysSharedFrameworkAndHostFxr() - { - var rid = BuildCacheClient.GetPlatformMoniker(); - var (extractDir, _, expectedManagedNames, expectedNativeNames) = BuildFakeBcsArchive(rid, includeHost: true); + // 2. Per-job home exists with BCS overlay applied. + Assert.True(Directory.Exists(bcsHome)); + var bcsNetCoreApp = Path.Combine(bcsHome, "shared", "Microsoft.NETCore.App", runtimeVersion); - const string runtimeVersion = "10.0.0-preview.1"; - var dotnetHome = Path.Combine(_testDir, "dotnetHome"); - var sharedFw = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); - var hostFxr = Path.Combine(dotnetHome, "host", "fxr", runtimeVersion); - Directory.CreateDirectory(sharedFw); - Directory.CreateDirectory(hostFxr); + foreach (var dll in managed) + { + Assert.True(File.Exists(Path.Combine(bcsNetCoreApp, dll)), $"Missing BCS managed {dll}"); + } + foreach (var n in native) + { + Assert.True(File.Exists(Path.Combine(bcsNetCoreApp, n)), $"Missing BCS native {n}"); + } - var copied = BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, runtimeVersion); + // 3. .version was rewritten with BCS commit. + var bcsVersion = File.ReadAllText(Path.Combine(bcsNetCoreApp, ".version")); + Assert.Contains(commitSha, bcsVersion); - Assert.True(copied > 0); - foreach (var dll in expectedManagedNames) - { - Assert.True(File.Exists(Path.Combine(sharedFw, dll)), $"Missing managed in shared FW: {dll}"); + // 4. ASP.NET Core dir was mirrored (from global, not overlaid). + Assert.True(Directory.Exists(Path.Combine(bcsHome, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion))); + + // 5. dotnet host binary is present. + var dotnetExeName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet"; + Assert.True(File.Exists(Path.Combine(bcsHome, dotnetExeName))); + + // 6. host/fxr was mirrored AND overlaid. + var hostFxrFile = Path.Combine(bcsHome, "host", "fxr", runtimeVersion, BuildCacheClient.GetNativeLibName("hostfxr")); + Assert.True(File.Exists(hostFxrFile)); } - foreach (var native in expectedNativeNames) + finally { - Assert.True(File.Exists(Path.Combine(sharedFw, native)), $"Missing native in shared FW: {native}"); + try { Directory.Delete(bcsHome, recursive: true); } catch { } } + } - Assert.True(File.Exists(Path.Combine(hostFxr, BuildCacheClient.GetNativeLibName("hostfxr")))); + [Fact] + public void CreateBuildCacheDotnetHome_NoBcsBitsForPlatform_Throws() + { + // Build a BCS archive layout for an RID that doesn't match the host RID, so the + // overlay finds nothing. + var hostRid = BuildCacheClient.GetPlatformMoniker(); + var wrongRid = hostRid == "linux-x64" ? "win-x64" : "linux-x64"; + var (extractDir, _, _, _) = BuildFakeBcsArchive(wrongRid, includeHost: false, includeApphost: false); + + const string runtimeVersion = "11.0.0-preview.5"; + const string aspNetCoreVersion = "11.0.0-preview.5"; + var globalHome = BuildFakeGlobalDotnetHome(runtimeVersion, aspNetCoreVersion); + + // Will resolve config from host RID and search for hostRid-shaped subtree → 0 files. + var ex = Assert.Throws(() => + BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir, runtimeVersion, aspNetCoreVersion, + "abcdef0123456789", buildCacheConfig: null)); + + Assert.Contains("0 files", ex.Message); } [Fact] - public void OverlayDotnetHome_WithCommitSha_RewritesVersionFile() + public void CreateBuildCacheDotnetHome_TwoConcurrentJobs_AreIsolated() { var rid = BuildCacheClient.GetPlatformMoniker(); - var (extractDir, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true); + var configKey = ConfigKeyForRid(rid); + var (extractDir1, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true, includeApphost: false); + var (extractDir2, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true, includeApphost: false); - const string runtimeVersion = "11.0.0-preview.5.26256.117"; - const string commitSha = "603403d9cb49d3d1c35b56bcff024ce99a8c5c3a"; - var dotnetHome = Path.Combine(_testDir, "dotnetHome-version"); - var sharedFw = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); - Directory.CreateDirectory(sharedFw); + const string runtimeVersion = "11.0.0-preview.5"; + var globalHome = BuildFakeGlobalDotnetHome(runtimeVersion, runtimeVersion); + var sha1 = "1111aaaa2222bbbb3333cccc4444dddd55556666"; + var sha2 = "6666eeee7777ffff8888aaaa9999bbbbccccdddd"; + + var home1 = BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir1, runtimeVersion, runtimeVersion, sha1, configKey); + var home2 = BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir2, runtimeVersion, runtimeVersion, sha2, configKey); + + try + { + Assert.NotEqual(home1, home2); - // Simulate dotnet-install having already written a .version file with the FEED commit. - File.WriteAllText(Path.Combine(sharedFw, ".version"), "feedfeedfeed\n" + runtimeVersion + "\n"); + var v1 = File.ReadAllText(Path.Combine(home1, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version")); + var v2 = File.ReadAllText(Path.Combine(home2, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version")); - BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, runtimeVersion, commitSha); + Assert.Contains(sha1, v1); + Assert.DoesNotContain(sha2, v1); + Assert.Contains(sha2, v2); + Assert.DoesNotContain(sha1, v2); - var versionFileContents = File.ReadAllText(Path.Combine(sharedFw, ".version")); - var lines = versionFileContents.Split('\n', StringSplitOptions.RemoveEmptyEntries); - Assert.Equal(commitSha, lines[0]); - Assert.Equal(runtimeVersion, lines[1]); + // Global home untouched. + var globalV = File.ReadAllText(Path.Combine(globalHome, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version")); + Assert.DoesNotContain(sha1, globalV); + Assert.DoesNotContain(sha2, globalV); + } + finally + { + try { Directory.Delete(home1, recursive: true); } catch { } + try { Directory.Delete(home2, recursive: true); } catch { } + } } + // ------------------------------------------------------------------- + // CleanupExtractDir + // ------------------------------------------------------------------- + [Fact] - public void OverlayDotnetHome_WithoutCommitSha_LeavesVersionFileUntouched() + public void CleanupExtractDir_DeletesDirectory() { - var rid = BuildCacheClient.GetPlatformMoniker(); - var (extractDir, _, _, _) = BuildFakeBcsArchive(rid, includeHost: true); - - const string runtimeVersion = "11.0.0-preview.5.26256.117"; - var dotnetHome = Path.Combine(_testDir, "dotnetHome-noversion"); - var sharedFw = Path.Combine(dotnetHome, "shared", "Microsoft.NETCore.App", runtimeVersion); - Directory.CreateDirectory(sharedFw); + var dir = Path.Combine(_testDir, "cleanup-target"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "x.txt"), "hi"); - const string original = "feedfeedfeed\n" + "11.0.0-preview.5.26256.117\n"; - File.WriteAllText(Path.Combine(sharedFw, ".version"), original); + BuildCacheClient.CleanupExtractDir(dir); - BuildCacheClient.OverlayDotnetHome(extractDir, dotnetHome, runtimeVersion); + Assert.False(Directory.Exists(dir)); + } - Assert.Equal(original, File.ReadAllText(Path.Combine(sharedFw, ".version"))); + [Fact] + public void CleanupExtractDir_MissingDir_DoesNotThrow() + { + BuildCacheClient.CleanupExtractDir(Path.Combine(_testDir, "never-existed")); + BuildCacheClient.CleanupExtractDir(null); + BuildCacheClient.CleanupExtractDir(""); } // ------------------------------------------------------------------- - // Fake BCS archive helpers + // Helpers // ------------------------------------------------------------------- + private static string ConfigKeyForRid(string rid) + => BuildCacheClient.PlatformToBcsConfig.TryGetValue(rid, out var v) ? v.configKey : null; + + /// + /// Builds a fake "global" dotnet home with .version files containing a FEED commit so + /// tests can detect whether the .version was overwritten with the BCS commit. + /// + private string BuildFakeGlobalDotnetHome(string runtimeVersion, string aspNetCoreVersion) + { + var home = Path.Combine(_testDir, "global-home-" + Guid.NewGuid().ToString("N")); + var netCoreApp = Path.Combine(home, "shared", "Microsoft.NETCore.App", runtimeVersion); + var aspNetCoreApp = Path.Combine(home, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); + var hostFxr = Path.Combine(home, "host", "fxr", runtimeVersion); + + Directory.CreateDirectory(netCoreApp); + Directory.CreateDirectory(aspNetCoreApp); + Directory.CreateDirectory(hostFxr); + + File.WriteAllText(Path.Combine(netCoreApp, ".version"), "FEED_COMMIT_DO_NOT_TOUCH\n" + runtimeVersion + "\n"); + File.WriteAllText(Path.Combine(netCoreApp, "System.Private.CoreLib.dll"), "feed managed"); + File.WriteAllText(Path.Combine(netCoreApp, BuildCacheClient.GetNativeLibName("hostpolicy")), "feed hostpolicy"); + + File.WriteAllText(Path.Combine(aspNetCoreApp, ".version"), "FEED_ASPNET\n" + aspNetCoreVersion + "\n"); + File.WriteAllText(Path.Combine(aspNetCoreApp, "Microsoft.AspNetCore.dll"), "feed aspnet"); + + File.WriteAllText(Path.Combine(hostFxr, BuildCacheClient.GetNativeLibName("hostfxr")), "feed hostfxr"); + + var dotnetExeName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet"; + File.WriteAllText(Path.Combine(home, dotnetExeName), "feed dotnet host"); + + return home; + } + /// - /// Builds an on-disk fake of an extracted BCS archive matching the layout the agent expects: - /// microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net10.0/*.dll, - /// microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/native/*, and - /// optionally {rid}.Release/corehost/*. + /// Builds a fake BCS extraction at microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/ + /// + corehost layout. adds the renamed-by-SDK apphost binary. /// - private (string extractDir, string runtimesDir, System.Collections.Generic.List managed, System.Collections.Generic.List native) - BuildFakeBcsArchive(string rid, bool includeHost) + private (string extractDir, string runtimesDir, List managed, List native) + BuildFakeBcsArchive(string rid, bool includeHost, bool includeApphost) { var extractDir = Path.Combine(_testDir, "extracted-" + Guid.NewGuid().ToString("N")); var nugetPkg = Path.Combine(extractDir, $"microsoft.netcore.app.runtime.{rid}"); var runtimesDir = Path.Combine(nugetPkg, "Release", "runtimes", rid); - var libDir = Path.Combine(runtimesDir, "lib", "net10.0"); + var libDir = Path.Combine(runtimesDir, "lib", "net11.0"); var nativeDir = Path.Combine(runtimesDir, "native"); Directory.CreateDirectory(libDir); Directory.CreateDirectory(nativeDir); - var managed = new System.Collections.Generic.List + var managed = new List { "System.Private.CoreLib.dll", "System.Runtime.dll", @@ -412,37 +524,55 @@ public void OverlayDotnetHome_WithoutCommitSha_LeavesVersionFileUntouched() }; foreach (var dll in managed) { - File.WriteAllText(Path.Combine(libDir, dll), "fake managed " + dll); + File.WriteAllText(Path.Combine(libDir, dll), "BCS managed " + dll); } - var native = new System.Collections.Generic.List(); - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + List native; + if (rid.StartsWith("win-", StringComparison.OrdinalIgnoreCase)) { - native.AddRange(new[] { "coreclr.dll", "clrjit.dll" }); + native = new List { "coreclr.dll", "clrjit.dll" }; } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + else if (rid.StartsWith("osx-", StringComparison.OrdinalIgnoreCase)) { - native.AddRange(new[] { "libcoreclr.dylib", "libclrjit.dylib" }); + native = new List { "libcoreclr.dylib", "libclrjit.dylib" }; } else { - native.AddRange(new[] { "libcoreclr.so", "libclrjit.so" }); + native = new List { "libcoreclr.so", "libclrjit.so" }; } foreach (var n in native) { - File.WriteAllText(Path.Combine(nativeDir, n), "fake native " + n); + File.WriteAllText(Path.Combine(nativeDir, n), "BCS native " + n); } if (includeHost) { var hostDir = Path.Combine(extractDir, $"{rid}.Release", "corehost"); Directory.CreateDirectory(hostDir); - File.WriteAllText(Path.Combine(hostDir, BuildCacheClient.GetNativeLibName("hostpolicy")), "hostpolicy"); - File.WriteAllText(Path.Combine(hostDir, BuildCacheClient.GetNativeLibName("hostfxr")), "hostfxr"); - File.WriteAllText(Path.Combine(hostDir, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet"), "dotnet"); + File.WriteAllText(Path.Combine(hostDir, NativeLibForRid(rid, "hostpolicy")), "BCS hostpolicy"); + File.WriteAllText(Path.Combine(hostDir, NativeLibForRid(rid, "hostfxr")), "BCS hostfxr"); + File.WriteAllText(Path.Combine(hostDir, rid.StartsWith("win-") ? "dotnet.exe" : "dotnet"), "BCS dotnet host"); + + if (includeApphost) + { + File.WriteAllText(Path.Combine(hostDir, rid.StartsWith("win-") ? "apphost.exe" : "apphost"), "BCS apphost"); + } } return (extractDir, runtimesDir, managed, native); } + + private static string NativeLibForRid(string rid, string baseName) + { + if (rid.StartsWith("win-", StringComparison.OrdinalIgnoreCase)) + { + return $"{baseName}.dll"; + } + if (rid.StartsWith("osx-", StringComparison.OrdinalIgnoreCase)) + { + return $"lib{baseName}.dylib"; + } + return $"lib{baseName}.so"; + } } } From f185064b9f90b71937ab4a5b8c8328fc4ab2ea50 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 19 Jun 2026 12:25:27 -0700 Subject: [PATCH 5/7] Add aspnetcore BCS flavour to the buildcache channel Extends the buildcache channel (currently runtime/coreclr only) to also resolve dotnet/aspnetcore builds from the Build Cache Service, selected per-job via a new buildCacheRepo property (runtime|aspnetcore, default runtime). The runtime path is unchanged; aspnetcore is purely additive. - Job.cs: BuildCacheRepo property + build-key plumbing. - BuildCacheClient.cs: BuildCacheFlavor enum + ParseFlavor/GetConfigMap/ AllConfigs; flavour-aware ResolveBuildCacheConfig/GetArtifactFile/ GetRidForConfig; CreateBuildCacheDotnetHome + OverlayPublishedOutput overlay Microsoft.AspNetCore.App (managed-only, no host) for aspnetcore. - Startup.cs: flavour-aware selection, version defaulting/sentinels, aspNetCoreVersion re-resolution, Q2 single-component collision handling, apply + reporting (+buildcache.{sha} on AspNetCoreVersion). - Tests: aspnetcore ParseFlavor/GetRidForConfig/overlay/home tests (93 green). - Docs: buildCacheRepo usage, aspnetcore archive layout, single-component caveat. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/build_cache_requirements.md | 26 ++- docs/dotnet_versions.md | 29 ++- src/Microsoft.Crank.Agent/BuildCacheClient.cs | 211 ++++++++++++++---- src/Microsoft.Crank.Agent/Startup.cs | 120 ++++++++-- src/Microsoft.Crank.Models/Job.cs | 9 +- .../BuildCacheClientTests.cs | 179 +++++++++++++++ 6 files changed, 498 insertions(+), 76 deletions(-) diff --git a/docs/build_cache_requirements.md b/docs/build_cache_requirements.md index 3efac3690..209ce8285 100644 --- a/docs/build_cache_requirements.md +++ b/docs/build_cache_requirements.md @@ -22,10 +22,12 @@ GET https://pvscmdupload.z22.web.core.windows.net/builds/{repoName}/buildArtifac ``` Where: -- `repoName` = `runtime` (initially; `aspnetcore` in the future) +- `repoName` = `runtime` or `aspnetcore` (selected per-job via the `buildCacheRepo` property; defaults to the agent's `--build-cache-repo-name`) - `branch` = e.g., `main`, `release/10.0` -- `configKey` = e.g., `coreclr_x64_linux`, `coreclr_arm64_windows` -- `artifactFile` = e.g., `BuildArtifacts_linux_x64_Release_coreclr.tar.gz` +- `configKey` = e.g., `coreclr_x64_linux`, `coreclr_arm64_windows` (runtime); `aspnetcore_x64_linux`, `aspnetcore_arm64_windows` (aspnetcore) +- `artifactFile` = e.g., `BuildArtifacts_linux_x64_Release_coreclr.tar.gz` (runtime); `BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz` (aspnetcore) + +The aspnetcore `latestBuilds.json` lives at `builds/aspnetcore/latest/{branch}/latestBuilds.json` and contains only the 5 `aspnetcore_*` config keys plus an `all` entry and `branch_name`; it does **not** carry the runtime `coreclr_*` keys. Crank's parser enumerates keys dynamically, so it tolerates either repo's file. --- @@ -47,7 +49,7 @@ If automated bisection tooling is built in the future, it can query GitHub direc ## Requirement 4: Artifact Layout Stability -Crank extracts runtime artifacts using this path convention inside the archive: +Crank extracts **runtime** artifacts using this path convention inside the archive: ``` microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net{X}.0/ → managed DLLs @@ -55,9 +57,21 @@ microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/native/ → na {rid}.Release/corehost/ → host binaries (dotnet, libhostfxr, libhostpolicy) ``` -Where `{rid}` = `linux-x64`, `linux-arm64`, `win-x64`, etc. +For **aspnetcore** artifacts crank uses the parallel convention (managed-only — the ASP.NET Core +runtime pack ships no host binaries, and native is optional): + +``` +microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net{X}.0/ → managed Microsoft.AspNetCore.*.dll +microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/native/ → native libs (optional) +``` + +Where `{rid}` = `linux-x64`, `linux-arm64`, `win-x64`, `win-arm64`, `win-x86`. (aspnetcore v1 has no +musl/osx/arm32 configs.) -This layout was confirmed by inspecting `BuildArtifacts_linux_arm64_Release_coreclr.tar.gz`. **If this layout changes in future builds, the crank extraction will break.** Consider treating it as a stable contract or documenting it. +The runtime layout was confirmed by inspecting `BuildArtifacts_linux_arm64_Release_coreclr.tar.gz`; +the aspnetcore layout is asserted by dotnet/performance's `pack-bcs-archives.ps1`. +**If either layout changes in future builds, the crank extraction will break.** Consider treating it +as a stable contract or documenting it. --- diff --git a/docs/dotnet_versions.md b/docs/dotnet_versions.md index 70e21181c..e7e6a218b 100644 --- a/docs/dotnet_versions.md +++ b/docs/dotnet_versions.md @@ -122,14 +122,36 @@ The following command uses the `edge` channel but ASP.NET is fixed so it doesn't ## Using the Build Cache channel -The `buildcache` channel resolves the .NET runtime from the Build Cache Service (BCS), which caches pre-built runtime binaries for individual commits. This is useful for performance regression bisection where VMR feed gaps make it hard to pinpoint which commit caused a regression. +The `buildcache` channel resolves pre-built binaries for individual commits from the Build Cache Service (BCS). This is useful for performance regression bisection where VMR feed gaps make it hard to pinpoint which commit caused a regression. + +The channel can resolve from one of two repositories, selected per-job with the `buildCacheRepo` property: + +- `runtime` (default) — overlays the base .NET runtime (`Microsoft.NETCore.App`) with BCS bits built from a [dotnet/runtime](https://github.com/dotnet/runtime) commit. +- `aspnetcore` — overlays the ASP.NET Core shared framework (`Microsoft.AspNetCore.App`) with BCS bits built from a [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore) commit. The base runtime stays at the feed-resolved version. + +When no `buildCacheRepo` is supplied the job falls back to the agent-level `--build-cache-repo-name` (which itself defaults to `runtime`), so existing runtime usage is unchanged. ### Basic usage (latest cached build on main) ``` +# runtime (default) > crank --config benchmarks.yml --scenario json --profile aspnet-perf-lin --application.channel buildcache ``` +### Bisecting ASP.NET Core + +Add `--application.buildCacheRepo aspnetcore` to resolve an ASP.NET Core commit instead. The `buildCacheCommitSha` / `buildCacheBranch` values are then interpreted as **dotnet/aspnetcore** commit/branch: + +``` +# latest cached aspnetcore build on main +> crank --config benchmarks.yml --scenario json --profile aspnet-perf-lin --application.channel buildcache --application.buildCacheRepo aspnetcore + +# specific aspnetcore commit +> crank --config benchmarks.yml --scenario json --profile aspnet-perf-lin --application.channel buildcache --application.buildCacheRepo aspnetcore --application.buildCacheCommitSha a1b2c3d4e5f6... +``` + +> **One BCS component per job (v1).** A single job overrides EITHER the base runtime OR the ASP.NET Core shared framework from BCS, not both at once. The `buildCacheRepo` selector decides which one; the other framework comes from the normal feed. If a job somehow requests both, the selected flavour wins and the other is skipped with a log warning. + ### Specific commit SHA ``` @@ -154,9 +176,10 @@ If the commit is not found in the cache, crank will fail with an error rather th | Property | Default | Description | |----------|---------|-------------| -| `buildCacheCommitSha` | (empty) | Specific runtime commit SHA. If empty, uses the latest cached build for the branch. | +| `buildCacheRepo` | (agent `--build-cache-repo-name`, i.e. `runtime`) | Which BCS repository to resolve from: `runtime` (overlays `Microsoft.NETCore.App`) or `aspnetcore` (overlays `Microsoft.AspNetCore.App`). | +| `buildCacheCommitSha` | (empty) | Specific commit SHA in the selected repo (runtime or aspnetcore). If empty, uses the latest cached build for the branch. | | `buildCacheBranch` | `main` | Branch to query for the latest build. | -| `buildCacheConfig` | (auto-detected) | BCS configuration key (e.g., `coreclr_x64_linux`). Auto-detected from agent platform. | +| `buildCacheConfig` | (auto-detected) | BCS configuration key (e.g., `coreclr_x64_linux` for runtime, `aspnetcore_x64_linux` for aspnetcore). Auto-detected from agent platform and the selected repo. | ### Agent configuration diff --git a/src/Microsoft.Crank.Agent/BuildCacheClient.cs b/src/Microsoft.Crank.Agent/BuildCacheClient.cs index 08d35f0ed..cdb80193f 100644 --- a/src/Microsoft.Crank.Agent/BuildCacheClient.cs +++ b/src/Microsoft.Crank.Agent/BuildCacheClient.cs @@ -57,6 +57,67 @@ internal static class BuildCacheClient ["win-x86"] = ("coreclr_x86_windows", "BuildArtifacts_windows_x86_Release_coreclr.zip", "win-x86"), }; + /// + /// ASP.NET Core (dotnet/aspnetcore) variant of . Maps the + /// agent's platform (RID) to the aspnetcore BCS configuration key and artifact filename. + /// The configKey / artifact tokens and the archive's internal layout + /// (microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/...) are a + /// load-bearing contract asserted by dotnet/performance's pack-bcs-archives.ps1. + /// v1 has no musl/osx/arm32 entries. + /// + internal static readonly IReadOnlyDictionary PlatformToBcsConfigAspNetCore = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["linux-x64"] = ("aspnetcore_x64_linux", "BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz", "linux-x64"), + ["linux-arm64"] = ("aspnetcore_arm64_linux", "BuildArtifacts_linux_arm64_Release_aspnetcore.tar.gz", "linux-arm64"), + ["win-x64"] = ("aspnetcore_x64_windows", "BuildArtifacts_windows_x64_Release_aspnetcore.zip", "win-x64"), + ["win-arm64"] = ("aspnetcore_arm64_windows", "BuildArtifacts_windows_arm64_Release_aspnetcore.zip", "win-arm64"), + ["win-x86"] = ("aspnetcore_x86_windows", "BuildArtifacts_windows_x86_Release_aspnetcore.zip", "win-x86"), + }; + + /// + /// Which BCS repository the buildcache channel resolves from. Selects the platform→config + /// map, the latestBuilds.json / artifact path RepoName, and the overlay target inside the + /// dotnet home (Runtime → Microsoft.NETCore.App; AspNetCore → Microsoft.AspNetCore.App). + /// + internal enum BuildCacheFlavor + { + Runtime, + AspNetCore, + } + + /// BCS RepoName for each flavour (segment in BCS blob paths). + internal const string RepoNameRuntime = "runtime"; + internal const string RepoNameAspNetCore = "aspnetcore"; + + /// + /// Maps a BCS RepoName (the per-job buildCacheRepo selector) to a flavour. Anything + /// other than "aspnetcore" (including empty / "runtime") resolves to , + /// preserving the proven runtime default. + /// + internal static BuildCacheFlavor ParseFlavor(string repoName) + { + return string.Equals(repoName, RepoNameAspNetCore, StringComparison.OrdinalIgnoreCase) + ? BuildCacheFlavor.AspNetCore + : BuildCacheFlavor.Runtime; + } + + /// Selects the platform→config map for a flavour. + private static IReadOnlyDictionary GetConfigMap(BuildCacheFlavor flavor) + { + return flavor == BuildCacheFlavor.AspNetCore ? PlatformToBcsConfigAspNetCore : PlatformToBcsConfig; + } + + /// + /// All config entries across every flavour. configKeys are globally unique (coreclr_* vs + /// aspnetcore_*), so config→artifact and config→RID lookups can search this union without + /// the caller needing to know the flavour. + /// + private static IEnumerable<(string configKey, string artifactFile, string rid)> AllConfigs() + { + return PlatformToBcsConfig.Values.Concat(PlatformToBcsConfigAspNetCore.Values); + } + /// /// Sentinel thrown for HTTP responses that are definitively not retryable (e.g. 404). /// Distinguishes "the build doesn't exist" from "transient network blip". @@ -98,7 +159,7 @@ internal static void ValidateCommitSha(string commitSha) CancellationToken cancellationToken = default) { ValidateCommitSha(commitSha); - buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig, ParseFlavor(repoName)); if (string.IsNullOrEmpty(commitSha)) { @@ -145,7 +206,7 @@ public static async Task DownloadAndExtractAsync( throw new ArgumentException("commitSha must be provided.", nameof(commitSha)); } - buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig, ParseFlavor(repoName)); var artifactFile = GetArtifactFile(buildCacheConfig); var normalizedBaseUrl = (baseUrl ?? string.Empty).TrimEnd('/'); @@ -225,6 +286,13 @@ public static void CleanupExtractDir(string extractDir) /// Builds a per-job dotnet home that mirrors the relevant subtrees of the global dotnet /// home (runtime, asp.net, host) and overlays BCS bits on top. The global dotnet home is /// NOT modified, so concurrent jobs and subsequent non-buildcache jobs are unaffected. + /// + /// For the BCS bits overlay the base runtime + /// (Microsoft.NETCore.App) plus host binaries. For + /// the BCS bits overlay the ASP.NET Core shared framework (Microsoft.AspNetCore.App) only; + /// the aspnetcore runtime pack is managed-only and ships no host binaries, so the feed + /// runtime/host are left in place and only the managed Microsoft.AspNetCore.*.dll set is + /// replaced. /// /// Absolute path to the per-job dotnet home root. Caller owns it. public static string CreateBuildCacheDotnetHome( @@ -233,14 +301,22 @@ public static string CreateBuildCacheDotnetHome( string runtimeVersion, string aspNetCoreVersion, string commitSha, - string buildCacheConfig) + string buildCacheConfig, + BuildCacheFlavor flavor = BuildCacheFlavor.Runtime) { if (string.IsNullOrEmpty(runtimeVersion)) { throw new ArgumentException("runtimeVersion must be provided.", nameof(runtimeVersion)); } - buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); + if (flavor == BuildCacheFlavor.AspNetCore && string.IsNullOrEmpty(aspNetCoreVersion)) + { + throw new ArgumentException( + "aspNetCoreVersion must be provided for the aspnetcore build cache flavour (it is the overlay target).", + nameof(aspNetCoreVersion)); + } + + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig, flavor); var rid = GetRidForConfig(buildCacheConfig); // Per-job, never reused across jobs to avoid pollution. @@ -277,47 +353,70 @@ public static string CreateBuildCacheDotnetHome( } // 4. Mirror shared/Microsoft.AspNetCore.App/{aspNetCoreVersion}. + var dstAspNet = string.IsNullOrEmpty(aspNetCoreVersion) + ? null + : Path.Combine(bcsHomeRoot, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); if (!string.IsNullOrEmpty(aspNetCoreVersion)) { var srcAspNet = Path.Combine(globalDotnetHome, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); - var dstAspNet = Path.Combine(bcsHomeRoot, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); if (Directory.Exists(srcAspNet)) { CopyDirectory(srcAspNet, dstAspNet); } } - // 5. Overlay BCS managed + native into the per-job NETCore.App. int filesOverlaid = 0; - var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); - if (nugetPackageDir != null) + + if (flavor == BuildCacheFlavor.AspNetCore) { - var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); - if (Directory.Exists(runtimesDir)) + // 5a. Overlay BCS managed (+ optional native) Microsoft.AspNetCore.*.dll into the + // per-job ASP.NET Core shared framework. The aspnetcore runtime pack is + // essentially managed-only; native is optional (CopyNative returns 0 if absent) + // and there are no host binaries to overlay (the feed runtime/host stay in place). + var aspNetPackageDir = FindDirectory(extractDir, $"microsoft.aspnetcore.app.runtime.{rid}"); + if (aspNetPackageDir != null) { - filesOverlaid += CopyManaged(runtimesDir, dstNetCoreApp); - filesOverlaid += CopyNative(runtimesDir, dstNetCoreApp); + var runtimesDir = Path.Combine(aspNetPackageDir, "Release", "runtimes", rid); + if (Directory.Exists(runtimesDir)) + { + filesOverlaid += CopyManaged(runtimesDir, dstAspNet); + filesOverlaid += CopyNative(runtimesDir, dstAspNet); + } } } - - // 6. Overlay BCS host binaries. - var corehostDir = FindCorehostDirectory(extractDir, rid); - if (corehostDir != null) + else { - filesOverlaid += CopyHostBinaryIfPresent(corehostDir, dstNetCoreApp, GetNativeLibName("hostpolicy")); - - if (Directory.Exists(dstHostFxr)) + // 5. Overlay BCS managed + native into the per-job NETCore.App. + var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); + if (nugetPackageDir != null) { - filesOverlaid += CopyHostBinaryIfPresent(corehostDir, dstHostFxr, GetNativeLibName("hostfxr")); + var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); + if (Directory.Exists(runtimesDir)) + { + filesOverlaid += CopyManaged(runtimesDir, dstNetCoreApp); + filesOverlaid += CopyNative(runtimesDir, dstNetCoreApp); + } } - var dstDotnetHost = Path.Combine(bcsHomeRoot, dotnetExeName); - var copied = CopyHostBinaryIfPresent(corehostDir, bcsHomeRoot, dotnetExeName); - if (copied > 0) + // 6. Overlay BCS host binaries. + var corehostDir = FindCorehostDirectory(extractDir, rid); + if (corehostDir != null) { - EnsureExecutable(dstDotnetHost); + filesOverlaid += CopyHostBinaryIfPresent(corehostDir, dstNetCoreApp, GetNativeLibName("hostpolicy")); + + if (Directory.Exists(dstHostFxr)) + { + filesOverlaid += CopyHostBinaryIfPresent(corehostDir, dstHostFxr, GetNativeLibName("hostfxr")); + } + + var dstDotnetHost = Path.Combine(bcsHomeRoot, dotnetExeName); + var copied = CopyHostBinaryIfPresent(corehostDir, bcsHomeRoot, dotnetExeName); + if (copied > 0) + { + EnsureExecutable(dstDotnetHost); + } + filesOverlaid += copied; } - filesOverlaid += copied; } if (filesOverlaid == 0) @@ -326,38 +425,58 @@ public static string CreateBuildCacheDotnetHome( // Tear it down and let the caller fail the job loudly. try { Directory.Delete(bcsHomeRoot, recursive: true); } catch { } throw new InvalidOperationException( - $"Build Cache: overlay copied 0 files for commit {ShortSha(commitSha)} (config '{buildCacheConfig}', rid '{rid}'). " + + $"Build Cache: overlay copied 0 files for commit {ShortSha(commitSha)} (config '{buildCacheConfig}', rid '{rid}', repo '{flavor}'). " + "The archive layout may have changed or the platform is not supported."); } - // 7. Rewrite .version so any consumer (the agent's own BenchmarksNetCoreAppVersion - // measurement, GetDependencies, etc.) reports the BCS commit. - File.WriteAllText( - Path.Combine(dstNetCoreApp, ".version"), - $"{commitSha}\n{runtimeVersion}\n"); + // 7. Rewrite the overlaid framework's .version so any consumer (the agent's own + // version measurement, GetDependencies, etc.) reports the BCS commit. For aspnetcore + // that is the Microsoft.AspNetCore.App folder; for runtime it is Microsoft.NETCore.App. + if (flavor == BuildCacheFlavor.AspNetCore) + { + File.WriteAllText( + Path.Combine(dstAspNet, ".version"), + $"{commitSha}\n{aspNetCoreVersion}\n"); + } + else + { + File.WriteAllText( + Path.Combine(dstNetCoreApp, ".version"), + $"{commitSha}\n{runtimeVersion}\n"); + } Log.Info($"Build Cache: Per-job dotnet home built at {bcsHomeRoot} ({filesOverlaid} BCS files overlaid)"); return bcsHomeRoot; } /// - /// Overlays BCS runtime binaries (managed + native + apphost) into a self-contained - /// published output directory. For SCD the runtime ships next to the app, so this is the - /// only way to make the benchmark actually run BCS bits. The BCS apphost is renamed to - /// match the published app's executable name (the SDK renames apphost → AssemblyName). + /// Overlays BCS runtime binaries into a self-contained published output directory. For SCD + /// the runtime ships next to the app, so this is the only way to make the benchmark + /// actually run BCS bits. + /// + /// For the base runtime managed + native binaries + /// and host (hostpolicy) are overlaid. For only + /// the managed (+ optional native) Microsoft.AspNetCore.*.dll set is overlaid — the + /// aspnetcore pack ships no host binaries and the base runtime stays the published one. + /// The SDK-bound apphost is never replaced (see note below). /// /// Number of files overlaid. public static int OverlayPublishedOutput( string extractDir, string outputFolder, string buildCacheConfig, - string assemblyName) + string assemblyName, + BuildCacheFlavor flavor = BuildCacheFlavor.Runtime) { - buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig); + buildCacheConfig = ResolveBuildCacheConfig(buildCacheConfig, flavor); var rid = GetRidForConfig(buildCacheConfig); int filesCopied = 0; - var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); + var packageName = flavor == BuildCacheFlavor.AspNetCore + ? $"microsoft.aspnetcore.app.runtime.{rid}" + : $"microsoft.netcore.app.runtime.{rid}"; + + var nugetPackageDir = FindDirectory(extractDir, packageName); if (nugetPackageDir != null) { var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); @@ -368,7 +487,8 @@ public static int OverlayPublishedOutput( } } - var corehostDir = FindCorehostDirectory(extractDir, rid); + // The aspnetcore pack ships no host binaries, so host overlay only applies to runtime. + var corehostDir = flavor == BuildCacheFlavor.AspNetCore ? null : FindCorehostDirectory(extractDir, rid); if (corehostDir != null) { filesCopied += CopyHostBinaryIfPresent(corehostDir, outputFolder, GetNativeLibName("hostpolicy")); @@ -788,7 +908,7 @@ private static string FindCorehostDirectory(string extractDir, string rid) // --- Platform / RID mapping --------------------------------------------------- - private static string ResolveBuildCacheConfig(string buildCacheConfig) + private static string ResolveBuildCacheConfig(string buildCacheConfig, BuildCacheFlavor flavor = BuildCacheFlavor.Runtime) { if (!string.IsNullOrEmpty(buildCacheConfig)) { @@ -796,18 +916,18 @@ private static string ResolveBuildCacheConfig(string buildCacheConfig) } var rid = GetPlatformMoniker(); - if (PlatformToBcsConfig.TryGetValue(rid, out var mapped)) + if (GetConfigMap(flavor).TryGetValue(rid, out var mapped)) { return mapped.configKey; } throw new InvalidOperationException( - $"No Build Cache configuration mapping for platform '{rid}'. Specify buildCacheConfig explicitly."); + $"No Build Cache configuration mapping for platform '{rid}' (repo '{flavor}'). Specify buildCacheConfig explicitly."); } private static string GetArtifactFile(string buildCacheConfig) { - var match = PlatformToBcsConfig.Values.FirstOrDefault(v => + var match = AllConfigs().FirstOrDefault(v => string.Equals(v.configKey, buildCacheConfig, StringComparison.OrdinalIgnoreCase)); if (match.artifactFile == null) @@ -822,11 +942,12 @@ private static string GetArtifactFile(string buildCacheConfig) /// /// Maps a BCS config key back to its RID. Use this for overlay path discovery so an /// explicit musl/cross-arch override actually finds the right runtime pack inside the - /// archive instead of falling back to the host's detected RID. + /// archive instead of falling back to the host's detected RID. Searches every flavour's + /// map since config keys are globally unique. /// internal static string GetRidForConfig(string buildCacheConfig) { - var match = PlatformToBcsConfig.Values.FirstOrDefault(v => + var match = AllConfigs().FirstOrDefault(v => string.Equals(v.configKey, buildCacheConfig, StringComparison.OrdinalIgnoreCase)); if (match.rid == null) diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index be2dbf116..aa63b8dc7 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -2971,13 +2971,24 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str } } + // Build Cache Service flavour selection. The per-job buildCacheRepo selector (falling + // back to the agent-level --build-cache-repo-name default) decides which BCS repository + // the buildcache channel resolves from: "runtime" overrides Microsoft.NETCore.App, + // "aspnetcore" overrides Microsoft.AspNetCore.App. + var buildCacheRepo = !String.IsNullOrEmpty(job.BuildCacheRepo) ? job.BuildCacheRepo : _buildCacheRepoName; + var buildCacheFlavor = BuildCacheClient.ParseFlavor(buildCacheRepo); + var isBuildCacheChannel = String.Equals(channel, "buildcache", StringComparison.OrdinalIgnoreCase); + var isAspNetCoreBuildCache = isBuildCacheChannel && buildCacheFlavor == BuildCacheClient.BuildCacheFlavor.AspNetCore; + if (String.IsNullOrEmpty(runtimeVersion)) { - runtimeVersion = channel; + // For the aspnetcore flavour the base runtime resolves to a real feed version; the + // BCS sentinel rides on aspNetCoreVersion instead of runtimeVersion. + runtimeVersion = isAspNetCoreBuildCache ? "latest" : channel; } - // For buildcache channel, SDK/ASP.NET/Desktop use "latest" since BCS only has runtime - var nonRuntimeChannel = String.Equals(channel, "buildcache", StringComparison.OrdinalIgnoreCase) ? "latest" : channel; + // For buildcache channel, the components NOT overridden by BCS use "latest" from feeds. + var nonRuntimeChannel = isBuildCacheChannel ? "latest" : channel; if (String.IsNullOrEmpty(desktopVersion)) { @@ -2986,7 +2997,9 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str if (String.IsNullOrEmpty(aspNetCoreVersion)) { - aspNetCoreVersion = nonRuntimeChannel; + // aspnetcore flavour: the buildcache sentinel rides on aspNetCoreVersion (mirrors how + // the runtime flavour puts the sentinel on runtimeVersion via the channel). + aspNetCoreVersion = isAspNetCoreBuildCache ? "buildcache" : nonRuntimeChannel; } if (String.IsNullOrEmpty(sdkVersion)) @@ -2996,9 +3009,24 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str runtimeVersion = await ResolveRuntimeVersion(buildToolsPath, targetFramework, runtimeVersion); - // Build Cache Service: if the runtime version is "BuildCache", prepare BCS artifacts - // and resolve a real "Latest" runtime version for the NuGet build. - var useBuildCache = String.Equals(runtimeVersion, "BuildCache", StringComparison.OrdinalIgnoreCase); + // Build Cache Service: if either the runtime version is "BuildCache" (runtime flavour) or + // the asp.net version is "buildcache" (aspnetcore flavour), prepare BCS artifacts and + // resolve a real "Latest" version for the overridden component's NuGet build. + var useBuildCacheRuntime = String.Equals(runtimeVersion, "BuildCache", StringComparison.OrdinalIgnoreCase); + var useBuildCacheAspNet = String.Equals(aspNetCoreVersion, "buildcache", StringComparison.OrdinalIgnoreCase); + var useBuildCache = useBuildCacheRuntime || useBuildCacheAspNet; + + // The selector is authoritative for which component BCS overrides. In the rare case both + // sentinels are explicitly requested (Q2: v1 overrides only one component per job), honour + // the selector's flavour and skip the other below. + if (useBuildCacheAspNet && !useBuildCacheRuntime) + { + buildCacheFlavor = BuildCacheClient.BuildCacheFlavor.AspNetCore; + } + else if (useBuildCacheRuntime && !useBuildCacheAspNet) + { + buildCacheFlavor = BuildCacheClient.BuildCacheFlavor.Runtime; + } string buildCacheCommitSha = null; string buildCacheExtractDir = null; @@ -3025,26 +3053,57 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str var commitSha = job.BuildCacheCommitSha; var buildCacheConfig = job.BuildCacheConfig; - // Resolve which commit and config to use + // Resolve which commit and config to use. ResolveCommitAsync/DownloadAndExtractAsync + // derive the flavour (and therefore the config map + RepoName path segment) from the + // repoName we pass, so "runtime" and "aspnetcore" route to their own BCS blobs. var resolved = await BuildCacheClient.ResolveCommitAsync( - _buildCacheBaseUrl, _buildCacheRepoName, branch, commitSha, buildCacheConfig, cancellationToken); + _buildCacheBaseUrl, buildCacheRepo, branch, commitSha, buildCacheConfig, cancellationToken); buildCacheCommitSha = resolved.commitSha; buildCacheConfigResolved = resolved.buildCacheConfig; // Download and extract the BCS artifacts to a per-job temp directory buildCacheExtractDir = await BuildCacheClient.DownloadAndExtractAsync( - _buildCacheBaseUrl, _buildCacheRepoName, buildCacheCommitSha, buildCacheConfigResolved, + _buildCacheBaseUrl, buildCacheRepo, buildCacheCommitSha, buildCacheConfigResolved, cancellationToken); var shortSha = BuildCacheClient.ShortSha(buildCacheCommitSha); - Log.Info($"Build Cache: Artifacts for commit {shortSha} ready for post-build overlay"); + Log.Info($"Build Cache: Artifacts for commit {shortSha} (repo '{buildCacheRepo}') ready for post-build overlay"); - // Resolve a REAL runtime version from feeds for the NuGet build. We deliberately keep - // runtimeVersion pointing at this feed-resolved version so PatchRuntimeConfig and the - // dotnet-install steps agree; the BCS bits are overlaid on top of that exact version. - runtimeVersion = await ResolveRuntimeVersion(buildToolsPath, targetFramework, "Latest"); - Log.Info($"Runtime for build: {runtimeVersion} (Latest from feeds, will be overlaid with BCS commit {shortSha})"); + if (buildCacheFlavor == BuildCacheClient.BuildCacheFlavor.AspNetCore) + { + // The ASP.NET Core shared-framework version folder is feed-resolved (Latest) and + // the BCS bits overlay onto it — mirroring how the runtime flavour overlays onto a + // feed-resolved runtimeVersion. Set "Latest" here so ResolveAspNetCoreVersion below + // turns it into a real version that names the shared/Microsoft.AspNetCore.App/{ver} + // overlay target. runtimeVersion already holds a real feed runtime version. + aspNetCoreVersion = "Latest"; + Log.Info($"ASP.NET Core for build: Latest from feeds, will be overlaid with BCS commit {shortSha}"); + + // Q2: v1 overrides only one BCS component per job. If a runtime sentinel was also + // requested, honour the selector (aspnetcore) and let the base runtime resolve normally. + if (useBuildCacheRuntime) + { + Log.Info("Build Cache: both runtime and aspnetcore overrides were requested; honouring buildCacheRepo='aspnetcore' and resolving the base runtime from feeds."); + runtimeVersion = await ResolveRuntimeVersion(buildToolsPath, targetFramework, "Latest"); + } + } + else + { + // Resolve a REAL runtime version from feeds for the NuGet build. We deliberately keep + // runtimeVersion pointing at this feed-resolved version so PatchRuntimeConfig and the + // dotnet-install steps agree; the BCS bits are overlaid on top of that exact version. + runtimeVersion = await ResolveRuntimeVersion(buildToolsPath, targetFramework, "Latest"); + Log.Info($"Runtime for build: {runtimeVersion} (Latest from feeds, will be overlaid with BCS commit {shortSha})"); + + // Q2: if an aspnetcore sentinel was also requested, honour the selector (runtime) + // and let asp.net resolve normally from feeds. + if (useBuildCacheAspNet) + { + Log.Info("Build Cache: both runtime and aspnetcore overrides were requested; honouring buildCacheRepo='runtime' and resolving asp.net from feeds."); + aspNetCoreVersion = nonRuntimeChannel; + } + } } catch (Exception ex) { @@ -3326,7 +3385,8 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => runtimeVersion, aspNetCoreVersion, buildCacheCommitSha, - job.BuildCacheConfig); + job.BuildCacheConfig, + buildCacheFlavor); runtimeHomeDir = bcsHome; @@ -3594,7 +3654,8 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => buildCacheExtractDir, outputFolder, job.BuildCacheConfig, - assemblyName); + assemblyName, + buildCacheFlavor); Log.Info($"Build Cache: Overlaid {publishedOverlay} files into published output (commit {shortSha})"); } @@ -3614,9 +3675,18 @@ await ProcessUtil.RetryOnExceptionAsync(3, async () => return null; } - // Record the BCS commit alongside the runtime version for reporting. We append rather than - // replace so PatchRuntimeConfig still sees a valid feed-resolved version below. - job.RuntimeVersion = $"{runtimeVersion}+buildcache.{shortSha}"; + // Record the BCS commit alongside the overridden component's version for reporting. + // We append rather than replace so PatchRuntimeConfig still sees a valid feed-resolved + // version below. For the aspnetcore flavour annotate the asp.net version field (and + // leave the runtime version as the feed runtime); for runtime annotate the runtime field. + if (buildCacheFlavor == BuildCacheClient.BuildCacheFlavor.AspNetCore) + { + job.AspNetCoreVersion = $"{aspNetCoreVersion}+buildcache.{shortSha}"; + } + else + { + job.RuntimeVersion = $"{runtimeVersion}+buildcache.{shortSha}"; + } } PatchRuntimeConfig(job, outputFolder, aspNetCoreVersion, runtimeVersion); @@ -4143,6 +4213,14 @@ private static async Task ResolveAspNetCoreVersion(string aspNetCoreVers switch (aspNetCoreVersion.ToLowerInvariant()) { + case "buildcache": + // Defensive: the aspnetcore buildcache flavour rewrites aspNetCoreVersion to "Latest" + // in the BCS prep block before this point, so this normally isn't hit. Treat the raw + // sentinel as Latest so the shared-framework version still resolves to a real feed + // version (the folder the BCS bits overlay onto). + aspNetCoreVersion = await ResolveAspNetCoreVersion("Latest", targetFramework); + Log.Info($"ASP.NET: {aspNetCoreVersion} (BuildCache → Latest)"); + break; case "current": aspNetCoreVersion = string.IsNullOrEmpty(currentAspNetCoreVersion) ? await ResolveAspNetCoreVersion("Latest", targetFramework) diff --git a/src/Microsoft.Crank.Models/Job.cs b/src/Microsoft.Crank.Models/Job.cs index ee1537cb4..70fca9a6d 100644 --- a/src/Microsoft.Crank.Models/Job.cs +++ b/src/Microsoft.Crank.Models/Job.cs @@ -76,6 +76,11 @@ public class Job public string BuildCacheBranch { get; set; } = ""; public string BuildCacheConfig { get; set; } = ""; + // Selects which BCS repository the buildcache channel resolves from: "runtime" (default, + // overrides Microsoft.NETCore.App) or "aspnetcore" (overrides Microsoft.AspNetCore.App). + // Empty falls back to the agent-level --build-cache-repo-name default. + public string BuildCacheRepo { get; set; } = ""; + // Delay from the process started to the console receiving "Application started" public TimeSpan StartupMainMethod { get; set; } public TimeSpan BuildTime { get; set; } @@ -407,7 +412,8 @@ public BuildKeyData GetBuildKeyData() DockerContextDirectory = DockerContextDirectory, BuildCacheCommitSha = BuildCacheCommitSha, BuildCacheBranch = BuildCacheBranch, - BuildCacheConfig = BuildCacheConfig + BuildCacheConfig = BuildCacheConfig, + BuildCacheRepo = BuildCacheRepo }; } @@ -519,5 +525,6 @@ public class BuildKeyData public string BuildCacheCommitSha { get; set; } public string BuildCacheBranch { get; set; } public string BuildCacheConfig { get; set; } + public string BuildCacheRepo { get; set; } } } diff --git a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs index 21cfe42bd..382391efc 100644 --- a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs +++ b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs @@ -204,6 +204,39 @@ public void GetRidForConfig_UnknownConfig_Throws() Assert.Throws(() => BuildCacheClient.GetRidForConfig("totally_unknown")); } + // ------------------------------------------------------------------- + // ASP.NET Core config map (locked external contract — see + // dotnet/performance pack-bcs-archives.ps1). Pins configKey / RID / + // artifact filename so an accidental token change is caught. + // ------------------------------------------------------------------- + + [Theory] + [InlineData("linux-x64", "aspnetcore_x64_linux", "BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz")] + [InlineData("linux-arm64", "aspnetcore_arm64_linux", "BuildArtifacts_linux_arm64_Release_aspnetcore.tar.gz")] + [InlineData("win-x64", "aspnetcore_x64_windows", "BuildArtifacts_windows_x64_Release_aspnetcore.zip")] + [InlineData("win-arm64", "aspnetcore_arm64_windows", "BuildArtifacts_windows_arm64_Release_aspnetcore.zip")] + [InlineData("win-x86", "aspnetcore_x86_windows", "BuildArtifacts_windows_x86_Release_aspnetcore.zip")] + public void PlatformToBcsConfigAspNetCore_MatchesLockedContract(string rid, string expectedConfigKey, string expectedArtifact) + { + Assert.True(BuildCacheClient.PlatformToBcsConfigAspNetCore.TryGetValue(rid, out var entry), $"Missing aspnetcore entry for '{rid}'."); + Assert.Equal(expectedConfigKey, entry.configKey); + Assert.Equal(expectedArtifact, entry.artifactFile); + Assert.Equal(rid, entry.rid); + } + + [Fact] + public void PlatformToBcsConfigAspNetCore_HasExactlyTheFiveV1Platforms() + { + var rids = BuildCacheClient.PlatformToBcsConfigAspNetCore.Keys.OrderBy(k => k, StringComparer.Ordinal).ToArray(); + Assert.Equal( + new[] { "linux-arm64", "linux-x64", "win-arm64", "win-x64", "win-x86" }, + rids); + + // v1 explicitly excludes musl / osx / arm32. + Assert.DoesNotContain(BuildCacheClient.PlatformToBcsConfigAspNetCore.Keys, k => k.Contains("musl", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(BuildCacheClient.PlatformToBcsConfigAspNetCore.Keys, k => k.StartsWith("osx", StringComparison.OrdinalIgnoreCase)); + } + // ------------------------------------------------------------------- // SelectHighestManagedDir (numeric-aware) // ------------------------------------------------------------------- @@ -440,6 +473,122 @@ public void CreateBuildCacheDotnetHome_TwoConcurrentJobs_AreIsolated() } } + // ------------------------------------------------------------------- + // ASP.NET Core flavour: ParseFlavor + config→RID + overlay targeting + // ------------------------------------------------------------------- + + [Theory] + [InlineData("aspnetcore", true)] + [InlineData("ASPNETCORE", true)] + [InlineData("runtime", false)] + [InlineData("RUNTIME", false)] + [InlineData("", false)] + [InlineData(null, false)] + [InlineData("something-else", false)] + public void ParseFlavor_MapsRepoNameToFlavor(string repoName, bool expectedAspNetCore) + { + var expected = expectedAspNetCore + ? BuildCacheClient.BuildCacheFlavor.AspNetCore + : BuildCacheClient.BuildCacheFlavor.Runtime; + Assert.Equal(expected, BuildCacheClient.ParseFlavor(repoName)); + } + + [Theory] + [InlineData("aspnetcore_x64_linux", "linux-x64")] + [InlineData("aspnetcore_arm64_linux", "linux-arm64")] + [InlineData("aspnetcore_x64_windows", "win-x64")] + [InlineData("aspnetcore_arm64_windows", "win-arm64")] + [InlineData("aspnetcore_x86_windows", "win-x86")] + public void GetRidForConfig_AspNetCoreConfig_ReturnsMatchingRid(string configKey, string expectedRid) + { + // GetRidForConfig searches the union of both flavour maps; aspnetcore keys must resolve. + Assert.Equal(expectedRid, BuildCacheClient.GetRidForConfig(configKey)); + } + + [Fact] + public void OverlayPublishedOutput_AspNetCore_CopiesManagedAspNetDllsAndNoHost() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var configKey = AspNetConfigKeyForRid(rid); + var (extractDir, managed) = BuildFakeAspNetCoreBcsArchive(rid); + + var outputFolder = Path.Combine(_testDir, "published-aspnet"); + Directory.CreateDirectory(outputFolder); + + var copied = BuildCacheClient.OverlayPublishedOutput( + extractDir, outputFolder, configKey, "MyApp", BuildCacheClient.BuildCacheFlavor.AspNetCore); + + // Managed-only pack: every Microsoft.AspNetCore.*.dll is overlaid; no host binaries. + Assert.Equal(managed.Count, copied); + foreach (var dll in managed) + { + Assert.True(File.Exists(Path.Combine(outputFolder, dll)), $"Missing aspnetcore managed file {dll}"); + } + Assert.False(File.Exists(Path.Combine(outputFolder, BuildCacheClient.GetNativeLibName("hostpolicy")))); + } + + [Fact] + public void CreateBuildCacheDotnetHome_AspNetCore_OverlaysAspNetCoreAppNotRuntime() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var configKey = AspNetConfigKeyForRid(rid); + var (extractDir, managed) = BuildFakeAspNetCoreBcsArchive(rid); + + const string runtimeVersion = "11.0.0-preview.5.26256.117"; + const string aspNetCoreVersion = "11.0.0-preview.5.26256.117"; + var globalHome = BuildFakeGlobalDotnetHome(runtimeVersion, aspNetCoreVersion); + var commitSha = "aaaabbbbccccddddeeeeffff0000111122223333"; + + var bcsHome = BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir, runtimeVersion, aspNetCoreVersion, commitSha, configKey, + BuildCacheClient.BuildCacheFlavor.AspNetCore); + + try + { + // 1. BCS managed Microsoft.AspNetCore.*.dll overlaid into the per-job AspNetCore.App. + var bcsAspNet = Path.Combine(bcsHome, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); + foreach (var dll in managed) + { + Assert.True(File.Exists(Path.Combine(bcsAspNet, dll)), $"Missing BCS aspnetcore managed {dll}"); + } + + // 2. AspNetCore.App/.version rewritten with the BCS (aspnetcore) commit. + var aspNetVersion = File.ReadAllText(Path.Combine(bcsAspNet, ".version")); + Assert.Contains(commitSha, aspNetVersion); + Assert.Contains(aspNetCoreVersion, aspNetVersion); + + // 3. The base runtime (NETCore.App) is mirrored from the feed and its .version is + // NOT rewritten — the aspnetcore flavour overrides only the asp.net framework. + var netCoreVersion = File.ReadAllText(Path.Combine(bcsHome, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version")); + Assert.Contains("FEED_COMMIT", netCoreVersion); + Assert.DoesNotContain(commitSha, netCoreVersion); + + // 4. Global home untouched. + var globalAspNet = File.ReadAllText(Path.Combine(globalHome, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion, ".version")); + Assert.DoesNotContain(commitSha, globalAspNet); + } + finally + { + try { Directory.Delete(bcsHome, recursive: true); } catch { } + } + } + + [Fact] + public void CreateBuildCacheDotnetHome_AspNetCore_MissingAspNetVersion_Throws() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var configKey = AspNetConfigKeyForRid(rid); + var (extractDir, _) = BuildFakeAspNetCoreBcsArchive(rid); + + const string runtimeVersion = "11.0.0-preview.5"; + var globalHome = BuildFakeGlobalDotnetHome(runtimeVersion, runtimeVersion); + + Assert.Throws(() => + BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir, runtimeVersion, aspNetCoreVersion: "", + "abcdef0123456789", configKey, BuildCacheClient.BuildCacheFlavor.AspNetCore)); + } + // ------------------------------------------------------------------- // CleanupExtractDir // ------------------------------------------------------------------- @@ -574,5 +723,35 @@ private static string NativeLibForRid(string rid, string baseName) } return $"lib{baseName}.so"; } + + private static string AspNetConfigKeyForRid(string rid) + => BuildCacheClient.PlatformToBcsConfigAspNetCore.TryGetValue(rid, out var v) ? v.configKey : null; + + /// + /// Builds a fake aspnetcore BCS extraction at + /// microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net11.0/ with + /// managed Microsoft.AspNetCore.*.dll files. The aspnetcore runtime pack is managed-only: + /// no native dir, no corehost (mirrors dotnet/performance#5243's pack-bcs-archives.ps1). + /// + private (string extractDir, List managed) BuildFakeAspNetCoreBcsArchive(string rid) + { + var extractDir = Path.Combine(_testDir, "extracted-aspnet-" + Guid.NewGuid().ToString("N")); + var nugetPkg = Path.Combine(extractDir, $"microsoft.aspnetcore.app.runtime.{rid}"); + var libDir = Path.Combine(nugetPkg, "Release", "runtimes", rid, "lib", "net11.0"); + Directory.CreateDirectory(libDir); + + var managed = new List + { + "Microsoft.AspNetCore.dll", + "Microsoft.AspNetCore.Mvc.Core.dll", + "Microsoft.AspNetCore.Routing.dll", + }; + foreach (var dll in managed) + { + File.WriteAllText(Path.Combine(libDir, dll), "BCS aspnetcore managed " + dll); + } + + return (extractDir, managed); + } } } From 7b597c82d8aaf4c727ae53689549904a5c13df2f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 23 Jun 2026 15:41:31 -0700 Subject: [PATCH 6/7] Store aspnetcore BCS artifact as the raw runtime-pack nupkg Switch the aspnetcore build-cache flavour from a transformed microsoft.aspnetcore.app.runtime.{rid}/Release/... archive to the verbatim runtime-pack nupkg. A nupkg is a zip on every OS, so it extracts with the same ZipFile path and exposes runtimes/{rid}/... at the archive root (no Release wrapper). This keeps the full shipped artifact (crank already filters managed/ native at consume time) instead of a lossy projection. - PlatformToBcsConfigAspNetCore artifact files -> .nupkg - CreateBuildCacheDotnetHome / OverlayPublishedOutput aspnetcore overlay reads runtimes/{rid} from the extract root; runtime flavour unchanged - ExtractArchiveAsync treats .nupkg as a zip - Tests + docs updated for the raw-nupkg layout Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/build_cache_requirements.md | 12 ++-- src/Microsoft.Crank.Agent/BuildCacheClient.cs | 60 ++++++++++--------- .../BuildCacheClientTests.cs | 23 ++++--- 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/docs/build_cache_requirements.md b/docs/build_cache_requirements.md index 209ce8285..2d896db82 100644 --- a/docs/build_cache_requirements.md +++ b/docs/build_cache_requirements.md @@ -57,19 +57,21 @@ microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/native/ → na {rid}.Release/corehost/ → host binaries (dotnet, libhostfxr, libhostpolicy) ``` -For **aspnetcore** artifacts crank uses the parallel convention (managed-only — the ASP.NET Core -runtime pack ships no host binaries, and native is optional): +For **aspnetcore** artifacts the stored blob is the **verbatim runtime-pack nupkg** (a zip), so the +layout is the nupkg's own — `runtimes/{rid}` sits at the archive root, with no +`microsoft.aspnetcore.app.runtime.{rid}/Release` wrapper (managed-only — the ASP.NET Core runtime +pack ships no host binaries, and native is optional): ``` -microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net{X}.0/ → managed Microsoft.AspNetCore.*.dll -microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/native/ → native libs (optional) +runtimes/{rid}/lib/net{X}.0/ → managed Microsoft.AspNetCore.*.dll +runtimes/{rid}/native/ → native libs (optional) ``` Where `{rid}` = `linux-x64`, `linux-arm64`, `win-x64`, `win-arm64`, `win-x86`. (aspnetcore v1 has no musl/osx/arm32 configs.) The runtime layout was confirmed by inspecting `BuildArtifacts_linux_arm64_Release_coreclr.tar.gz`; -the aspnetcore layout is asserted by dotnet/performance's `pack-bcs-archives.ps1`. +the aspnetcore artifact is the raw nupkg produced by dotnet/performance's `stage-bcs-nupkg-aspnetcore.ps1`. **If either layout changes in future builds, the crank extraction will break.** Consider treating it as a stable contract or documenting it. diff --git a/src/Microsoft.Crank.Agent/BuildCacheClient.cs b/src/Microsoft.Crank.Agent/BuildCacheClient.cs index cdb80193f..3e564e33a 100644 --- a/src/Microsoft.Crank.Agent/BuildCacheClient.cs +++ b/src/Microsoft.Crank.Agent/BuildCacheClient.cs @@ -60,19 +60,20 @@ internal static class BuildCacheClient /// /// ASP.NET Core (dotnet/aspnetcore) variant of . Maps the /// agent's platform (RID) to the aspnetcore BCS configuration key and artifact filename. - /// The configKey / artifact tokens and the archive's internal layout - /// (microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/...) are a - /// load-bearing contract asserted by dotnet/performance's pack-bcs-archives.ps1. + /// The stored artifact is the verbatim runtime-pack nupkg, so the archive's internal layout + /// is the nupkg's own (runtimes/{rid}/lib/net{X}.0/... at the root). This and the + /// configKey / artifact tokens are a load-bearing contract produced by dotnet/performance's + /// stage-bcs-nupkg-aspnetcore.ps1. /// v1 has no musl/osx/arm32 entries. /// internal static readonly IReadOnlyDictionary PlatformToBcsConfigAspNetCore = new Dictionary(StringComparer.OrdinalIgnoreCase) { - ["linux-x64"] = ("aspnetcore_x64_linux", "BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz", "linux-x64"), - ["linux-arm64"] = ("aspnetcore_arm64_linux", "BuildArtifacts_linux_arm64_Release_aspnetcore.tar.gz", "linux-arm64"), - ["win-x64"] = ("aspnetcore_x64_windows", "BuildArtifacts_windows_x64_Release_aspnetcore.zip", "win-x64"), - ["win-arm64"] = ("aspnetcore_arm64_windows", "BuildArtifacts_windows_arm64_Release_aspnetcore.zip", "win-arm64"), - ["win-x86"] = ("aspnetcore_x86_windows", "BuildArtifacts_windows_x86_Release_aspnetcore.zip", "win-x86"), + ["linux-x64"] = ("aspnetcore_x64_linux", "BuildArtifacts_linux_x64_Release_aspnetcore.nupkg", "linux-x64"), + ["linux-arm64"] = ("aspnetcore_arm64_linux", "BuildArtifacts_linux_arm64_Release_aspnetcore.nupkg", "linux-arm64"), + ["win-x64"] = ("aspnetcore_x64_windows", "BuildArtifacts_windows_x64_Release_aspnetcore.nupkg", "win-x64"), + ["win-arm64"] = ("aspnetcore_arm64_windows", "BuildArtifacts_windows_arm64_Release_aspnetcore.nupkg", "win-arm64"), + ["win-x86"] = ("aspnetcore_x86_windows", "BuildArtifacts_windows_x86_Release_aspnetcore.nupkg", "win-x86"), }; /// @@ -373,15 +374,13 @@ public static string CreateBuildCacheDotnetHome( // per-job ASP.NET Core shared framework. The aspnetcore runtime pack is // essentially managed-only; native is optional (CopyNative returns 0 if absent) // and there are no host binaries to overlay (the feed runtime/host stay in place). - var aspNetPackageDir = FindDirectory(extractDir, $"microsoft.aspnetcore.app.runtime.{rid}"); - if (aspNetPackageDir != null) + // The stored artifact is the raw runtime-pack nupkg, so runtimes/{rid} sits at the + // archive root (no microsoft.aspnetcore.app.runtime.{rid}/Release wrapper). + var runtimesDir = Path.Combine(extractDir, "runtimes", rid); + if (Directory.Exists(runtimesDir)) { - var runtimesDir = Path.Combine(aspNetPackageDir, "Release", "runtimes", rid); - if (Directory.Exists(runtimesDir)) - { - filesOverlaid += CopyManaged(runtimesDir, dstAspNet); - filesOverlaid += CopyNative(runtimesDir, dstAspNet); - } + filesOverlaid += CopyManaged(runtimesDir, dstAspNet); + filesOverlaid += CopyNative(runtimesDir, dstAspNet); } } else @@ -472,19 +471,23 @@ public static int OverlayPublishedOutput( var rid = GetRidForConfig(buildCacheConfig); int filesCopied = 0; - var packageName = flavor == BuildCacheFlavor.AspNetCore - ? $"microsoft.aspnetcore.app.runtime.{rid}" - : $"microsoft.netcore.app.runtime.{rid}"; + // The aspnetcore artifact is the raw runtime-pack nupkg (runtimes/{rid} at the archive + // root); the runtime artifact wraps it in microsoft.netcore.app.runtime.{rid}/Release. + string runtimesDir; + if (flavor == BuildCacheFlavor.AspNetCore) + { + runtimesDir = Path.Combine(extractDir, "runtimes", rid); + } + else + { + var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); + runtimesDir = nugetPackageDir != null ? Path.Combine(nugetPackageDir, "Release", "runtimes", rid) : null; + } - var nugetPackageDir = FindDirectory(extractDir, packageName); - if (nugetPackageDir != null) + if (runtimesDir != null && Directory.Exists(runtimesDir)) { - var runtimesDir = Path.Combine(nugetPackageDir, "Release", "runtimes", rid); - if (Directory.Exists(runtimesDir)) - { - filesCopied += CopyManaged(runtimesDir, outputFolder); - filesCopied += CopyNative(runtimesDir, outputFolder); - } + filesCopied += CopyManaged(runtimesDir, outputFolder); + filesCopied += CopyNative(runtimesDir, outputFolder); } // The aspnetcore pack ships no host binaries, so host overlay only applies to runtime. @@ -697,7 +700,8 @@ private static Task ExtractArchiveAsync(string archivePath, string outputDir, Ca return ExtractTarGzAsync(archivePath, outputDir, cancellationToken); } - if (archivePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + if (archivePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) || + archivePath.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase)) { return Task.Run(() => ZipFile.ExtractToDirectory(archivePath, outputDir, overwriteFiles: true), cancellationToken); } diff --git a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs index 382391efc..cdadb5ae0 100644 --- a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs +++ b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs @@ -206,16 +206,16 @@ public void GetRidForConfig_UnknownConfig_Throws() // ------------------------------------------------------------------- // ASP.NET Core config map (locked external contract — see - // dotnet/performance pack-bcs-archives.ps1). Pins configKey / RID / + // dotnet/performance stage-bcs-nupkg-aspnetcore.ps1). Pins configKey / RID / // artifact filename so an accidental token change is caught. // ------------------------------------------------------------------- [Theory] - [InlineData("linux-x64", "aspnetcore_x64_linux", "BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz")] - [InlineData("linux-arm64", "aspnetcore_arm64_linux", "BuildArtifacts_linux_arm64_Release_aspnetcore.tar.gz")] - [InlineData("win-x64", "aspnetcore_x64_windows", "BuildArtifacts_windows_x64_Release_aspnetcore.zip")] - [InlineData("win-arm64", "aspnetcore_arm64_windows", "BuildArtifacts_windows_arm64_Release_aspnetcore.zip")] - [InlineData("win-x86", "aspnetcore_x86_windows", "BuildArtifacts_windows_x86_Release_aspnetcore.zip")] + [InlineData("linux-x64", "aspnetcore_x64_linux", "BuildArtifacts_linux_x64_Release_aspnetcore.nupkg")] + [InlineData("linux-arm64", "aspnetcore_arm64_linux", "BuildArtifacts_linux_arm64_Release_aspnetcore.nupkg")] + [InlineData("win-x64", "aspnetcore_x64_windows", "BuildArtifacts_windows_x64_Release_aspnetcore.nupkg")] + [InlineData("win-arm64", "aspnetcore_arm64_windows", "BuildArtifacts_windows_arm64_Release_aspnetcore.nupkg")] + [InlineData("win-x86", "aspnetcore_x86_windows", "BuildArtifacts_windows_x86_Release_aspnetcore.nupkg")] public void PlatformToBcsConfigAspNetCore_MatchesLockedContract(string rid, string expectedConfigKey, string expectedArtifact) { Assert.True(BuildCacheClient.PlatformToBcsConfigAspNetCore.TryGetValue(rid, out var entry), $"Missing aspnetcore entry for '{rid}'."); @@ -728,16 +728,15 @@ private static string AspNetConfigKeyForRid(string rid) => BuildCacheClient.PlatformToBcsConfigAspNetCore.TryGetValue(rid, out var v) ? v.configKey : null; /// - /// Builds a fake aspnetcore BCS extraction at - /// microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}/lib/net11.0/ with - /// managed Microsoft.AspNetCore.*.dll files. The aspnetcore runtime pack is managed-only: - /// no native dir, no corehost (mirrors dotnet/performance#5243's pack-bcs-archives.ps1). + /// Builds a fake aspnetcore BCS extraction at runtimes/{rid}/lib/net11.0/ (the raw + /// runtime-pack nupkg layout) with managed Microsoft.AspNetCore.*.dll files. The aspnetcore + /// runtime pack is managed-only: no native dir, no corehost (mirrors dotnet/performance#5243's + /// stage-bcs-nupkg-aspnetcore.ps1). /// private (string extractDir, List managed) BuildFakeAspNetCoreBcsArchive(string rid) { var extractDir = Path.Combine(_testDir, "extracted-aspnet-" + Guid.NewGuid().ToString("N")); - var nugetPkg = Path.Combine(extractDir, $"microsoft.aspnetcore.app.runtime.{rid}"); - var libDir = Path.Combine(nugetPkg, "Release", "runtimes", rid, "lib", "net11.0"); + var libDir = Path.Combine(extractDir, "runtimes", rid, "lib", "net11.0"); Directory.CreateDirectory(libDir); var managed = new List From b040c99e29216ff7c8b742fb37945428e0ba52c8 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 24 Jun 2026 13:45:21 -0700 Subject: [PATCH 7/7] Place aspnetcore BCS framework directly from the verbatim nupkg Builds on the previous commit (which switched the aspnetcore artifact to the verbatim runtime-pack nupkg and fixed the .nupkg extension / extractor / bare runtimes/{rid} layout). The verbatim nupkg carries the host-resolvable Microsoft.AspNetCore.App.deps.json + runtimeconfig.json next to the managed assemblies, so the per-job dotnet home can build the ASP.NET Core shared framework ENTIRELY from BCS instead of overlaying onto a feed copy. - CreateBuildCacheDotnetHome: for the aspnetcore flavour, place Microsoft.AspNetCore.App directly from the pack (all managed incl deps.json + runtimeconfig.json + native, synthesized .version); do not clone the feed copy of that framework. Fail loud (BuildCacheIncompleteException) if the pack is missing managed assemblies, deps.json, or runtimeconfig.json -- for perf runs, erroring beats silently running mixed/feed bits. Base runtime + host stay feed-cloned. New PlaceAspNetFrameworkFromPack + ResolveAspNetRuntimesDir (bare layout, wrapped fallback). - OverlayPublishedOutput (SCD): unchanged behaviour (managed *.dll + native only, app's own .deps.json governs); now resolves the pack via ResolveAspNetRuntimesDir. - Runtime flavour: overlay path unchanged. - Tests: aspnetcore archive helper now includes deps.json + runtimeconfig.json; FDD test asserts pristine direct-placement (metadata present, no feed leak); +2 fail-loud negatives. Docs updated. 95/95 unit tests green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/build_cache_requirements.md | 36 +++- docs/dotnet_versions.md | 6 +- src/Microsoft.Crank.Agent/BuildCacheClient.cs | 189 ++++++++++++++---- src/Microsoft.Crank.Agent/Startup.cs | 13 +- .../BuildCacheClientTests.cs | 80 +++++++- 5 files changed, 258 insertions(+), 66 deletions(-) diff --git a/docs/build_cache_requirements.md b/docs/build_cache_requirements.md index 2d896db82..45e53a587 100644 --- a/docs/build_cache_requirements.md +++ b/docs/build_cache_requirements.md @@ -25,7 +25,7 @@ Where: - `repoName` = `runtime` or `aspnetcore` (selected per-job via the `buildCacheRepo` property; defaults to the agent's `--build-cache-repo-name`) - `branch` = e.g., `main`, `release/10.0` - `configKey` = e.g., `coreclr_x64_linux`, `coreclr_arm64_windows` (runtime); `aspnetcore_x64_linux`, `aspnetcore_arm64_windows` (aspnetcore) -- `artifactFile` = e.g., `BuildArtifacts_linux_x64_Release_coreclr.tar.gz` (runtime); `BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz` (aspnetcore) +- `artifactFile` = e.g., `BuildArtifacts_linux_x64_Release_coreclr.tar.gz` (runtime); `BuildArtifacts_linux_x64_Release_aspnetcore.nupkg` (aspnetcore — the verbatim runtime-pack nupkg) The aspnetcore `latestBuilds.json` lives at `builds/aspnetcore/latest/{branch}/latestBuilds.json` and contains only the 5 `aspnetcore_*` config keys plus an `all` entry and `branch_name`; it does **not** carry the runtime `coreclr_*` keys. Crank's parser enumerates keys dynamically, so it tolerates either repo's file. @@ -57,21 +57,39 @@ microsoft.netcore.app.runtime.{rid}/Release/runtimes/{rid}/native/ → na {rid}.Release/corehost/ → host binaries (dotnet, libhostfxr, libhostpolicy) ``` -For **aspnetcore** artifacts the stored blob is the **verbatim runtime-pack nupkg** (a zip), so the -layout is the nupkg's own — `runtimes/{rid}` sits at the archive root, with no -`microsoft.aspnetcore.app.runtime.{rid}/Release` wrapper (managed-only — the ASP.NET Core runtime -pack ships no host binaries, and native is optional): +For **aspnetcore** artifacts the stored blob is the **verbatim runtime-pack nupkg** (a zip, +extension `.nupkg`), so the layout is the nupkg's own — `runtimes/{rid}` sits at the archive root, +with no `microsoft.aspnetcore.app.runtime.{rid}/Release` wrapper. Crucially, the verbatim nupkg +carries the host-resolvable framework metadata next to the managed assemblies: ``` -runtimes/{rid}/lib/net{X}.0/ → managed Microsoft.AspNetCore.*.dll -runtimes/{rid}/native/ → native libs (optional) +runtimes/{rid}/lib/net{X}.0/Microsoft.AspNetCore.*.dll → managed assemblies +runtimes/{rid}/lib/net{X}.0/Microsoft.AspNetCore.App.deps.json → host-resolvable metadata (REQUIRED) +runtimes/{rid}/lib/net{X}.0/Microsoft.AspNetCore.App.runtimeconfig.json → host-resolvable metadata (REQUIRED) +runtimes/{rid}/native/ → native libs (optional) ``` +Because the nupkg is a complete framework, crank **places `Microsoft.AspNetCore.App` directly** from +the pack into the per-job dotnet home (the whole managed set incl `deps.json`/`runtimeconfig.json`, +no feed contribution) and **fails the job** (`BuildCacheIncompleteException`) if the pack is missing +managed assemblies, `deps.json`, or `runtimeconfig.json` — for perf runs, erroring is preferable to +silently running mixed/feed bits. The base runtime + host stay feed-resolved (the aspnetcore pack +ships neither). Self-contained (SCD) publishes are the exception: the framework is co-mingled with +the app under the app's own `.deps.json`, so for SCD crank overlays only the managed `*.dll` +(+ native), not the framework metadata. + +This differs from **runtime**, whose archive is raw build output (no shared-framework +`deps.json`/`runtimeconfig.json`), so the runtime flavour overlays BCS binaries onto a feed-installed +runtime (reusing the feed's metadata) rather than placing directly. + Where `{rid}` = `linux-x64`, `linux-arm64`, `win-x64`, `win-arm64`, `win-x86`. (aspnetcore v1 has no musl/osx/arm32 configs.) -The runtime layout was confirmed by inspecting `BuildArtifacts_linux_arm64_Release_coreclr.tar.gz`; -the aspnetcore artifact is the raw nupkg produced by dotnet/performance's `stage-bcs-nupkg-aspnetcore.ps1`. +The runtime layout was confirmed by inspecting `BuildArtifacts_linux_x64_Release_coreclr.tar.gz` (no +framework metadata in the pack lib). The aspnetcore contract — a verbatim runtime-pack nupkg carrying +managed + `deps.json` + `runtimeconfig.json`, uploaded as `BuildArtifacts_{os}_{arch}_Release_aspnetcore.nupkg` +— was confirmed against a locally-built `Microsoft.AspNetCore.App.Runtime.win-x64.nupkg` and +dotnet/performance#5243's `stage-bcs-nupkg-aspnetcore.ps1`. **If either layout changes in future builds, the crank extraction will break.** Consider treating it as a stable contract or documenting it. diff --git a/docs/dotnet_versions.md b/docs/dotnet_versions.md index e7e6a218b..12dfdee75 100644 --- a/docs/dotnet_versions.md +++ b/docs/dotnet_versions.md @@ -126,8 +126,8 @@ The `buildcache` channel resolves pre-built binaries for individual commits from The channel can resolve from one of two repositories, selected per-job with the `buildCacheRepo` property: -- `runtime` (default) — overlays the base .NET runtime (`Microsoft.NETCore.App`) with BCS bits built from a [dotnet/runtime](https://github.com/dotnet/runtime) commit. -- `aspnetcore` — overlays the ASP.NET Core shared framework (`Microsoft.AspNetCore.App`) with BCS bits built from a [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore) commit. The base runtime stays at the feed-resolved version. +- `runtime` (default) — **overlays** the base .NET runtime (`Microsoft.NETCore.App`) with BCS bits built from a [dotnet/runtime](https://github.com/dotnet/runtime) commit. The runtime archive is raw build output (no shared-framework metadata), so BCS binaries are overlaid onto a feed-installed runtime. +- `aspnetcore` — **places** the ASP.NET Core shared framework (`Microsoft.AspNetCore.App`) directly from a [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore) commit's BCS build. The aspnetcore archive is the runtime-pack nupkg stored verbatim (carrying `deps.json` + `runtimeconfig.json`), so the framework folder is built entirely from BCS and the job **fails** if the pack is incomplete. The base runtime stays at the feed-resolved version. When no `buildCacheRepo` is supplied the job falls back to the agent-level `--build-cache-repo-name` (which itself defaults to `runtime`), so existing runtime usage is unchanged. @@ -176,7 +176,7 @@ If the commit is not found in the cache, crank will fail with an error rather th | Property | Default | Description | |----------|---------|-------------| -| `buildCacheRepo` | (agent `--build-cache-repo-name`, i.e. `runtime`) | Which BCS repository to resolve from: `runtime` (overlays `Microsoft.NETCore.App`) or `aspnetcore` (overlays `Microsoft.AspNetCore.App`). | +| `buildCacheRepo` | (agent `--build-cache-repo-name`, i.e. `runtime`) | Which BCS repository to resolve from: `runtime` (overlays `Microsoft.NETCore.App` from a feed install) or `aspnetcore` (places `Microsoft.AspNetCore.App` directly from the verbatim runtime-pack nupkg; fails loud if incomplete). | | `buildCacheCommitSha` | (empty) | Specific commit SHA in the selected repo (runtime or aspnetcore). If empty, uses the latest cached build for the branch. | | `buildCacheBranch` | `main` | Branch to query for the latest build. | | `buildCacheConfig` | (auto-detected) | BCS configuration key (e.g., `coreclr_x64_linux` for runtime, `aspnetcore_x64_linux` for aspnetcore). Auto-detected from agent platform and the selected repo. | diff --git a/src/Microsoft.Crank.Agent/BuildCacheClient.cs b/src/Microsoft.Crank.Agent/BuildCacheClient.cs index 3e564e33a..74bd87bd5 100644 --- a/src/Microsoft.Crank.Agent/BuildCacheClient.cs +++ b/src/Microsoft.Crank.Agent/BuildCacheClient.cs @@ -128,6 +128,17 @@ public class BuildCacheNotFoundException : InvalidOperationException public BuildCacheNotFoundException(string message) : base(message) { } } + /// + /// Thrown when a BCS archive is present but does not contain a complete framework + /// (missing managed assemblies, deps.json/runtimeconfig.json). Used by the aspnetcore + /// direct-placement path, which deliberately fails loudly rather than producing a partial + /// framework — for perf runs, silently running an incomplete framework is worse than erroring. + /// + public class BuildCacheIncompleteException : InvalidOperationException + { + public BuildCacheIncompleteException(string message) : base(message) { } + } + /// /// Validates a user-supplied commit SHA. Accepts 8-40 lowercase/uppercase hex chars. /// @@ -284,16 +295,20 @@ public static void CleanupExtractDir(string extractDir) } /// - /// Builds a per-job dotnet home that mirrors the relevant subtrees of the global dotnet - /// home (runtime, asp.net, host) and overlays BCS bits on top. The global dotnet home is - /// NOT modified, so concurrent jobs and subsequent non-buildcache jobs are unaffected. + /// Builds a per-job dotnet home. The global dotnet home is NOT modified, so concurrent jobs + /// and subsequent non-buildcache jobs are unaffected. + /// + /// For the home mirrors the feed runtime/asp.net/host + /// subtrees and OVERLAYS BCS bits onto the base runtime (Microsoft.NETCore.App) + host. The + /// runtime archive is raw build output and does not carry the framework's deps.json/ + /// runtimeconfig.json, so overlay-onto-feed is required (it reuses the feed metadata). /// - /// For the BCS bits overlay the base runtime - /// (Microsoft.NETCore.App) plus host binaries. For - /// the BCS bits overlay the ASP.NET Core shared framework (Microsoft.AspNetCore.App) only; - /// the aspnetcore runtime pack is managed-only and ships no host binaries, so the feed - /// runtime/host are left in place and only the managed Microsoft.AspNetCore.*.dll set is - /// replaced. + /// For the ASP.NET Core shared framework + /// (Microsoft.AspNetCore.App) is PLACED DIRECTLY from the BCS pack — the aspnetcore archive is + /// the runtime-pack nupkg stored verbatim, which carries the managed assemblies AND the + /// host-resolvable deps.json/runtimeconfig.json, so the framework folder is built entirely from + /// BCS (no feed contribution) and fails loud if incomplete. The base runtime + host stay + /// feed-cloned (the aspnetcore pack ships neither). /// /// Absolute path to the per-job dotnet home root. Caller owns it. public static string CreateBuildCacheDotnetHome( @@ -354,10 +369,14 @@ public static string CreateBuildCacheDotnetHome( } // 4. Mirror shared/Microsoft.AspNetCore.App/{aspNetCoreVersion}. + // For the RUNTIME flavour this is the non-overridden framework, cloned from the feed. + // For the ASPNETCORE flavour we do NOT clone the feed copy — the framework is placed + // directly from the BCS pack below, so no feed assembly can leak into the framework + // that actually runs. var dstAspNet = string.IsNullOrEmpty(aspNetCoreVersion) ? null : Path.Combine(bcsHomeRoot, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); - if (!string.IsNullOrEmpty(aspNetCoreVersion)) + if (flavor == BuildCacheFlavor.Runtime && !string.IsNullOrEmpty(aspNetCoreVersion)) { var srcAspNet = Path.Combine(globalDotnetHome, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); if (Directory.Exists(srcAspNet)) @@ -366,24 +385,32 @@ public static string CreateBuildCacheDotnetHome( } } - int filesOverlaid = 0; - if (flavor == BuildCacheFlavor.AspNetCore) { - // 5a. Overlay BCS managed (+ optional native) Microsoft.AspNetCore.*.dll into the - // per-job ASP.NET Core shared framework. The aspnetcore runtime pack is - // essentially managed-only; native is optional (CopyNative returns 0 if absent) - // and there are no host binaries to overlay (the feed runtime/host stay in place). - // The stored artifact is the raw runtime-pack nupkg, so runtimes/{rid} sits at the - // archive root (no microsoft.aspnetcore.app.runtime.{rid}/Release wrapper). - var runtimesDir = Path.Combine(extractDir, "runtimes", rid); - if (Directory.Exists(runtimesDir)) + // 5a. Place the ASP.NET Core shared framework DIRECTLY from the BCS pack. The aspnetcore + // artifact is the runtime-pack nupkg stored verbatim, so its bare runtimes/{rid}/lib/ + // net{X}.0/ already carries the managed assemblies AND the host-resolvable + // Microsoft.AspNetCore.App.deps.json + runtimeconfig.json. We build the framework + // folder entirely from that (no feed clone) and synthesize .version; the base runtime + // + host stay feed-cloned (steps 1-3) since the aspnetcore pack ships neither. Fails + // loud if the placed framework is incomplete. + try + { + PlaceAspNetFrameworkFromPack(extractDir, dstAspNet, rid, aspNetCoreVersion, commitSha); + } + catch { - filesOverlaid += CopyManaged(runtimesDir, dstAspNet); - filesOverlaid += CopyNative(runtimesDir, dstAspNet); + try { Directory.Delete(bcsHomeRoot, recursive: true); } catch { } + throw; } + + Log.Info($"Build Cache: Per-job dotnet home built at {bcsHomeRoot} (Microsoft.AspNetCore.App placed directly from BCS commit {ShortSha(commitSha)})"); + return bcsHomeRoot; } - else + + // ===== Runtime flavour: overlay BCS bits onto the feed-cloned base runtime. ===== + int filesOverlaid = 0; + { // 5. Overlay BCS managed + native into the per-job NETCore.App. var nugetPackageDir = FindDirectory(extractDir, $"microsoft.netcore.app.runtime.{rid}"); @@ -428,24 +455,109 @@ public static string CreateBuildCacheDotnetHome( "The archive layout may have changed or the platform is not supported."); } - // 7. Rewrite the overlaid framework's .version so any consumer (the agent's own - // version measurement, GetDependencies, etc.) reports the BCS commit. For aspnetcore - // that is the Microsoft.AspNetCore.App folder; for runtime it is Microsoft.NETCore.App. - if (flavor == BuildCacheFlavor.AspNetCore) + // 7. Rewrite the overlaid runtime's .version so any consumer (the agent's own version + // measurement, GetDependencies, etc.) reports the BCS commit. + File.WriteAllText( + Path.Combine(dstNetCoreApp, ".version"), + $"{commitSha}\n{runtimeVersion}\n"); + + Log.Info($"Build Cache: Per-job dotnet home built at {bcsHomeRoot} ({filesOverlaid} BCS files overlaid)"); + return bcsHomeRoot; + } + + /// + /// Resolves the runtimes/{rid} directory inside an extracted aspnetcore archive. The + /// aspnetcore archive is the runtime-pack nupkg stored verbatim, so the directory lives at the + /// archive root (runtimes/{rid}). Falls back to the wrapped build-output layout + /// (microsoft.aspnetcore.app.runtime.{rid}/Release/runtimes/{rid}) for resilience. + /// + private static string ResolveAspNetRuntimesDir(string extractDir, string rid) + { + var bare = Path.Combine(extractDir, "runtimes", rid); + if (Directory.Exists(bare)) { - File.WriteAllText( - Path.Combine(dstAspNet, ".version"), - $"{commitSha}\n{aspNetCoreVersion}\n"); + return bare; } - else + + var pkg = FindDirectory(extractDir, $"microsoft.aspnetcore.app.runtime.{rid}"); + if (pkg != null) { - File.WriteAllText( - Path.Combine(dstNetCoreApp, ".version"), - $"{commitSha}\n{runtimeVersion}\n"); + var wrapped = Path.Combine(pkg, "Release", "runtimes", rid); + if (Directory.Exists(wrapped)) + { + return wrapped; + } } - Log.Info($"Build Cache: Per-job dotnet home built at {bcsHomeRoot} ({filesOverlaid} BCS files overlaid)"); - return bcsHomeRoot; + return null; + } + + /// + /// Builds the ASP.NET Core shared-framework folder directly from the verbatim runtime-pack + /// nupkg's runtimes/{rid}/lib/net{X}.0/ + runtimes/{rid}/native/. Copies the + /// WHOLE managed set (assemblies + Microsoft.AspNetCore.App.deps.json + runtimeconfig.json, + /// skipping only debug symbols), then synthesizes .version carrying the BCS commit. + /// Throws if the result is not a complete, + /// host-resolvable framework — no feed fallback. + /// + private static void PlaceAspNetFrameworkFromPack( + string extractDir, string destFrameworkDir, string rid, string version, string commitSha) + { + var runtimesDir = ResolveAspNetRuntimesDir(extractDir, rid); + if (runtimesDir == null) + { + throw new BuildCacheIncompleteException( + $"Build Cache: aspnetcore runtime pack (runtimes/{rid}) not found in archive for commit {ShortSha(commitSha)}. " + + "The archive may not be the expected verbatim Microsoft.AspNetCore.App.Runtime.{rid} nupkg."); + } + + var managedDir = SelectHighestManagedDir(Path.Combine(runtimesDir, "lib")); + if (managedDir == null) + { + throw new BuildCacheIncompleteException( + $"Build Cache: no managed lib/net*.0 directory in aspnetcore pack for commit {ShortSha(commitSha)} (rid '{rid}')."); + } + + Directory.CreateDirectory(destFrameworkDir); + + // Copy the ENTIRE managed set (assemblies + deps.json + runtimeconfig.json). This is the + // load-bearing difference from an overlay: the verbatim nupkg carries the host-resolvable + // metadata, so the placed folder is a complete shared framework with no feed contribution. + foreach (var file in Directory.GetFiles(managedDir)) + { + var name = Path.GetFileName(file); + if (name.EndsWith(".pdb", StringComparison.OrdinalIgnoreCase) || + name.EndsWith(".dbg", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var dest = Path.Combine(destFrameworkDir, name); + File.Copy(file, dest, overwrite: true); + EnsureExecutable(dest); + } + + // Native (e.g. aspnetcorev2_inprocess on Windows) — optional; 0 if absent. + CopyNative(runtimesDir, destFrameworkDir); + + // Synthesize the host-resolvable .version carrying the BCS aspnetcore commit. + File.WriteAllText(Path.Combine(destFrameworkDir, ".version"), $"{commitSha}\n{version}\n"); + + // Fail-loud: a complete framework needs managed assemblies AND the host-resolvable metadata. + void Require(bool condition, string what) + { + if (!condition) + { + throw new BuildCacheIncompleteException( + $"Build Cache: directly-placed Microsoft.AspNetCore.App for commit {ShortSha(commitSha)} is incomplete — missing {what}. " + + "The BCS archive must be the complete runtime-pack nupkg (managed assemblies + deps.json + runtimeconfig.json). " + + "Failing the job rather than running an incomplete framework."); + } + } + + Require(Directory.GetFiles(destFrameworkDir, "*.dll").Length > 0, "managed assemblies (*.dll)"); + Require(File.Exists(Path.Combine(destFrameworkDir, "Microsoft.AspNetCore.App.deps.json")), "Microsoft.AspNetCore.App.deps.json"); + Require(File.Exists(Path.Combine(destFrameworkDir, "Microsoft.AspNetCore.App.runtimeconfig.json")), "Microsoft.AspNetCore.App.runtimeconfig.json"); } /// @@ -476,7 +588,7 @@ public static int OverlayPublishedOutput( string runtimesDir; if (flavor == BuildCacheFlavor.AspNetCore) { - runtimesDir = Path.Combine(extractDir, "runtimes", rid); + runtimesDir = ResolveAspNetRuntimesDir(extractDir, rid); } else { @@ -486,6 +598,9 @@ public static int OverlayPublishedOutput( if (runtimesDir != null && Directory.Exists(runtimesDir)) { + // SCD co-mingles the framework with the app in one folder governed by the app's own + // .deps.json, so overlay only the managed *.dll (+ native) — CopyManaged copies *.dll + // only, so the framework's deps.json/runtimeconfig.json are intentionally NOT copied. filesCopied += CopyManaged(runtimesDir, outputFolder); filesCopied += CopyNative(runtimesDir, outputFolder); } diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index aa63b8dc7..839ee1623 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -3072,13 +3072,14 @@ private static async Task CloneRestoreAndBuild(string path, Job job, str if (buildCacheFlavor == BuildCacheClient.BuildCacheFlavor.AspNetCore) { - // The ASP.NET Core shared-framework version folder is feed-resolved (Latest) and - // the BCS bits overlay onto it — mirroring how the runtime flavour overlays onto a - // feed-resolved runtimeVersion. Set "Latest" here so ResolveAspNetCoreVersion below - // turns it into a real version that names the shared/Microsoft.AspNetCore.App/{ver} - // overlay target. runtimeVersion already holds a real feed runtime version. + // The ASP.NET Core shared-framework VERSION (folder name) is feed-resolved + // (Latest) so PatchRuntimeConfig + the app's framework reference resolve to a real + // shared/Microsoft.AspNetCore.App/{ver} dir; the framework CONTENTS are then placed + // DIRECTLY from the BCS pack into that folder (the feed copy is not used). Set + // "Latest" here so ResolveAspNetCoreVersion below turns it into a real version. + // runtimeVersion already holds a real feed runtime version. aspNetCoreVersion = "Latest"; - Log.Info($"ASP.NET Core for build: Latest from feeds, will be overlaid with BCS commit {shortSha}"); + Log.Info($"ASP.NET Core for build: version resolved from feeds, framework placed directly from BCS commit {shortSha}"); // Q2: v1 overrides only one BCS component per job. If a runtime sentinel was also // requested, honour the selector (aspnetcore) and let the base runtime resolve normally. diff --git a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs index cdadb5ae0..50a94e6c4 100644 --- a/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs +++ b/test/Microsoft.Crank.UnitTests/BuildCacheClientTests.cs @@ -518,17 +518,20 @@ public void OverlayPublishedOutput_AspNetCore_CopiesManagedAspNetDllsAndNoHost() var copied = BuildCacheClient.OverlayPublishedOutput( extractDir, outputFolder, configKey, "MyApp", BuildCacheClient.BuildCacheFlavor.AspNetCore); - // Managed-only pack: every Microsoft.AspNetCore.*.dll is overlaid; no host binaries. + // SCD overlay copies managed *.dll only (the app's own .deps.json governs, so the + // framework's deps.json/runtimeconfig.json are NOT copied); no host binaries. Assert.Equal(managed.Count, copied); foreach (var dll in managed) { Assert.True(File.Exists(Path.Combine(outputFolder, dll)), $"Missing aspnetcore managed file {dll}"); } + Assert.False(File.Exists(Path.Combine(outputFolder, "Microsoft.AspNetCore.App.deps.json"))); + Assert.False(File.Exists(Path.Combine(outputFolder, "Microsoft.AspNetCore.App.runtimeconfig.json"))); Assert.False(File.Exists(Path.Combine(outputFolder, BuildCacheClient.GetNativeLibName("hostpolicy")))); } [Fact] - public void CreateBuildCacheDotnetHome_AspNetCore_OverlaysAspNetCoreAppNotRuntime() + public void CreateBuildCacheDotnetHome_AspNetCore_PlacesAspNetCoreAppFromPackNotRuntime() { var rid = BuildCacheClient.GetPlatformMoniker(); var configKey = AspNetConfigKeyForRid(rid); @@ -545,20 +548,27 @@ public void CreateBuildCacheDotnetHome_AspNetCore_OverlaysAspNetCoreAppNotRuntim try { - // 1. BCS managed Microsoft.AspNetCore.*.dll overlaid into the per-job AspNetCore.App. var bcsAspNet = Path.Combine(bcsHome, "shared", "Microsoft.AspNetCore.App", aspNetCoreVersion); + + // 1. BCS managed Microsoft.AspNetCore.*.dll placed into the per-job AspNetCore.App. foreach (var dll in managed) { Assert.True(File.Exists(Path.Combine(bcsAspNet, dll)), $"Missing BCS aspnetcore managed {dll}"); } - // 2. AspNetCore.App/.version rewritten with the BCS (aspnetcore) commit. + // 1b. Pristine direct-placement: the pack's host-resolvable metadata is present and the + // feed-only marker did NOT leak (the folder is built purely from BCS). + Assert.True(File.Exists(Path.Combine(bcsAspNet, "Microsoft.AspNetCore.App.deps.json"))); + Assert.True(File.Exists(Path.Combine(bcsAspNet, "Microsoft.AspNetCore.App.runtimeconfig.json"))); + Assert.False(File.Exists(Path.Combine(bcsAspNet, "FeedOnlyAspNet.dll"))); + + // 2. AspNetCore.App/.version carries the BCS (aspnetcore) commit. var aspNetVersion = File.ReadAllText(Path.Combine(bcsAspNet, ".version")); Assert.Contains(commitSha, aspNetVersion); Assert.Contains(aspNetCoreVersion, aspNetVersion); - // 3. The base runtime (NETCore.App) is mirrored from the feed and its .version is - // NOT rewritten — the aspnetcore flavour overrides only the asp.net framework. + // 3. The base runtime (NETCore.App) is cloned from the feed and its .version is NOT + // rewritten — the aspnetcore flavour overrides only the asp.net framework. var netCoreVersion = File.ReadAllText(Path.Combine(bcsHome, "shared", "Microsoft.NETCore.App", runtimeVersion, ".version")); Assert.Contains("FEED_COMMIT", netCoreVersion); Assert.DoesNotContain(commitSha, netCoreVersion); @@ -573,6 +583,40 @@ public void CreateBuildCacheDotnetHome_AspNetCore_OverlaysAspNetCoreAppNotRuntim } } + [Fact] + public void CreateBuildCacheDotnetHome_AspNetCore_MissingDepsJson_Throws() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var configKey = AspNetConfigKeyForRid(rid); + var (extractDir, _) = BuildFakeAspNetCoreBcsArchive(rid, includeDeps: false); + + const string version = "11.0.0-preview.5"; + var globalHome = BuildFakeGlobalDotnetHome(version, version); + + var ex = Assert.Throws(() => + BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir, version, version, "abcdef0123456789", configKey, + BuildCacheClient.BuildCacheFlavor.AspNetCore)); + Assert.Contains("deps.json", ex.Message); + } + + [Fact] + public void CreateBuildCacheDotnetHome_AspNetCore_MissingRuntimeConfig_Throws() + { + var rid = BuildCacheClient.GetPlatformMoniker(); + var configKey = AspNetConfigKeyForRid(rid); + var (extractDir, _) = BuildFakeAspNetCoreBcsArchive(rid, includeRuntimeConfig: false); + + const string version = "11.0.0-preview.5"; + var globalHome = BuildFakeGlobalDotnetHome(version, version); + + var ex = Assert.Throws(() => + BuildCacheClient.CreateBuildCacheDotnetHome( + globalHome, extractDir, version, version, "abcdef0123456789", configKey, + BuildCacheClient.BuildCacheFlavor.AspNetCore)); + Assert.Contains("runtimeconfig.json", ex.Message); + } + [Fact] public void CreateBuildCacheDotnetHome_AspNetCore_MissingAspNetVersion_Throws() { @@ -641,6 +685,9 @@ private string BuildFakeGlobalDotnetHome(string runtimeVersion, string aspNetCor File.WriteAllText(Path.Combine(aspNetCoreApp, ".version"), "FEED_ASPNET\n" + aspNetCoreVersion + "\n"); File.WriteAllText(Path.Combine(aspNetCoreApp, "Microsoft.AspNetCore.dll"), "feed aspnet"); + // A feed-only asp.net assembly the BCS pack does NOT ship; when the aspnetcore framework is + // placed directly from BCS this must NOT appear in the per-job home. + File.WriteAllText(Path.Combine(aspNetCoreApp, "FeedOnlyAspNet.dll"), "feed only aspnet"); File.WriteAllText(Path.Combine(hostFxr, BuildCacheClient.GetNativeLibName("hostfxr")), "feed hostfxr"); @@ -728,12 +775,14 @@ private static string AspNetConfigKeyForRid(string rid) => BuildCacheClient.PlatformToBcsConfigAspNetCore.TryGetValue(rid, out var v) ? v.configKey : null; /// - /// Builds a fake aspnetcore BCS extraction at runtimes/{rid}/lib/net11.0/ (the raw - /// runtime-pack nupkg layout) with managed Microsoft.AspNetCore.*.dll files. The aspnetcore - /// runtime pack is managed-only: no native dir, no corehost (mirrors dotnet/performance#5243's - /// stage-bcs-nupkg-aspnetcore.ps1). + /// Builds a fake aspnetcore BCS extraction at runtimes/{rid}/lib/net11.0/ (the verbatim + /// runtime-pack nupkg layout) with managed Microsoft.AspNetCore.*.dll files PLUS the + /// host-resolvable Microsoft.AspNetCore.App.deps.json + runtimeconfig.json (which the real + /// nupkg carries next to the assemblies). Negative-case flags omit the metadata to exercise the + /// direct-placement fail-loud path. /// - private (string extractDir, List managed) BuildFakeAspNetCoreBcsArchive(string rid) + private (string extractDir, List managed) BuildFakeAspNetCoreBcsArchive( + string rid, bool includeDeps = true, bool includeRuntimeConfig = true) { var extractDir = Path.Combine(_testDir, "extracted-aspnet-" + Guid.NewGuid().ToString("N")); var libDir = Path.Combine(extractDir, "runtimes", rid, "lib", "net11.0"); @@ -750,6 +799,15 @@ private static string AspNetConfigKeyForRid(string rid) File.WriteAllText(Path.Combine(libDir, dll), "BCS aspnetcore managed " + dll); } + if (includeDeps) + { + File.WriteAllText(Path.Combine(libDir, "Microsoft.AspNetCore.App.deps.json"), "{}"); + } + if (includeRuntimeConfig) + { + File.WriteAllText(Path.Combine(libDir, "Microsoft.AspNetCore.App.runtimeconfig.json"), "{}"); + } + return (extractDir, managed); } }