From 983b1cdfa30756e15293ebec79c2da4879658ba9 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Wed, 13 May 2026 18:50:31 +0200 Subject: [PATCH 01/29] feat: CommunityToolkit.Aspire.Hosting.K3s --- .github/workflows/tests.yaml | 1 + CommunityToolkit.Aspire.slnx | 5 + Directory.Packages.props | 1 + README.md | 6 + ...yToolkit.Aspire.Hosting.K3s.AppHost.csproj | 14 + .../Program.cs | 34 ++ .../Properties/launchSettings.json | 30 ++ .../Annotations/HelmReleaseAnnotation.cs | 35 ++ .../KubernetesDashboardAnnotation.cs | 11 + .../Annotations/KustomizeAnnotation.cs | 11 + ...CommunityToolkit.Aspire.Hosting.K3s.csproj | 19 + .../HelmReleaseHealthCheck.cs | 19 + .../HelmReleaseResource.cs | 45 ++ .../K3sAgentResource.cs | 22 + .../K3sBuilderExtensions.Helm.cs | 427 ++++++++++++++++ .../K3sBuilderExtensions.Manifest.cs | 400 +++++++++++++++ .../K3sBuilderExtensions.cs | 464 ++++++++++++++++++ .../K3sClusterOptions.cs | 40 ++ .../K3sClusterResource.cs | 63 +++ .../K3sContainerImageTags.cs | 13 + .../K3sInProcessPortForwarder.cs | 108 ++++ .../K3sReadinessHealthCheck.cs | 190 +++++++ .../K8sManifestResource.cs | 47 ++ .../KubeconfigInjectionStrategy.cs | 33 ++ .../README.md | 67 +++ .../CommunityToolkit.Aspire.Hosting.K3s.cs | 73 +++ ...ityToolkit.Aspire.Hosting.K3s.Tests.csproj | 10 + .../HelmReleaseResourceTests.cs | 406 +++++++++++++++ .../K3sAgentNodeTests.cs | 201 ++++++++ .../K3sClusterResourceTests.cs | 295 +++++++++++ .../K3sPublicApiTests.cs | 109 ++++ .../K8sManifestResourceTests.cs | 191 +++++++ 32 files changed, 3390 insertions(+) create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/README.md create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ddf4f4c47..4fae786ae 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -37,6 +37,7 @@ jobs: Hosting.Golang.Tests, Hosting.JavaScript.Extensions.Tests, Hosting.Java.Tests, + Hosting.K3s.Tests, Hosting.k6.Tests, Hosting.Keycloak.Extensions.Tests, Hosting.KurrentDB.Tests, diff --git a/CommunityToolkit.Aspire.slnx b/CommunityToolkit.Aspire.slnx index 1233718eb..3382bdbb0 100644 --- a/CommunityToolkit.Aspire.slnx +++ b/CommunityToolkit.Aspire.slnx @@ -59,6 +59,9 @@ + + + @@ -214,6 +217,7 @@ + @@ -276,6 +280,7 @@ + diff --git a/Directory.Packages.props b/Directory.Packages.props index 668a4938a..66d3b15ac 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -93,6 +93,7 @@ + diff --git a/README.md b/README.md index ce0170510..e9d3b7336 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This repository contains the source code for the Aspire Community Toolkit, a col | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | - **Learn More**: [`Hosting.Golang`][golang-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.Golang][golang-shields]][golang-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.Golang][golang-shields-preview]][golang-nuget-preview] | A hosting integration Golang apps. | | - **Learn More**: [`Hosting.Java`][java-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.Java][java-shields]][java-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.Java][java-shields-preview]][java-nuget-preview] | An integration for running Java code in Aspire either using the local JDK or using a container. | +| - **Learn More**: [`Hosting.K3s`][k3s-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.K3s][k3s-shields]][k3s-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.K3s][k3s-shields-preview]][k3s-nuget-preview] | An Aspire hosting integration for [k3s](https://k3s.io/), a lightweight Kubernetes distribution by Rancher. | | - **Learn More**: [`Hosting.NodeJS.Extensions`][nodejs-ext-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.NodeJS.Extensions][nodejs-ext-shields]][nodejs-ext-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.JavaScript.Extensions][nodejs-ext-shields-preview]][nodejs-ext-nuget-preview] | An integration that contains some additional extensions for running Node.js applications | | - **Learn More**: [`Hosting.Ollama`][ollama-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.Ollama][ollama-shields]][ollama-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.Ollama][ollama-shields-preview]][ollama-nuget-preview] | An Aspire hosting integration leveraging the [Ollama](https://ollama.com) container with support for downloading a model on startup. | | - **Learn More**: [`OllamaSharp`][ollama-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.OllamaSharp][ollamasharp-shields]][ollamasharp-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.OllamaSharp][ollamasharp-shields-preview]][ollamasharp-nuget-preview] | An Aspire client integration for the [OllamaSharp](https://github.com/awaescher/OllamaSharp) package. | @@ -106,6 +107,11 @@ This project is supported by the [.NET Foundation](https://dotnetfoundation.org) [java-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.Java/ [java-shields-preview]: https://img.shields.io/nuget/vpre/CommunityToolkit.Aspire.Hosting.Java?label=nuget%20(preview) [java-nuget-preview]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.Java/absoluteLatest +[k3s-integration-docs]: https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-k3s +[k3s-shields]: https://img.shields.io/nuget/v/CommunityToolkit.Aspire.Hosting.K3s +[k3s-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.K3s/ +[k3s-shields-preview]: https://img.shields.io/nuget/vpre/CommunityToolkit.Aspire.Hosting.K3s?label=nuget%20(preview) +[k3s-nuget-preview]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.K3s/absoluteLatest [nodejs-ext-integration-docs]: https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-nodejs-extensions [nodejs-ext-shields]: https://img.shields.io/nuget/v/CommunityToolkit.Aspire.Hosting.JavaScript.Extensions [nodejs-ext-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.JavaScript.Extensions/ diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj new file mode 100644 index 000000000..65e9770a0 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -0,0 +1,14 @@ + + + + Exe + enable + enable + true + + + + + + + diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs new file mode 100644 index 000000000..00f476326 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -0,0 +1,34 @@ +// K3s hosting example +// ────────────────────────────────────────────────────────────────────────────── +// Prerequisites (host machine): +// • Docker with --privileged support +// • helm → https://helm.sh/docs/intro/install/ +// • kubectl → https://kubernetes.io/docs/tasks/tools/ +// +// What this demonstrates: +// 1. A k3s cluster starts inside a Docker container. +// 2. Headlamp (https://headlamp.dev) is installed as a child Helm release — +// a clickable http URL appears in the Aspire dashboard. +// 3. podinfo is installed — a lightweight demo app that shows the helm lifecycle. +// 4. Both releases are children of k8s in the Aspire resource tree. +// 5. WithPersistentState keeps the cluster data alive across AppHost restarts. +// ────────────────────────────────────────────────────────────────────────────── + +var builder = DistributedApplication.CreateBuilder(args); + +var cluster = builder.AddK3sCluster("k8s", configure: opts => + { + opts.AgentCount = 2; + }) + .WithLifetime(ContainerLifetime.Persistent); + +// cluster.AddHelmRelease( +// name: "podinfo", +// chart: "podinfo", +// repo: "https://stefanprodan.github.io/podinfo", +// version: "6.7.1", +// @namespace: "podinfo") +// .WithHelmValue("service.type", "NodePort") +// .WithEndpoint("podinfo", servicePort: 9898, name: "web"); + +builder.Build().Run(); diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json new file mode 100644 index 000000000..dcfbae059 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:53201;http://localhost:53202", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:53203", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:53204" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:53202", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:53205", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:53206" + } + } + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs new file mode 100644 index 000000000..b1d44bf9c --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs @@ -0,0 +1,35 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that describes a Helm chart release to install into the cluster. +/// +/// The Helm release name. +/// The chart name or local path. +/// The Kubernetes namespace to install into. +/// Optional Helm repository URL containing the chart. +/// Optional chart version. +public sealed class HelmReleaseAnnotation( + string releaseName, + string chart, + string @namespace, + string? repoUrl, + string? version) : IResourceAnnotation +{ + /// Gets the Helm release name. + public string ReleaseName { get; } = releaseName; + + /// Gets the chart name or local path. + public string Chart { get; } = chart; + + /// Gets the target Kubernetes namespace. + public string Namespace { get; } = @namespace; + + /// Gets the optional Helm repository URL. + public string? RepoUrl { get; } = repoUrl; + + /// Gets the optional chart version. + public string? Version { get; } = version; + + /// Gets the extra --set values passed to helm install. + public IDictionary Values { get; } = new Dictionary(); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs new file mode 100644 index 000000000..635eebd5e --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs @@ -0,0 +1,11 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that signals the Kubernetes Dashboard should be installed alongside the cluster. +/// +/// The Kubernetes Dashboard chart version to install. +public sealed class KubernetesDashboardAnnotation(string? version = null) : IResourceAnnotation +{ + /// Gets the dashboard chart version, or to use the latest. + public string? Version { get; } = version; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs new file mode 100644 index 000000000..4525fa5a1 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs @@ -0,0 +1,11 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that describes a Kustomize overlay to apply into the cluster. +/// +/// Path to the kustomization directory or remote URL. +public sealed class KustomizeAnnotation(string path) : IResourceAnnotation +{ + /// Gets the kustomization path. + public string Path { get; } = path; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj new file mode 100644 index 000000000..1800d1555 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -0,0 +1,19 @@ + + + + kubernetes k3s hosting cluster + An Aspire hosting integration for k3s — a lightweight Kubernetes distribution. + + + + + + + + + + + + + + diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs new file mode 100644 index 000000000..eb0845e2d --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs @@ -0,0 +1,19 @@ +using Aspire.Hosting.ApplicationModel; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Lightweight health check that gates WaitFor(helmRelease) on completion of +/// the helm install lifecycle. Returns only +/// after is set by RunReleaseAsync. +/// +internal sealed class HelmReleaseHealthCheck(HelmReleaseResource release) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + => Task.FromResult(release.IsReady + ? HealthCheckResult.Healthy("Helm release is running") + : HealthCheckResult.Unhealthy("Helm release not yet ready")); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs new file mode 100644 index 000000000..247f26e22 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -0,0 +1,45 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a Helm chart release deployed to a k3s cluster. +/// Appears as a distinct dashboard entry; transitions Starting → Running +/// when all pods reach the Ready state. +/// +/// The Aspire resource name (also used as the Helm release name). +/// The Helm release name passed to helm upgrade --install. +/// The Kubernetes namespace to install into. +/// The parent k3s cluster resource. +public sealed class HelmReleaseResource( + string name, + string releaseName, + string @namespace, + K3sClusterResource cluster) + : Resource(name), IResourceWithParent, IResourceWithWaitSupport +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the Helm release name. + public string ReleaseName { get; } = releaseName ?? throw new ArgumentNullException(nameof(releaseName)); + + /// Gets the target Kubernetes namespace. + public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); + + internal string? Chart { get; set; } + internal string? RepoUrl { get; set; } + internal string? Version { get; set; } + internal Dictionary HelmValues { get; } = new(StringComparer.Ordinal); + internal List EndpointDefinitions { get; } = []; + + /// + /// Set to by the lifecycle when the helm install completes and + /// all pods are ready. The WaitFor(helmRelease) health check polls this flag. + /// + internal volatile bool IsReady; +} + +/// Describes a Kubernetes service endpoint to expose from a Helm release. +/// The Kubernetes service name. +/// The service port number. +/// A friendly name shown in the dashboard. +internal sealed record HelmEndpointDefinition(string ServiceName, int ServicePort, string EndpointName); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs new file mode 100644 index 000000000..f6850faaa --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs @@ -0,0 +1,22 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a k3s agent (worker) node that is a child of a . +/// +/// Agent nodes run k3s agent and join the cluster by connecting to the server's API +/// server at https://{serverName}:6443, resolved via DCP's built-in Docker DNS. +/// Agents start immediately alongside the server (no WaitFor dependency) and use +/// k3s's built-in retry loop to connect once the server becomes reachable. The cluster's +/// health check waits for all 1 + nodes +/// to reach Ready state before transitioning to Running. +/// +/// +/// The resource name (e.g. k8s-agent-0). +/// The parent k3s cluster resource. +public sealed class K3sAgentResource(string name, K3sClusterResource cluster) + : ContainerResource(name), IResourceWithParent +{ + /// + public K3sClusterResource Parent { get; } = cluster + ?? throw new ArgumentNullException(nameof(cluster)); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs new file mode 100644 index 000000000..156c83cdb --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -0,0 +1,427 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using k8s; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for adding Helm release resources to a k3s cluster. +/// +public static class K3sHelmBuilderExtensions +{ + /// + /// Adds a Helm release as a child resource of the k3s cluster. + /// + /// Follows the same pattern as PostgresServerResource.AddDatabase: the release is + /// registered on the parent cluster, and the cluster's own + /// handler drives the install lifecycle for all registered releases. Helm output streams to + /// each release's individual log tab in the dashboard. + /// + /// + /// The k3s cluster resource builder. + /// Resource name — also the Helm release name. + /// Chart name. Add for remote charts. + /// Optional Helm repository URL (passed as --repo). + /// Optional chart version (--version). + /// Target namespace (created automatically). + /// A builder for the . + [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] + public static IResourceBuilder AddHelmRelease( + this IResourceBuilder builder, + [ResourceName] string name, + string chart, + string? repo = null, + string? version = null, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(chart); + + var cluster = builder.Resource; + + var release = new HelmReleaseResource(name, releaseName: name, @namespace, cluster) + { + Chart = chart, + RepoUrl = repo, + Version = version, + }; + + // Register the release on the parent cluster — mirrors PostgresServerResource.AddDatabase(). + cluster.AddHelmRelease(release.Name, release.ReleaseName); + + // Health check that satisfies WaitFor(helmRelease) on dependent resources. + // Returns Healthy only after the install lifecycle sets release.IsReady = true. + var healthCheckKey = $"helm_{name}_ready"; + builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + healthCheckKey, + sp => new HelmReleaseHealthCheck(release), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + return builder.ApplicationBuilder + .AddResource(release) + .ExcludeFromManifest() + .WithHealthCheck(healthCheckKey) + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "Helm Release", + State = KnownResourceStates.NotStarted, + Properties = + [ + new ResourcePropertySnapshot("ReleaseName", name), + new ResourcePropertySnapshot("Chart", chart), + new ResourcePropertySnapshot("Namespace", @namespace), + new ResourcePropertySnapshot("Version", version ?? "latest"), + ], + }); + } + + /// + /// Adds a Helm --set key=value argument to this release. + /// + public static IResourceBuilder WithHelmValue( + this IResourceBuilder builder, + string key, + string value) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(value); + + builder.Resource.HelmValues[key] = value; + return builder; + } + + /// + /// Exposes a Kubernetes service from this release as a clickable endpoint in the dashboard. + /// The NodePort is auto-discovered and forwarded in-process via the KubernetesClient WebSocket API. + /// + public static IResourceBuilder WithEndpoint( + this IResourceBuilder builder, + string serviceName, + int servicePort, + string name) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(serviceName); + ArgumentNullException.ThrowIfNull(name); + + builder.Resource.EndpointDefinitions.Add( + new HelmEndpointDefinition(serviceName, servicePort, name)); + + return builder; + } + + // ── Lifecycle (driven from the parent cluster's ResourceReadyEvent) ──────── + + /// + /// Runs the full install lifecycle for : + /// helm install → NodePort discovery → in-process port-forward → Running. + /// Called by the parent cluster's handler — + /// the same pattern Postgres uses for database creation in AddPostgres. + /// + internal static async Task RunReleaseAsync( + HelmReleaseResource release, + K3sClusterResource cluster, + ResourceNotificationService notifications, + ILogger logger, + CancellationToken ct) + { + await notifications.PublishUpdateAsync(release, + state => state with { State = KnownResourceStates.Starting }) + .ConfigureAwait(false); + + try + { + var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); + + if (kubeconfigYaml is null) + { + throw new InvalidOperationException( + "k3s kubeconfig is not yet available. " + + "The cluster ResourceReadyEvent fired before the health check populated the kubeconfig."); + } + + await RunHelmAsync(release, kubeconfigYaml, logger, ct).ConfigureAwait(false); + + var urls = release.EndpointDefinitions.Count > 0 + ? await DiscoverAndStartPortForwardAsync(release, kubeconfigYaml, logger, ct) + .ConfigureAwait(false) + : ImmutableArray.Empty; + + // Set before PublishUpdateAsync so the health check unblocks WaitFor callers + // as soon as the Running state notification is processed. + release.IsReady = true; + + await notifications.PublishUpdateAsync(release, state => state with + { + State = KnownResourceStates.Running, + Urls = urls, + // Merge: keep all existing properties (the orchestrator injects ParentName which + // drives parent-child display in the dashboard) and only update/add our own. + // Replacing the entire Properties array would wipe out ParentName, causing the + // release to lose its parent and appear at the top level after going Running. + Properties = + [ + .. state.Properties.Where(p => + p.Name is not ("ReleaseName" or "Chart" or "Namespace" or "Version" or "ChartVersion")), + new ResourcePropertySnapshot("ReleaseName", release.ReleaseName), + new ResourcePropertySnapshot("Chart", release.Chart!), + new ResourcePropertySnapshot("ChartVersion", release.Version ?? "latest"), + new ResourcePropertySnapshot("Namespace", release.Namespace), + ], + }).ConfigureAwait(false); + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + logger.LogError(ex, "Helm release '{Name}' failed.", release.ReleaseName); + + await notifications.PublishUpdateAsync(release, + state => state with { State = KnownResourceStates.FailedToStart }) + .ConfigureAwait(false); + } + } + + // ── helm subprocess ─────────────────────────────────────────────────────── + + private static async Task RunHelmAsync( + HelmReleaseResource release, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var tempKubeconfig = Path.Combine( + Path.GetTempPath(), + $"aspire-k3s-helm-{Environment.ProcessId}-{release.ReleaseName}.yaml"); + + await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) + .ConfigureAwait(false); + + try + { + // When a repo URL is provided, add/update the repo first. + // The --repo shorthand in helm upgrade --install does not work reliably for + // all chart repositories (e.g. kubernetes-dashboard returns 404 via --repo). + // Official docs always show: helm repo add → helm repo update → helm install. + string? repoAlias = null; + if (release.RepoUrl is not null) + { + repoAlias = $"aspire-k3s-{release.ReleaseName}"; + await RunHelmCommandAsync( + logger, + ["repo", "add", "--force-update", repoAlias, release.RepoUrl], + ct).ConfigureAwait(false); + + await RunHelmCommandAsync( + logger, + ["repo", "update", repoAlias], + ct).ConfigureAwait(false); + } + + var args = BuildHelmInstallArgs( + release.ReleaseName, release.Chart!, repoAlias, + release.Version, release.Namespace, release.HelmValues, + tempKubeconfig); + + logger.LogInformation("Running: helm {Args}", string.Join(' ', args)); + + await RunHelmCommandAsync(logger, args, ct).ConfigureAwait(false); + } + finally + { + try { File.Delete(tempKubeconfig); } catch { /* best effort */ } + } + } + + // ── Argument builder ────────────────────────────────────────────────────── + + /// Runs a helm subcommand, logging output and failing on non-zero exit. + private static async Task RunHelmCommandAsync( + ILogger logger, + IEnumerable args, + CancellationToken ct) + { + var psi = new ProcessStartInfo("helm") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + foreach (var arg in args) + { + psi.ArgumentList.Add(arg); + } + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start helm process."); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogInformation("{Line}", e.Data); + }; + process.ErrorDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogWarning("{Line}", e.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(ct).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"helm {string.Join(' ', psi.ArgumentList)} failed (exit code {process.ExitCode})."); + } + } + + internal static IReadOnlyList BuildHelmInstallArgs( + string releaseName, + string chart, + string? repoAlias, // null = no repo, non-null = "{alias}/{chart}" notation + string? version, + string @namespace, + IReadOnlyDictionary? values, + string kubeconfigPath) + { + // When a repo alias was registered via `helm repo add`, use "alias/chart" notation. + // Otherwise treat the chart as a path or OCI reference. + var chartRef = repoAlias is not null ? $"{repoAlias}/{chart}" : chart; + + var args = new List + { + "upgrade", "--install", + releaseName, + chartRef, + "--namespace", @namespace, + "--create-namespace", + "--wait", + "--timeout", "10m", + $"--kubeconfig={kubeconfigPath}", + }; + + if (version is not null) + { + args.Add("--version"); + args.Add(version); + } + + if (values is not null) + { + foreach (var (key, value) in values) + { + args.Add("--set"); + args.Add($"{key}={value}"); + } + } + + return args; + } + + // ── Port-forward ────────────────────────────────────────────────────────── + + private static async Task> DiscoverAndStartPortForwardAsync( + HelmReleaseResource release, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var urls = ImmutableArray.CreateBuilder(); + + using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); + using var k8sClient = new Kubernetes(config); + + foreach (var ep in release.EndpointDefinitions) + { + var nodePort = await DiscoverNodePortAsync( + k8sClient, release.ReleaseName, release.Namespace, + ep.ServiceName, ep.ServicePort, logger, ct).ConfigureAwait(false); + + var hostPort = AllocateHostPort(); + + var forwarder = new K3sInProcessPortForwarder( + kubeconfigYaml, + release.Namespace, + ep.ServiceName, // look up by service name, not release label + hostPort, + ep.ServicePort); + + _ = forwarder.RunAsync(logger, ct); + + var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); + + if (nodePort.HasValue) + { + urls.Add(new UrlSnapshot( + $"{ep.EndpointName} (container)", + $"{scheme}://{release.Parent.Name}:{nodePort.Value}", + IsInternal: true)); + } + } + + return urls.ToImmutable(); + } + + private static async Task DiscoverNodePortAsync( + Kubernetes k8sClient, + string releaseName, + string @namespace, + string serviceName, + int servicePort, + ILogger logger, + CancellationToken ct) + { + try + { + var services = await k8sClient.CoreV1.ListNamespacedServiceAsync( + @namespace, + labelSelector: $"app.kubernetes.io/instance={releaseName}", + cancellationToken: ct).ConfigureAwait(false); + + var port = services.Items + .FirstOrDefault(s => string.Equals( + s.Metadata.Name, serviceName, StringComparison.OrdinalIgnoreCase)) + ?.Spec.Ports + .FirstOrDefault(p => p.Port == servicePort); + + if (port?.NodePort is null) + { + logger.LogWarning( + "NodePort for {ServiceName}:{ServicePort} not found; container URL omitted.", + serviceName, servicePort); + } + + return port?.NodePort; + } + catch (Exception ex) + { + logger.LogWarning(ex, "NodePort discovery failed for service '{ServiceName}'.", serviceName); + return null; + } + } + + private static int AllocateHostPort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs new file mode 100644 index 000000000..f50f70ba4 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -0,0 +1,400 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using k8s; +using k8s.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for applying Kubernetes YAML manifests to a k3s cluster. +/// +public static class K3sManifestBuilderExtensions +{ + /// + /// Applies one or more Kubernetes YAML files to the cluster via + /// kubectl apply --server-side (Server-Side Apply). No bind-mount is required. + /// + /// A single file: cluster.AddK8sManifest("crd", "./k8s/widget-crd.yaml") + /// A directory: all .yaml/.yml files applied lexicographically. + /// A glob: "./k8s/crds/*.yaml" + /// + /// CRDs are detected automatically; the resource waits for the Established + /// condition via the KubernetesClient before transitioning to Running, so + /// WaitFor(manifest) correctly gates dependent operators. + /// + [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] + public static IResourceBuilder AddK8sManifest( + this IResourceBuilder builder, + [ResourceName] string name, + string path) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(path); + + var cluster = builder.Resource; + var manifest = new K8sManifestResource(name, path, cluster); + + cluster.AddManifest(manifest.Name); + + var healthCheckKey = $"manifest_{name}_ready"; + builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + healthCheckKey, + sp => new K8sManifestHealthCheck(manifest), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + return builder.ApplicationBuilder + .AddResource(manifest) + .ExcludeFromManifest() + .WithHealthCheck(healthCheckKey) + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K8s Manifest", + State = KnownResourceStates.NotStarted, + Properties = [new ResourcePropertySnapshot("Path", path)], + }); + } + + /// + /// Exposes a Kubernetes service from this manifest as a clickable endpoint in the dashboard. + /// Traffic is forwarded in-process via the KubernetesClient WebSocket API. + /// + /// The manifest resource builder. + /// The Kubernetes service name. + /// The service port number. + /// Friendly name shown in the dashboard. + /// + /// The namespace containing the service. Defaults to "default". + /// For remote manifests (HTTP URLs) the namespace must be specified explicitly. + /// + public static IResourceBuilder WithEndpoint( + this IResourceBuilder builder, + string serviceName, + int servicePort, + string name, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(serviceName); + ArgumentNullException.ThrowIfNull(name); + + builder.Resource.EndpointDefinitions.Add( + new ManifestEndpointDefinition(serviceName, servicePort, name, @namespace)); + + return builder; + } + + // ── Lifecycle (called from the cluster's ResourceReadyEvent) ────────────── + + internal static async Task RunManifestAsync( + K8sManifestResource manifest, + K3sClusterResource cluster, + ResourceNotificationService notifications, + ILogger logger, + CancellationToken ct) + { + await notifications.PublishUpdateAsync(manifest, + state => state with { State = KnownResourceStates.Starting }) + .ConfigureAwait(false); + + try + { + var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); + + if (kubeconfigYaml is null) + { + throw new InvalidOperationException( + "k3s kubeconfig is not yet available when applying manifest."); + } + + // Write a temp kubeconfig file — kubectl requires a path, same as helm. + var tempKubeconfig = Path.Combine( + Path.GetTempPath(), + $"aspire-k3s-manifest-{Environment.ProcessId}-{manifest.Name}.yaml"); + + await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) + .ConfigureAwait(false); + + try + { + var files = ResolveFiles(manifest.Path); + + logger.LogInformation( + "Applying {Count} manifest file(s) from '{Path}'", files.Count, manifest.Path); + + // kubectl apply --server-side (SSA) for each file. + foreach (var file in files) + { + await KubectlApplyAsync(file, tempKubeconfig, logger, ct) + .ConfigureAwait(false); + } + + // Wait for any CRDs to reach Established using the KubernetesClient. + await WaitForCrdsEstablishedAsync( + files, kubeconfigYaml, logger, ct).ConfigureAwait(false); + } + finally + { + try { File.Delete(tempKubeconfig); } catch { /* best effort */ } + } + + manifest.IsReady = true; + + // Start in-process port-forwards for any declared endpoints. + var urls = ImmutableArray.Empty; + if (manifest.EndpointDefinitions.Count > 0) + { + urls = await StartPortForwardAsync(manifest, kubeconfigYaml, logger, ct) + .ConfigureAwait(false); + } + + await notifications.PublishUpdateAsync(manifest, state => state with + { + State = KnownResourceStates.Running, + Urls = urls, + Properties = + [ + .. state.Properties.Where(p => p.Name is not "Path"), + new ResourcePropertySnapshot("Path", manifest.Path), + ], + }).ConfigureAwait(false); + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + logger.LogError(ex, "Failed to apply manifest '{Name}'.", manifest.Name); + + await notifications.PublishUpdateAsync(manifest, + state => state with { State = KnownResourceStates.FailedToStart }) + .ConfigureAwait(false); + } + } + + // ── kubectl apply ───────────────────────────────────────────────────────── + + private static async Task KubectlApplyAsync( + string file, + string kubeconfigPath, + ILogger logger, + CancellationToken ct) + { + logger.LogInformation("Applying {File}", file); + + var psi = new ProcessStartInfo("kubectl") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + psi.ArgumentList.Add("apply"); + psi.ArgumentList.Add("-f"); + psi.ArgumentList.Add(file); + psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); + psi.ArgumentList.Add("--server-side"); + psi.ArgumentList.Add("--field-manager=aspire-k3s"); + psi.ArgumentList.Add("--force-conflicts"); + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start kubectl process."); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogInformation("{Line}", e.Data); + }; + process.ErrorDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogWarning("{Line}", e.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(ct).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"kubectl apply failed for '{file}' (exit code {process.ExitCode})."); + } + } + + // ── Port-forward for manifest endpoints ────────────────────────────────── + + private static async Task> StartPortForwardAsync( + K8sManifestResource manifest, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var urls = ImmutableArray.CreateBuilder(); + + foreach (var ep in manifest.EndpointDefinitions) + { + var hostPort = AllocateHostPort(); + + var forwarder = new K3sInProcessPortForwarder( + kubeconfigYaml, + ep.Namespace, + ep.ServiceName, + hostPort, + ep.ServicePort); + + _ = forwarder.RunAsync(logger, ct); + + var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); + } + + return urls.ToImmutable(); + } + + private static int AllocateHostPort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + // ── CRD readiness (in-process via KubernetesClient) ────────────────────── + + private static async Task WaitForCrdsEstablishedAsync( + IReadOnlyList files, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var crdNames = new List(); + + foreach (var file in files) + { + // Skip remote URLs — kubectl downloaded and applied them, but we can't parse + // them with KubernetesYaml locally. CRD detection for remote files is skipped; + // if you need CRD readiness gating, use a local file path instead. + if (file.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + file.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var objects = await KubernetesYaml.LoadAllFromFileAsync(file) + .ConfigureAwait(false); + + foreach (var obj in objects) + { + if (obj is V1CustomResourceDefinition crd && crd.Metadata?.Name is { } crdName) + { + crdNames.Add(crdName); + } + } + } + + if (crdNames.Count == 0) + { + return; + } + + using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); + using var k8sClient = new Kubernetes(config); + + foreach (var crdName in crdNames) + { + await WaitForCrdEstablishedAsync(k8sClient, crdName, logger, ct) + .ConfigureAwait(false); + } + } + + private static async Task WaitForCrdEstablishedAsync( + Kubernetes k8sClient, + string crdName, + ILogger logger, + CancellationToken ct) + { + logger.LogInformation("Waiting for CRD '{Crd}' to reach Established...", crdName); + + while (!ct.IsCancellationRequested) + { + var crd = await k8sClient.ApiextensionsV1 + .ReadCustomResourceDefinitionAsync(crdName, cancellationToken: ct) + .ConfigureAwait(false); + + var established = crd.Status?.Conditions?.Any(c => + c.Type == "Established" && + string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true; + + if (established) + { + logger.LogInformation("CRD '{Crd}' is Established.", crdName); + return; + } + + await Task.Delay(TimeSpan.FromSeconds(2), ct).ConfigureAwait(false); + } + } + + // ── File resolution ─────────────────────────────────────────────────────── + + // Exposed for unit tests via InternalsVisibleTo. + internal static IReadOnlyList ResolveFilesForTest(string path) => + ResolveFiles(path); + + private static IReadOnlyList ResolveFiles(string path) + { + // kubectl apply -f supports HTTPS URLs natively — pass through as-is. + if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + return [path]; + } + + if (Directory.Exists(path)) + { + return [ + ..Directory.GetFiles(path, "*.yaml", SearchOption.TopDirectoryOnly) + .Concat(Directory.GetFiles(path, "*.yml", SearchOption.TopDirectoryOnly)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + ]; + } + + var dir = System.IO.Path.GetDirectoryName(path) ?? "."; + var pattern = System.IO.Path.GetFileName(path); + + if (pattern.Contains('*') || pattern.Contains('?')) + { + return [..Directory.GetFiles(dir, pattern).Order(StringComparer.OrdinalIgnoreCase)]; + } + + return [path]; + } +} + +/// +/// Health check that satisfies WaitFor(manifest). +/// Returns once all files are applied +/// and any CRDs have reached the Established condition. +/// +internal sealed class K8sManifestHealthCheck(K8sManifestResource manifest) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + => Task.FromResult(manifest.IsReady + ? HealthCheckResult.Healthy("Manifests applied") + : HealthCheckResult.Unhealthy("Manifests not yet applied")); +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs new file mode 100644 index 000000000..3399474a0 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -0,0 +1,464 @@ +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using k8s; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using System.Net; +using System.Net.Sockets; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental +#pragma warning disable ASPIRECERTIFICATES001 // WithHttpsDeveloperCertificate is experimental + +namespace Aspire.Hosting; + +/// +/// Provides extension methods for adding k3s cluster resources to an . +/// +public static class K3sBuilderExtensions +{ + /// + /// Adds a k3s Kubernetes cluster resource to the distributed application. + /// + /// The distributed application builder. + /// The resource name used for DNS resolution within the DCP network. + /// + /// Optional host port to bind the Kubernetes API server (port 6443) to. + /// When a random available port is assigned. + /// + /// Optional callback to configure . + /// A builder for the . + [AspireExport("addK3sCluster", Description = "Adds a k3s Kubernetes cluster resource")] + public static IResourceBuilder AddK3sCluster( + this IDistributedApplicationBuilder builder, + [ResourceName] string name, + int? apiServerPort = null, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + + var options = new K3sClusterOptions(); + configure?.Invoke(options); + + var resource = new K3sClusterResource(name); + var tag = options.ImageTag ?? K3sContainerImageTags.Tag; + + var resourceBuilder = builder.AddResource(resource) + .WithImage(K3sContainerImageTags.Image, tag) + .WithImageRegistry(K3sContainerImageTags.Registry) + + // ── k3d-style init entrypoint ───────────────────────────────────── + // Runs mount --make-rshared / and the cgroupsv2 evacuation fix before + // k3s starts — exactly what k3d does via /bin/k3d-entrypoint*.sh. + // WithContainerFiles injects the script via `docker cp` — no bind mounts, + // no host-side temp files, works on all platforms. + .WithContainerFiles("/", [new ContainerFile + { + Name = "aspire-k3s-entrypoint.sh", + Contents = K3sInitEntrypointScript, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]) + .WithEntrypoint("/bin/sh") + .WithArgs("/aspire-k3s-entrypoint.sh") + + // ── k3s server command ──────────────────────────────────────────── + .WithArgs("server") + + // NOTE: --cluster-init (embedded etcd) is intentionally NOT used. + // etcd stores the container's IP in its peer URL. When Docker assigns a new IP + // on container restart, etcd refuses to start ("not a member of the etcd cluster"). + // k3s uses SQLite by default for single-server clusters — it handles restarts and + // IP changes gracefully. k3d only adds --cluster-init for HA multi-server setups. + + // Add TLS SANs so the API server certificate is valid for all addresses + // that clients use to reach it: host-side (localhost), DCP-network containers + // ({resourceName}), and any forwarded address (0.0.0.0). + .WithArgs($"--tls-san=127.0.0.1") + .WithArgs($"--tls-san=localhost") + .WithArgs($"--tls-san={name}") + .WithArgs("--tls-san=0.0.0.0") + + // Disable components not needed for local development. + // servicelb and metrics-server are the biggest resource consumers and slowest + // to start; disabling them speeds up cluster readiness significantly. + .WithArgs("--disable=servicelb") + .WithArgs("--disable=metrics-server") + + // Reduce log verbosity — k3s writes everything to stderr which DCP surfaces + // as "error" level entries in the dashboard log viewer. + .WithArgs("-v", "0") + .WithArgs("--kube-apiserver-arg=v=0") + .WithArgs("--kube-controller-manager-arg=v=0") + .WithArgs("--kube-scheduler-arg=v=0") + // Suppress kubelet INFO-level noise including the harmless cgroupsv2 race warning + // "Failed to kill all the processes attached to cgroup / os: process not initialized". + // This is a known benign race condition in Docker-in-Docker: the kubelet tries to + // force-kill pod cgroup processes that are already dead. The cgroup is still cleaned + // up correctly; only the redundant kill attempt fails. + .WithArgs("--kubelet-arg=v=0") + + // ── API server endpoint ─────────────────────────────────────────── + .WithHttpsEndpoint( + targetPort: 6443, + port: apiServerPort, + name: K3sClusterResource.ApiServerEndpointName) + + // ── Docker / container runtime flags (mirrors k3d) ──────────────── + // Privileged mode is mandatory for iptables, network namespaces, and cgroups. + .WithContainerRuntimeArgs("--privileged") + // k3d uses Docker's --init (tini) so that k3s's child processes are properly + // reaped and signals are forwarded correctly. Without it, zombie processes + // accumulate and shutdown becomes unreliable. + .WithContainerRuntimeArgs("--init") + // Use the host user namespace — required when Docker is configured with userns-remap; + // a no-op otherwise. k3d always passes this flag. + .WithContainerRuntimeArgs("--userns=host") + // Share the host's (Docker Desktop VM's) cgroup namespace instead of creating + // a new one. Without this, the k3s kubelet fails to create the "kubepods" cgroup + // hierarchy because the new isolated namespace has domain controllers in an invalid + // state for cgroupsv2. k3d always passes --cgroupns=host for this reason. + .WithContainerRuntimeArgs("--cgroupns=host") + // Bind-mount the cgroup filesystem from the Docker Desktop VM into the container + // as read-write. With --cgroupns=host the container sees the host's cgroup namespace, + // but the mount is still read-only by default; making it rw lets the kubelet create + // sub-cgroups for pods (kubepods/besteffort/...). k3d always mounts this as rw. + .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") + // tmpfs mounts for runtime sockets and PIDs — same as k3d defaults. + .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + + // ── Environment ─────────────────────────────────────────────────── + // Set an explicit cluster token (k3d always sets K3S_TOKEN). + .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") + // World-readable kubeconfig so docker exec can read it without root. + .WithEnvironment("K3S_KUBECONFIG_MODE", "644") + + .WithIconName("Kubernetes") + .WithHttpsDeveloperCertificate(); + + if (options.ClusterCidr is not null) + { + resourceBuilder.WithArgs($"--cluster-cidr={options.ClusterCidr}"); + } + + if (options.ServiceCidr is not null) + { + resourceBuilder.WithArgs($"--service-cidr={options.ServiceCidr}"); + } + + foreach (var component in options.DisabledComponents) + { + resourceBuilder.WithArgs($"--disable={component}"); + } + + foreach (var arg in options.ExtraArgs) + { + resourceBuilder.WithArgs(arg); + } + + // Create agent nodes specified via options.AgentCount. + // Agents use DCP DNS: K3S_URL=https://{name}:6443 resolves to the server container. + // NO WaitFor — k3s agent retries indefinitely until the server is reachable. + // This avoids a deadlock where the cluster health check waits for nodes to be Ready + // while nodes wait for the cluster to be healthy. + for (var i = 0; i < options.AgentCount; i++) + { + resource.AgentCount++; + var agentName = $"{name}-agent-{i}"; + var agentResource = new K3sAgentResource(agentName, resource); + resource.AddAgentResource(agentResource); + + builder.AddResource(agentResource) + .WithImage(K3sContainerImageTags.Image, tag) + .WithImageRegistry(K3sContainerImageTags.Registry) + .WithContainerFiles("/", [new ContainerFile + { + Name = "aspire-k3s-entrypoint.sh", + Contents = K3sInitEntrypointScript, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]) + .WithEntrypoint("/bin/sh") + .WithArgs("/aspire-k3s-entrypoint.sh") + .WithArgs("agent") + .WithArgs("-v", "0") + .WithArgs("--kubelet-arg=v=0") + .WithEnvironment("K3S_URL", $"https://{name}:6443") + .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") + .WithEnvironment("K3S_NODE_NAME", agentName) + .WithContainerRuntimeArgs("--privileged") + .WithContainerRuntimeArgs("--init") + .WithContainerRuntimeArgs("--userns=host") + .WithContainerRuntimeArgs("--cgroupns=host") + .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") + .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K3s Agent", + State = KnownResourceStates.Starting, + Properties = [new ResourcePropertySnapshot("Cluster", name)], + }); + } + + resourceBuilder.WithHealthCheck($"k3s_{name}_ready"); + + builder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + $"k3s_{name}_ready", + sp => new K3sReadinessHealthCheck(resource, resource.ApiEndpoint), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + // Postgres pattern: the parent cluster's ResourceReadyEvent drives ALL registered + // child lifecycles — both HelmReleases and K8sManifests — in parallel. + builder.Eventing.Subscribe(resource, (@event, ct) => + { + var appModel = @event.Services.GetRequiredService(); + var notifications = @event.Services.GetRequiredService(); + var loggerService = @event.Services.GetRequiredService(); + + // Start all Helm release installs concurrently. + foreach (var release in appModel.Resources + .OfType() + .Where(r => ReferenceEquals(r.Parent, resource))) + { + var logger = loggerService.GetLogger(release); + _ = Task.Run(() => K3sHelmBuilderExtensions.RunReleaseAsync( + release, resource, notifications, logger, ct), ct); + } + + // Start all manifest applies concurrently. + foreach (var manifest in appModel.Resources + .OfType() + .Where(m => ReferenceEquals(m.Parent, resource))) + { + var logger = loggerService.GetLogger(manifest); + _ = Task.Run(() => K3sManifestBuilderExtensions.RunManifestAsync( + manifest, resource, notifications, logger, ct), ct); + } + + return Task.CompletedTask; + }); + + return resourceBuilder; + } + + /// Overrides the k3s server image version. + [AspireExport("withK3sVersion", Description = "Overrides the k3s server image version")] + public static IResourceBuilder WithK3sVersion( + this IResourceBuilder builder, + string tag) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(tag); + + return builder.WithImageTag(tag); + } + + /// Sets the pod subnet CIDR (--cluster-cidr). + [AspireExport("withPodSubnet", Description = "Sets the pod subnet CIDR for the k3s cluster")] + public static IResourceBuilder WithPodSubnet( + this IResourceBuilder builder, + string cidr) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(cidr); + + return builder.WithArgs($"--cluster-cidr={cidr}"); + } + + /// Sets the service subnet CIDR (--service-cidr). + [AspireExport("withServiceSubnet", Description = "Sets the service subnet CIDR for the k3s cluster")] + public static IResourceBuilder WithServiceSubnet( + this IResourceBuilder builder, + string cidr) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(cidr); + + return builder.WithArgs($"--service-cidr={cidr}"); + } + + /// Disables a built-in k3s component (e.g. traefik). + [AspireExport("withDisabledComponent", Description = "Disables a built-in k3s component")] + public static IResourceBuilder WithDisabledComponent( + this IResourceBuilder builder, + string component) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(component); + + return builder.WithArgs($"--disable={component}"); + } + + /// Appends a raw argument to the k3s server command. + [AspireExport("withExtraArg", Description = "Appends a raw argument to the k3s server command")] + public static IResourceBuilder WithExtraArg( + this IResourceBuilder builder, + string arg) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(arg); + + return builder.WithArgs(arg); + } + + /// + /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) + /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. + /// + /// The cluster resource builder. + /// + /// The volume name. When , an auto-generated name is used in the form + /// {appName}-{sha256}-{resourceName}-data — the same scheme used by + /// PostgresServerResource.WithDataVolume and all other Aspire hosting integrations. + /// + /// The resource builder for chaining. + public static IResourceBuilder WithDataVolume( + this IResourceBuilder builder, + string? name = null) + { + ArgumentNullException.ThrowIfNull(builder); + + return builder + .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s") + // Auto-restart on crash so a persistent cluster survives transient failures + // without requiring AppHost intervention. Docker will not restart the container + // on explicit stop (DCP shutdown), only on unexpected exits. + .WithContainerRuntimeArgs("--restart=unless-stopped"); + } + + /// + /// Injects the k3s kubeconfig into as a + /// KUBECONFIG_DATA environment variable (base-64-encoded YAML) — no files are written. + /// + /// + /// s (Auto or InlineData) receive the + /// container-network kubeconfig + /// (server: https://{resourceName}:6443). + /// + /// + /// Projects and executables (Auto or HostPath) receive the + /// host kubeconfig + /// (server: https://localhost:{allocatedPort}). + /// + /// + /// Consuming code reads the variable and builds a client without touching disk: + /// + /// var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); + /// using var stream = new MemoryStream(bytes); + /// var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); + /// + /// The variable is populated only after the cluster health check passes; use + /// WaitFor(cluster) on the dependent resource to guarantee ordering. + /// + [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] + public static IResourceBuilder WithReference( + this IResourceBuilder destination, + IResourceBuilder source, + KubeconfigInjectionStrategy strategy = KubeconfigInjectionStrategy.Auto) + where TDestination : IResourceWithEnvironment + { + ArgumentNullException.ThrowIfNull(destination); + ArgumentNullException.ThrowIfNull(source); + + var cluster = source.Resource; + + return destination.WithEnvironment(ctx => + { + // Select the right kubeconfig variant based on where the resource runs: + // • ContainerResource → container-network URL (reaches API server via DCP DNS) + // • Project/Executable → host URL (reaches API server via localhost port-mapping) + var useContainerVariant = + strategy == KubeconfigInjectionStrategy.ContainerNetwork + || (strategy == KubeconfigInjectionStrategy.Auto + && destination.Resource is ContainerResource); + + var cfg = useContainerVariant + ? cluster.ContainerKubeconfig + : cluster.AdminKubeconfig; + + if (cfg is not null) + { + var yaml = KubernetesYaml.Serialize(cfg); + ctx.EnvironmentVariables["KUBECONFIG_DATA"] = + Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); + } + }); + } + + /// + /// Serialises the admin kubeconfig (server: https://localhost:{port}) to YAML. + /// Returns if the health check has not yet populated the config. + /// + internal static string? GetAdminKubeconfigYaml(K3sClusterResource cluster) => + cluster.AdminKubeconfig is null + ? null + : KubernetesYaml.Serialize(cluster.AdminKubeconfig); + + /// + /// Sets the container lifetime for the k3s cluster and all its agent nodes. + /// When is used the agents must also be + /// persistent — otherwise they are recreated on every AppHost restart while the server + /// retains its state, causing the node re-join sequence to fail. + /// + public static IResourceBuilder WithLifetime( + this IResourceBuilder builder, + ContainerLifetime lifetime) + { + ArgumentNullException.ThrowIfNull(builder); + + // Apply to the cluster container (identical to the built-in generic WithLifetime). + builder.WithAnnotation( + new ContainerLifetimeAnnotation { Lifetime = lifetime }, + ResourceAnnotationMutationBehavior.Replace); + + // Propagate to every agent — they share the same cluster state volume and must + // have the same lifetime so nodes survive AppHost restarts in sync with the server. + foreach (var agent in builder.Resource.AgentResources) + { + var existing = agent.Annotations.OfType().ToList(); + foreach (var ann in existing) + { + agent.Annotations.Remove(ann); + } + + agent.Annotations.Add(new ContainerLifetimeAnnotation { Lifetime = lifetime }); + } + + return builder; + } + + // cgroupsv2 fix adapted from moby/moby (Apache-2.0, used with permission by k3d). + // See: https://github.com/k3d-io/k3d/blob/main/pkg/types/fixes/assets/k3d-entrypoint-cgroupv2.sh + private const string K3sInitEntrypointScript = """ + #!/bin/sh + # Aspire k3s init entrypoint — adapted from k3d (https://github.com/k3d-io/k3d) + # cgroupsv2 fix adapted from moby/moby (Apache-2.0), used with permission. + + # Make mountpoints recursively shared — required for volume propagation in Docker-in-Docker. + mount --make-rshared / 2>/dev/null || true + + # cgroupsv2: evacuate root cgroup so k3s kubelet can create pod sub-cgroups. + # Without this, writing to cgroup.subtree_control fails with EBUSY because + # init processes still live in the root cgroup. + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + mkdir -p /sys/fs/cgroup/init + if command -v xargs >/dev/null 2>&1; then + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + else + busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + fi + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null || true + fi + + exec k3s "$@" + """; +} + +#pragma warning restore ASPIREATS001 +#pragma warning restore ASPIRECERTIFICATES001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs new file mode 100644 index 000000000..cb33f7aa7 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -0,0 +1,40 @@ +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Configuration options for a k3s cluster resource. +/// +public sealed class K3sClusterOptions +{ + /// + /// Gets or sets the CIDR range for pod IPs (passed as --cluster-cidr). + /// + public string? ClusterCidr { get; set; } + + /// + /// Gets or sets the CIDR range for service IPs (passed as --service-cidr). + /// + public string? ServiceCidr { get; set; } + + /// + /// Gets the list of k3s components to disable (each passed as --disable=<component>). + /// + public IList DisabledComponents { get; } = new List(); + + /// + /// Gets the list of raw extra arguments appended to the k3s server command. + /// + public IList ExtraArgs { get; } = new List(); + + /// + /// Gets or sets the number of agent (worker) nodes to add to the cluster. + /// Equivalent to k3d's --agents N flag. + /// Defaults to 0 (single-node cluster). + /// + public int AgentCount { get; set; } + + /// + /// Gets or sets the k3s image tag (e.g. v1.31.4-k3s1). + /// When , the default tag embedded in the package is used. + /// + public string? ImageTag { get; set; } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs new file mode 100644 index 000000000..f7b983b5a --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -0,0 +1,63 @@ +using k8s.KubeConfigModels; + +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a k3s Kubernetes cluster running as a privileged container resource. +/// +/// The resource name. +public sealed class K3sClusterResource(string name) : ContainerResource(name) +{ + internal const string ApiServerEndpointName = "api"; + + /// + /// Gets the admin kubeconfig for host-side processes + /// (server: https://localhost:{allocatedPort}). + /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. + /// Serialise with KubernetesYaml.Serialize(AdminKubeconfig) when needed. + /// + internal K8SConfiguration? AdminKubeconfig { get; set; } + + /// + /// Gets the kubeconfig for containers on the DCP Docker network + /// (server: https://{resourceName}:6443). + /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. + /// + internal K8SConfiguration? ContainerKubeconfig { get; set; } + + private EndpointReference? _apiEndpoint; + + /// Gets the endpoint reference for the k3s API server (port 6443). + public EndpointReference ApiEndpoint => _apiEndpoint ??= new(this, ApiServerEndpointName); + + // ── Child resource tracking (Postgres pattern) ──────────────────────────── + + /// + /// Number of agent (worker) nodes. The health check waits for all + /// 1 + AgentCount nodes to reach Ready state before marking the cluster healthy. + /// + internal int AgentCount { get; set; } + + private readonly List _agentResources = []; + + /// Agent resource instances, used to propagate annotations (e.g. lifetime) to all nodes. + internal IReadOnlyList AgentResources => _agentResources; + + internal void AddAgentResource(K3sAgentResource agent) => _agentResources.Add(agent); + + private readonly Dictionary _helmReleases = + new(StringComparer.OrdinalIgnoreCase); + + /// A dictionary of registered Helm releases keyed by resource name. + public IReadOnlyDictionary HelmReleases => _helmReleases; + + internal void AddHelmRelease(string resourceName, string releaseName) => + _helmReleases.TryAdd(resourceName, releaseName); + + private readonly List _manifests = []; + + /// Names of registered children. + public IReadOnlyList Manifests => _manifests; + + internal void AddManifest(string resourceName) => _manifests.Add(resourceName); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs new file mode 100644 index 000000000..605e5c65c --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs @@ -0,0 +1,13 @@ +namespace Aspire.Hosting.ApplicationModel; + +internal static class K3sContainerImageTags +{ + /// docker.io + public const string Registry = "docker.io"; + + /// rancher/k3s + public const string Image = "rancher/k3s"; + + /// v1.31.4-k3s1 + public const string Tag = "v1.31.4-k3s1"; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs new file mode 100644 index 000000000..1fa3eb074 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -0,0 +1,108 @@ +using System.Diagnostics; +using System.Text; +using Microsoft.Extensions.Logging; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Forwards a local TCP port to a Kubernetes service by running +/// kubectl port-forward service/{name} {localPort}:{servicePort} -n {namespace} +/// as a managed subprocess. +/// +/// Mirrors what a developer types in a terminal — the most reliable approach for +/// k3s-in-Docker because kubectl handles WebSocket negotiation, kubelet routing, +/// and reconnect logic internally. +/// +/// +internal sealed class K3sInProcessPortForwarder( + string kubeconfigYaml, + string @namespace, + string serviceName, + int localPort, + int servicePort) +{ + public async Task RunAsync(ILogger logger, CancellationToken ct) + { + logger.LogInformation( + "Port-forward: localhost:{Local} → svc/{Service}.{Ns}:{Port}", + localPort, serviceName, @namespace, servicePort); + + var tempConfig = Path.Combine( + Path.GetTempPath(), + $"aspire-k3s-pf-{Environment.ProcessId}-{serviceName}.yaml"); + + await File.WriteAllTextAsync(tempConfig, kubeconfigYaml, Encoding.UTF8, ct) + .ConfigureAwait(false); + + try + { + while (!ct.IsCancellationRequested) + { + try + { + await RunKubectlAsync(tempConfig, logger, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + logger.LogWarning(ex, + "Port-forward for svc/{Service} exited; restarting in 5 s…", serviceName); + + await Task.Delay(TimeSpan.FromSeconds(5), ct).ConfigureAwait(false); + } + } + } + finally + { + try { File.Delete(tempConfig); } catch { /* best-effort cleanup */ } + } + } + + private async Task RunKubectlAsync(string kubeconfigPath, ILogger logger, CancellationToken ct) + { + var psi = new ProcessStartInfo("kubectl") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + psi.ArgumentList.Add("port-forward"); + psi.ArgumentList.Add($"service/{serviceName}"); + psi.ArgumentList.Add($"{localPort}:{servicePort}"); + psi.ArgumentList.Add("-n"); + psi.ArgumentList.Add(@namespace); + psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start kubectl port-forward."); + + using var reg = ct.Register(() => + { + try { process.Kill(entireProcessTree: true); } catch { /* already exited */ } + }); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogDebug("{Line}", e.Data); + }; + process.ErrorDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogDebug("{Line}", e.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(ct).ConfigureAwait(false); + + if (process.ExitCode != 0 && !ct.IsCancellationRequested) + { + throw new InvalidOperationException( + $"kubectl port-forward exited with code {process.ExitCode}."); + } + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs new file mode 100644 index 000000000..f0cc8f1fd --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -0,0 +1,190 @@ +using System.Diagnostics; +using Aspire.Hosting.ApplicationModel; +using k8s; +using k8s.KubeConfigModels; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Health check for . +/// +/// Instead of probing GET /healthz (which requires authentication in Kubernetes 1.28+ +/// because anonymous-auth defaults to false), this check runs +/// docker exec {container} kubectl get nodes inside the k3s container. +/// kubectl inside the container uses the default in-cluster kubeconfig, so no external +/// credentials are needed and the result is authoritative: a node in Ready state +/// proves the API server, scheduler, and kubelet are all functional. +/// +/// +/// On first success the kubeconfig is read from the container via docker exec cat, +/// parsed into two variants, and stored on the resource: +/// +/// — server: https://localhost:{port} +/// — server: https://{name}:6443 +/// +/// +/// +internal sealed class K3sReadinessHealthCheck : IHealthCheck +{ + private readonly K3sClusterResource _resource; + private readonly EndpointReference _endpoint; + private bool _kubeconfigRead; + + internal K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) + { + _resource = resource; + _endpoint = endpoint; + } + + /// + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + if (!_endpoint.IsAllocated) + { + return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); + } + + var port = _endpoint.Port; + + try + { + var containerId = await FindContainerIdAsync(cancellationToken); + + if (containerId is null) + { + return HealthCheckResult.Unhealthy("k3s container not yet found via docker ps"); + } + + // Run kubectl get nodes inside the container where the default kubeconfig is + // already configured — avoids any authentication issue from the outside. + var nodesOutput = await RunDockerAsync( + ["exec", containerId, + "kubectl", "get", "nodes", + "--kubeconfig", "/etc/rancher/k3s/k3s.yaml", + "--no-headers"], + cancellationToken); + + if (nodesOutput is null) + { + return HealthCheckResult.Unhealthy( + "kubectl get nodes failed — k3s API server not yet ready"); + } + + // Count nodes actually in Ready state (excluding NotReady ones). + var readyNodeLines = nodesOutput + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.Contains("Ready") && !line.Contains("NotReady")) + .ToArray(); + + // For multi-node clusters (WithAgentNodes), wait for server + all agents. + // For single-node, 1 Ready node is sufficient. + var expectedNodes = 1 + _resource.AgentCount; + + if (readyNodeLines.Length < expectedNodes) + { + return HealthCheckResult.Unhealthy( + $"k3s cluster: {readyNodeLines.Length}/{expectedNodes} nodes Ready"); + } + + if (!_kubeconfigRead) + { + var rawYaml = await RunDockerAsync( + ["exec", containerId, "cat", "/etc/rancher/k3s/k3s.yaml"], + cancellationToken); + + if (rawYaml is null) + { + return HealthCheckResult.Unhealthy( + "k3s kubeconfig not yet available inside the container"); + } + + var parsed = KubernetesYaml.Deserialize(rawYaml); + + _resource.AdminKubeconfig = + BuildConfig(parsed, $"https://localhost:{port}"); + _resource.ContainerKubeconfig = + BuildConfig(parsed, $"https://{_resource.Name}:6443"); + _kubeconfigRead = true; + } + + return HealthCheckResult.Healthy("k3s cluster is ready"); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy(ex.Message, ex); + } + } + + private static K8SConfiguration BuildConfig(K8SConfiguration source, string serverUrl) + { + var yaml = KubernetesYaml.Serialize(source); + var copy = KubernetesYaml.Deserialize(yaml); + + foreach (var cluster in copy.Clusters ?? []) + { + if (cluster.ClusterEndpoint is not null) + { + cluster.ClusterEndpoint.Server = serverUrl; + } + } + + return copy; + } + + private async Task FindContainerIdAsync(CancellationToken ct) + { + // docker ps --filter name=VALUE uses substring matching: "name=k8s" also matches + // "k8s-agent-0", "k8s-agent-1", etc. Use --format to get names alongside IDs and + // exclude agent containers whose names contain "-agent-". + var output = await RunDockerAsync( + ["ps", + "--filter", $"name={_resource.Name}", + "--format", "{{.ID}}\t{{.Names}}", + "--no-trunc"], + ct); + + if (output is null) + { + return null; + } + + return output + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(line => line.Split('\t', 2)) + .Where(parts => parts.Length == 2 && !parts[1].Contains("-agent-")) + .Select(parts => parts[0].Trim()) + .FirstOrDefault(id => !string.IsNullOrWhiteSpace(id)); + } + + private static async Task RunDockerAsync(string[] args, CancellationToken ct) + { + var psi = new ProcessStartInfo("docker") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + foreach (var arg in args) + { + psi.ArgumentList.Add(arg); + } + + using var process = Process.Start(psi); + if (process is null) + { + return null; + } + + var output = await process.StandardOutput.ReadToEndAsync(ct); + await process.WaitForExitAsync(ct); + + return process.ExitCode == 0 && !string.IsNullOrWhiteSpace(output) + ? output + : null; + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs new file mode 100644 index 000000000..f0a6a361b --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -0,0 +1,47 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via +/// Server-Side Apply. This is a child resource of , following +/// the same parent-child pattern as . +/// +/// No kubectl binary is required — the KubernetesClient library handles the apply. +/// CRDs reach the Established condition before the resource transitions to +/// Running, so dependent resources can safely WaitFor the manifest. +/// +/// +/// The Aspire resource name. +/// +/// Path to a single YAML file, a directory, or a glob pattern (*.yaml). +/// Directories and globs are expanded lexicographically. +/// +/// The parent k3s cluster resource. +public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) + : Resource(name), IResourceWithParent, IResourceWithWaitSupport +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the manifest path, directory, or glob. + public string Path { get; } = path ?? throw new ArgumentNullException(nameof(path)); + + /// + /// Set to by the lifecycle after all objects are applied and + /// (for CRDs) the Established condition is confirmed. + /// + internal volatile bool IsReady; + + /// Services to expose via in-process port-forward after the manifest is applied. + internal List EndpointDefinitions { get; } = []; +} + +/// Describes a service endpoint to expose from a . +/// Kubernetes service name. +/// Service port number. +/// Friendly name shown in the dashboard. +/// Kubernetes namespace where the service lives. +internal sealed record ManifestEndpointDefinition( + string ServiceName, + int ServicePort, + string EndpointName, + string Namespace); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs new file mode 100644 index 000000000..b747206b1 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs @@ -0,0 +1,33 @@ +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Controls which kubeconfig server-URL variant is injected via KUBECONFIG_DATA. +/// All variants are delivered as base-64-encoded YAML without writing any file. +/// +public enum KubeconfigInjectionStrategy +{ + /// + /// Selects the server URL automatically based on the resource type: + /// + /// Container resources receive the + /// container-network URL (https://{resourceName}:6443). + /// Projects and executables receive the host URL + /// (https://localhost:{allocatedPort}). + /// + /// + Auto, + + /// + /// Always inject the host-network kubeconfig (server: https://localhost:{port}). + /// Use when a container is launched with --network=host or when the caller + /// explicitly needs host-side connectivity. + /// + HostNetwork, + + /// + /// Always inject the DCP-network kubeconfig + /// (server: https://{resourceName}:6443). + /// Use when a host process needs to reach the cluster the same way containers do. + /// + ContainerNetwork, +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md new file mode 100644 index 000000000..f6a5d8150 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -0,0 +1,67 @@ +# CommunityToolkit.Aspire.Hosting.K3s + +An Aspire hosting integration for [k3s](https://k3s.io/) — a lightweight, certified Kubernetes distribution by Rancher/SUSE. + +## Getting started + +### Prerequisites + +- Docker with support for `--privileged` containers (Linux host or Docker Desktop on macOS/Windows) + +### Installation + +```sh +dotnet add package CommunityToolkit.Aspire.Hosting.K3s +``` + +## Usage + +```csharp +var builder = DistributedApplication.CreateBuilder(args); + +var cluster = builder.AddK3sCluster("k8s") + .WithPersistentState(); + +builder.AddProject("api") + .WithReference(cluster) + .WaitFor(cluster); + +builder.Build().Run(); +``` + +### Kubeconfig injection + +`WithReference(cluster)` automatically selects the injection mode: + +| Resource type | Environment variable set | +|---|---| +| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=/tmp/aspire-k3s-k8s/admin.yaml` | +| `ContainerResource` | `KUBECONFIG_DATA=` | + +### Configuration options + +```csharp +builder.AddK3sCluster("k8s", configure: opts => +{ + opts.ClusterCidr = "10.42.0.0/16"; + opts.ServiceCidr = "10.43.0.0/16"; + opts.DisabledComponents.Add("traefik"); +}); +``` + +Or use the fluent API: + +```csharp +builder.AddK3sCluster("k8s") + .WithK3sVersion("v1.32.3-k3s1") + .WithPodSubnet("10.42.0.0/16") + .WithServiceSubnet("10.43.0.0/16") + .WithDisabledComponent("traefik") + .WithPersistentState(); +``` + +## Known limitations + +- Requires a privileged Docker runtime; `--privileged` is passed automatically. +- On Linux hosts the `/lib/modules` directory should be present for CNI networking. +- The first cluster start can take 30–60 seconds while container images and CNI plugins are initialised. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs new file mode 100644 index 000000000..50701b0d7 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -0,0 +1,73 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Aspire.Hosting +{ + public static partial class K3sHelmBuilderExtensions + { + public static ApplicationModel.IResourceBuilder AddHelmRelease(this ApplicationModel.IResourceBuilder builder, string name, string chart, string? repo = null, string? version = null, string @namespace = "default") { throw null; } + public static ApplicationModel.IResourceBuilder WithHelmValue(this ApplicationModel.IResourceBuilder builder, string key, string value) { throw null; } + public static ApplicationModel.IResourceBuilder WithEndpoint(this ApplicationModel.IResourceBuilder builder, string serviceName, int servicePort, string name) { throw null; } + } + + public static partial class K3sBuilderExtensions + { + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source, CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy strategy = CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy.Auto) where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } + public static ApplicationModel.IResourceBuilder WithPersistentState(this ApplicationModel.IResourceBuilder builder, string? volumeName = null) { throw null; } + } +} + +namespace Aspire.Hosting.ApplicationModel +{ + public sealed partial class HelmReleaseResource : Resource, IResourceWithParent, IResourceWithWaitSupport + { + public HelmReleaseResource(string name, string releaseName, string @namespace, K3sClusterResource cluster) : base(default!) { } + public K3sClusterResource Parent { get { throw null; } } + public string ReleaseName { get { throw null; } } + public string Namespace { get { throw null; } } + } + + public sealed partial class K3sClusterResource : ContainerResource + { + public K3sClusterResource(string name) : base(default!) { } + public EndpointReference ApiEndpoint { get { throw null; } } + } +} + +namespace CommunityToolkit.Aspire.Hosting +{ + public sealed partial class K3sClusterOptions + { + public K3sClusterOptions() { } + public string? ClusterCidr { get { throw null; } set { } } + public string? ImageTag { get { throw null; } set { } } + public string? ServiceCidr { get { throw null; } set { } } + public System.Collections.Generic.IList DisabledComponents { get { throw null; } } + public System.Collections.Generic.IList ExtraArgs { get { throw null; } } + } + + public enum KubeconfigInjectionStrategy + { + Auto = 0, + HostNetwork = 1, + ContainerNetwork = 2, + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj new file mode 100644 index 000000000..bad157c30 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs new file mode 100644 index 000000000..1269e0a34 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -0,0 +1,406 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class HelmReleaseResourceTests +{ + [Fact] + public void AddHelmReleaseAddsHelmReleaseResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("argocd", resource.Name); + } + + [Fact] + public void AddHelmReleaseDoesNotCreateSeparateInstallResource() + { + // helm upgrade --install runs internally inside the HelmReleaseResource lifecycle; + // no separate ExecutableResource is added to the application model. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AddHelmReleaseDefaultsReleaseName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("my-release", "my-chart"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("my-release", resource.ReleaseName); + } + + [Fact] + public void AddHelmReleaseDefaultsNamespace() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("default", resource.Namespace); + } + + [Fact] + public void AddHelmReleaseWithExplicitNamespace() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd", @namespace: "argocd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("argocd", resource.Namespace); + } + + [Fact] + public void AddHelmReleaseStoresRepoUrl() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease( + "argocd", "argo-cd", + repo: "https://argoproj.github.io/argo-helm"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("https://argoproj.github.io/argo-helm", resource.RepoUrl); + } + + [Fact] + public void AddHelmReleaseStoresVersion() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd", version: "7.8.0"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("7.8.0", resource.Version); + } + + [Fact] + public void HelmReleaseParentIsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, resource.Parent); + } + + [Fact] + public void HelmReleaseImplementsNonGenericIResourceWithParent() + { + // The Aspire dashboard uses the non-generic IResourceWithParent to group + // child resources under their parent. Verify both the generic and non-generic + // interfaces are satisfied and point to the same cluster resource. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + + // Non-generic IResourceWithParent (used by the dashboard) + var nonGeneric = resource as IResourceWithParent; + Assert.NotNull(nonGeneric); + Assert.Same(cluster.Resource, nonGeneric.Parent); + } + + [Fact] + public void WithHelmValueAccumulatesValues() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValue("server.service.type", "NodePort") + .WithHelmValue("server.insecure", "true"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("NodePort", resource.HelmValues["server.service.type"]); + Assert.Equal("true", resource.HelmValues["server.insecure"]); + } + + [Fact] + public void WithEndpointAccumulatesEndpoints() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithEndpoint("argocd-server", servicePort: 443, name: "ui"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ep = Assert.Single(resource.EndpointDefinitions); + Assert.Equal("argocd-server", ep.ServiceName); + Assert.Equal(443, ep.ServicePort); + Assert.Equal("ui", ep.EndpointName); + } + + [Fact] + public void WithEndpointMultipleEndpoints() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithEndpoint("argocd-server", 443, "ui") + .WithEndpoint("argocd-server", 80, "http"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal(2, resource.EndpointDefinitions.Count); + } + + [Fact] + public void HelmReleaseIsExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); + } + + // ── BuildHelmInstallArgs tests (pure logic, no DI needed) ───────────────── + + [Fact] + public void BuildHelmInstallArgsIncludesUpgradeInstall() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "argocd", "argo-cd", null, null, "argocd", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("upgrade", list); + Assert.Contains("--install", list); + Assert.Contains("argocd", list); + Assert.Contains("argo-cd", list); + } + + [Fact] + public void BuildHelmInstallArgsIncludesKubeconfig() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.Contains("--kubeconfig=/tmp/admin.yaml", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesWait() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.Contains("--wait", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesNamespace() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "my-ns", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("--namespace", list); + Assert.Contains("my-ns", list); + Assert.Contains("--create-namespace", list); + } + + [Fact] + public void BuildHelmInstallArgsWithRepoAliasUsesPrefixedChartRef() + { + // When a repo alias is pre-registered via `helm repo add`, BuildHelmInstallArgs + // uses "{alias}/{chart}" notation — NOT the --repo flag (which is unreliable). + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", "my-repo-alias", null, "default", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.DoesNotContain("--repo", list); + Assert.Contains("my-repo-alias/chart", list); + } + + [Fact] + public void BuildHelmInstallArgsWithNullAliasUsesChartDirectly() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "oci://registry/chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.DoesNotContain("--repo", args); + Assert.Contains("oci://registry/chart", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesVersion() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, "7.8.0", "default", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("--version", list); + Assert.Contains("7.8.0", list); + } + + [Fact] + public void BuildHelmInstallArgsOmitsRepoWhenNull() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.DoesNotContain("--repo", args); + } + + [Fact] + public void BuildHelmInstallArgsOmitsVersionWhenNull() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.DoesNotContain("--version", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesSetValues() + { + var values = new Dictionary + { + ["service.type"] = "NodePort", + ["replicaCount"] = "2", + }; + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", values, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("--set", list); + Assert.Contains("service.type=NodePort", list); + Assert.Contains("replicaCount=2", list); + } + + // ── WaitFor support ─────────────────────────────────────────────────────── + + [Fact] + public void HelmReleaseHasHealthCheckForWaitForSupport() + { + // WaitFor(helmRelease) is satisfied by the HelmReleaseHealthCheck, + // which flips IsReady once RunReleaseAsync completes. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(resource.Annotations.OfType(), a => + a.Key == "helm_argocd_ready"); + } + + [Fact] + public void HelmReleaseIsReadyFlagStartsFalse() + { + var resource = new HelmReleaseResource( + "argocd", "argocd", "default", new K3sClusterResource("k8s")); + + Assert.False(resource.IsReady); + } + + // ── Public API null-guard tests ─────────────────────────────────────────── + + [Fact] + public void AddHelmReleaseShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.AddHelmRelease("argocd", "argo-cd"); + Assert.Throws(action); + } + + [Fact] + public void AddHelmReleaseShouldThrowWhenNameIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddHelmRelease(null!, "argo-cd"); + Assert.Throws(action); + } + + [Fact] + public void AddHelmReleaseShouldThrowWhenChartIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddHelmRelease("argocd", null!); + Assert.Throws(action); + } + + [Fact] + public void WithHelmValueShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithHelmValue("key", "value"); + Assert.Throws(action); + } + + [Fact] + public void WithEndpointShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithEndpoint("svc", 443, "ui"); + Assert.Throws(action); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs new file mode 100644 index 000000000..b9dcf1f0c --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs @@ -0,0 +1,201 @@ +using Aspire.Hosting; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sAgentNodeTests +{ + // ── Agent creation via K3sClusterOptions ───────────────────────────────── + + [Fact] + public void AgentCountInOptionsCreatesK3sAgentResources() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + + Assert.Equal(2, agents.Count); + Assert.Contains(agents, a => a.Name == "k8s-agent-0"); + Assert.Contains(agents, a => a.Name == "k8s-agent-1"); + } + + [Fact] + public void AgentNodesAreChildrenOfCluster() + { + // Implements IResourceWithParent so they appear nested under k8s in the dashboard. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, agent.Parent); + + // Non-generic IResourceWithParent — used by the dashboard for grouping. + var nonGeneric = agent as IResourceWithParent; + Assert.NotNull(nonGeneric); + Assert.Same(cluster.Resource, nonGeneric.Parent); + } + + [Fact] + public void AgentCountZeroProducesNoAgentResources() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 0); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AgentCountUpdatesClusterAgentCount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 3); + + Assert.Equal(3, cluster.Resource.AgentCount); + } + + [Fact] + public void AgentNodesDoNotHaveWaitForDependencyOnCluster() + { + // Agents must NOT WaitFor the cluster — that would create a deadlock because the + // cluster health check waits for all nodes (including agents) to be Ready. + // Instead, k3s agent retries connecting to the server independently. + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var waitAnnotations = agent.Annotations.OfType().ToList(); + + Assert.DoesNotContain(waitAnnotations, w => w.Resource is K3sClusterResource); + } + + [Fact] + public void AgentNodesUseSameImageAsServer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var img = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(K3sContainerImageTags.Image, img.Image); + Assert.Equal(K3sContainerImageTags.Registry, img.Registry); + } + + [Fact] + public void AgentNodesAreExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, agent.Annotations); + } + + [Fact] + public void AgentNodesHaveEnvironmentAnnotations() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var envCallbacks = agent.Annotations.OfType().ToList(); + Assert.True(envCallbacks.Count >= 3, + $"Expected at least 3 env annotations (K3S_URL, K3S_TOKEN, K3S_NODE_NAME), got {envCallbacks.Count}"); + } + + [Fact] + public void DefaultClusterHasNoAgentNodes() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AgentNodeNamesFollowConvention() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("mycluster", configure: opts => opts.AgentCount = 3); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-0"); + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-1"); + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-2"); + } + + [Fact] + public void NegativeAgentCountIsIgnored() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = -1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void WithLifetimePersistentPropagatestoAgentNodes() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder + .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2) + .WithLifetime(ContainerLifetime.Persistent); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + Assert.Equal(2, agents.Count); + + foreach (var agent in agents) + { + var annotation = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Persistent, annotation.Lifetime); + } + } + + [Fact] + public void WithLifetimeSessionDoesNotAddPersistentAnnotationToAgents() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder + .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1) + .WithLifetime(ContainerLifetime.Session); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var annotation = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Session, annotation.Lifetime); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs new file mode 100644 index 000000000..9d1811255 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -0,0 +1,295 @@ +using System.Net.Sockets; +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sClusterResourceTests +{ + [Fact] + public void AddK3sClusterAddsResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.Equal("k8s", resource.Name); + } + + [Fact] + public void AddK3sClusterAddsCorrectContainerImage() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + + Assert.Equal(K3sContainerImageTags.Image, annotation.Image); + Assert.Equal(K3sContainerImageTags.Tag, annotation.Tag); + Assert.Equal(K3sContainerImageTags.Registry, annotation.Registry); + } + + [Fact] + public void AddK3sClusterAddsApiServerEndpointOnPort6443() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var endpoint = Assert.Single( + resource.Annotations.OfType(), + e => e.Name == K3sClusterResource.ApiServerEndpointName); + + Assert.Equal(6443, endpoint.TargetPort); + Assert.Equal(ProtocolType.Tcp, endpoint.Protocol); + Assert.Equal("https", endpoint.UriScheme); + } + + [Fact] + public void AddK3sClusterWithExplicitPortBindsToThatPort() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", apiServerPort: 16443); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var endpoint = Assert.Single( + resource.Annotations.OfType(), + e => e.Name == K3sClusterResource.ApiServerEndpointName); + + Assert.Equal(6443, endpoint.TargetPort); + Assert.Equal(16443, endpoint.Port); + } + + [Fact] + public void AddK3sClusterAddsServerArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var commandLineArgs = resource.Annotations.OfType(); + Assert.NotEmpty(commandLineArgs); + } + + [Fact] + public void WithK3sVersionOverridesImageTag() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithK3sVersion("v1.32.3-k3s1"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + + Assert.Equal("v1.32.3-k3s1", annotation.Tag); + } + + [Fact] + public void AddK3sClusterHasNoVolumeByDefault() + { + // Persistence is opt-in via WithPersistentState(). + // No volume is mounted by default so the cluster is ephemeral. + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.DoesNotContain( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + } + + [Fact] + public void WithDataVolumeAddsSingleVolumeMount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDataVolume(); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var volume = Assert.Single( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + + // VolumeNameGenerator format: {appName}-{sha256}-{resourceName}-data + Assert.EndsWith("-k8s-data", volume.Source); + } + + [Fact] + public void WithDataVolumeUsesCustomName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDataVolume("my-k3s-data"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var volume = Assert.Single( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + + Assert.Equal("my-k3s-data", volume.Source); + } + + [Fact] + public void AddK3sClusterWithClusterCidrViaOptions() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => + { + opts.ClusterCidr = "10.99.0.0/16"; + }); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.NotNull(resource); + } + + [Fact] + public void WithReferenceSetsKubeconfigEnvForProject() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + var cluster = appBuilder.AddK3sCluster("k8s"); + + // ProjectResource would need a project file; use ExecutableResource as a proxy + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + exe.WithReference(cluster); + + // Verify the environment callback was added (no exception thrown) + using var app = appBuilder.Build(); + Assert.NotNull(app); + } + + [Fact] + public void WithReferenceSetsKubeconfigDataEnvForContainer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + var cluster = appBuilder.AddK3sCluster("k8s"); + var container = appBuilder.AddContainer("operator", "myorg/operator"); + container.WithReference(cluster); + + using var app = appBuilder.Build(); + Assert.NotNull(app); + } + + [Fact] + public void K3sClusterResourceHasNoKubeconfigDirectoryByDefault() + { + // Kubeconfig is now stored in-memory (K8SConfiguration objects) and + // never written to disk by the resource itself — docker exec reads it. + var resource = new K3sClusterResource("k8s"); + Assert.Null(resource.AdminKubeconfig); + Assert.Null(resource.ContainerKubeconfig); + } + + [Fact] + public void WithPodSubnetAddsClusterCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithPodSubnet("10.88.0.0/16"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithServiceSubnetAddsServiceCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithServiceSubnet("10.89.0.0/16"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithDisabledComponentAddsDisableArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDisabledComponent("traefik"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithExtraArgAddsRawArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithExtraArg("--write-kubeconfig-mode=644"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void ApiEndpointReturnsEndpointReferenceWithCorrectName() + { + var resource = new K3sClusterResource("k8s"); + var endpoint = resource.ApiEndpoint; + + Assert.NotNull(endpoint); + Assert.Equal(K3sClusterResource.ApiServerEndpointName, endpoint.EndpointName); + } + + [Fact] + public void ApiEndpointIsCached() + { + var resource = new K3sClusterResource("k8s"); + + var first = resource.ApiEndpoint; + var second = resource.ApiEndpoint; + + Assert.Same(first, second); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs new file mode 100644 index 000000000..f803d976f --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs @@ -0,0 +1,109 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sPublicApiTests +{ + [Fact] + public void AddK3sClusterShouldThrowWhenBuilderIsNull() + { + IDistributedApplicationBuilder builder = null!; + + var action = () => builder.AddK3sCluster("k8s"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void AddK3sClusterShouldThrowWhenNameIsNull() + { + IDistributedApplicationBuilder builder = new DistributedApplicationBuilder([]); + string name = null!; + + var action = () => builder.AddK3sCluster(name); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(name), exception.ParamName); + } + + [Fact] + public void WithK3sVersionShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithK3sVersion("v1.32.3-k3s1"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithPodSubnetShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithPodSubnet("10.42.0.0/16"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithServiceSubnetShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithServiceSubnet("10.43.0.0/16"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithDisabledComponentShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithDisabledComponent("traefik"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithExtraArgShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithExtraArg("--write-kubeconfig-mode=644"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithReferenceShouldThrowWhenDestinationIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + IResourceBuilder destination = null!; + + var action = () => destination.WithReference(cluster); + + Assert.Throws(action); + } + + [Fact] + public void WithReferenceShouldThrowWhenSourceIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var container = appBuilder.AddContainer("app", "myimage"); + IResourceBuilder source = null!; + + var action = () => container.WithReference(source); + + Assert.Throws(action); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs new file mode 100644 index 000000000..906d590fd --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -0,0 +1,191 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K8sManifestResourceTests +{ + // ── AddK8sManifest registration ─────────────────────────────────────────── + + [Fact] + public void AddK8sManifestAddsResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./k8s/widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("widget-crd", resource.Name); + } + + [Fact] + public void AddK8sManifestStoresPath() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("crds", "./k8s/crds/"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("./k8s/crds/", resource.Path); + } + + [Fact] + public void AddK8sManifestParentIsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, resource.Parent); + } + + [Fact] + public void AddK8sManifestImplementsIResourceWithParent() + { + var resource = new K8sManifestResource( + "crd", "./crd.yaml", new K3sClusterResource("k8s")); + + var nonGeneric = resource as IResourceWithParent; + Assert.NotNull(nonGeneric); + Assert.Same(resource.Parent, nonGeneric.Parent); + } + + [Fact] + public void AddK8sManifestImplementsIResourceWithWaitSupport() + { + var resource = new K8sManifestResource( + "crd", "./crd.yaml", new K3sClusterResource("k8s")); + + Assert.IsAssignableFrom(resource); + } + + [Fact] + public void AddK8sManifestIsExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); + } + + [Fact] + public void ClusterTracksRegisteredManifests() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + cluster.AddK8sManifest("rbac", "./rbac.yaml"); + + Assert.Contains("widget-crd", cluster.Resource.Manifests); + Assert.Contains("rbac", cluster.Resource.Manifests); + Assert.Equal(2, cluster.Resource.Manifests.Count); + } + + [Fact] + public void MultipleManifestsCanBeRegisteredOnSameCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("crd1", "./crd1.yaml"); + cluster.AddK8sManifest("crd2", "./crd2.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Equal(2, model.Resources.OfType().Count()); + } + + // ── Public API null-guard tests ─────────────────────────────────────────── + + [Fact] + public void AddK8sManifestShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.AddK8sManifest("crd", "./crd.yaml"); + Assert.Throws(action); + } + + [Fact] + public void AddK8sManifestShouldThrowWhenNameIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddK8sManifest(null!, "./crd.yaml"); + Assert.Throws(action); + } + + [Fact] + public void AddK8sManifestShouldThrowWhenPathIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddK8sManifest("crd", null!); + Assert.Throws(action); + } + + // ── File resolution ─────────────────────────────────────────────────────── + + [Fact] + public void ResolveFilesSingleFile() + { + // Create a temp file to test with + var file = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid():N}.yaml"); + File.WriteAllText(file, "apiVersion: v1\nkind: ConfigMap"); + + try + { + var files = K3sManifestBuilderExtensions.ResolveFilesForTest(file); + Assert.Single(files, f => f == file); + } + finally + { + File.Delete(file); + } + } + + [Fact] + public void ResolveFilesDirectory() + { + var dir = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "b.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "a.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "c.yml"), ""); + + try + { + var files = K3sManifestBuilderExtensions.ResolveFilesForTest(dir); + + // Lexicographic order, all YAML extensions + Assert.Equal(3, files.Count); + Assert.Equal("a.yaml", Path.GetFileName(files[0])); + Assert.Equal("b.yaml", Path.GetFileName(files[1])); + Assert.Equal("c.yml", Path.GetFileName(files[2])); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } +} From 4dda24bf0f2e1743eef8fd678d05e2a7297c31af Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Thu, 14 May 2026 01:27:32 +0200 Subject: [PATCH 02/29] feat: proper support for service endpoints, helm and manifests resources --- .github/workflows/tests.yaml | 1 + .gitignore | 2 + CommunityToolkit.Aspire.slnx | 1 + .../Program.cs | 39 +- .../HelmContainerImageTags.cs | 8 + .../HelmReleaseHealthCheck.cs | 19 - .../HelmReleaseResource.cs | 23 +- .../K3sBuilderExtensions.Helm.cs | 396 +++-------------- .../K3sBuilderExtensions.Manifest.cs | 402 ++++-------------- .../K3sBuilderExtensions.ServiceEndpoint.cs | 218 ++++++++++ .../K3sBuilderExtensions.cs | 177 ++++---- .../K3sClusterOptions.cs | 40 ++ .../K3sClusterResource.cs | 29 +- .../K3sInProcessPortForwarder.cs | 169 ++++---- .../K3sReadinessHealthCheck.cs | 209 ++++----- .../K3sServiceEndpointResource.cs | 47 ++ .../K8sManifestResource.cs | 38 +- .../KubeconfigInjectionStrategy.cs | 33 -- .../KubectlContainerImageTags.cs | 11 + ...Aspire.Hosting.K3s.IntegrationTests.csproj | 8 + .../K3sIntegrationTests.cs | 158 +++++++ .../HelmReleaseResourceTests.cs | 210 ++++----- .../K3sClusterResourceTests.cs | 13 +- .../K8sManifestResourceTests.cs | 26 +- 24 files changed, 1027 insertions(+), 1250 deletions(-) create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4fae786ae..f24815979 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -38,6 +38,7 @@ jobs: Hosting.JavaScript.Extensions.Tests, Hosting.Java.Tests, Hosting.K3s.Tests, + Hosting.K3s.IntegrationTests, Hosting.k6.Tests, Hosting.Keycloak.Extensions.Tests, Hosting.KurrentDB.Tests, diff --git a/.gitignore b/.gitignore index 4795a4c5d..eb34c74c4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ examples/perl/**/local/* **cpanfile.snapshot **/.modules/ **/*.AppHost.TypeScript/nuget.config + +**/.k3s/ diff --git a/CommunityToolkit.Aspire.slnx b/CommunityToolkit.Aspire.slnx index 3382bdbb0..2d71685f9 100644 --- a/CommunityToolkit.Aspire.slnx +++ b/CommunityToolkit.Aspire.slnx @@ -281,6 +281,7 @@ + diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs index 00f476326..527bf16d0 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -1,34 +1,37 @@ // K3s hosting example // ────────────────────────────────────────────────────────────────────────────── // Prerequisites (host machine): -// • Docker with --privileged support +// • Docker with --privileged support (Linux or Docker Desktop on Mac/Windows) // • helm → https://helm.sh/docs/intro/install/ // • kubectl → https://kubernetes.io/docs/tasks/tools/ // // What this demonstrates: // 1. A k3s cluster starts inside a Docker container. -// 2. Headlamp (https://headlamp.dev) is installed as a child Helm release — -// a clickable http URL appears in the Aspire dashboard. -// 3. podinfo is installed — a lightweight demo app that shows the helm lifecycle. -// 4. Both releases are children of k8s in the Aspire resource tree. -// 5. WithPersistentState keeps the cluster data alive across AppHost restarts. +// 2. podinfo is installed via Helm — a lightweight demo app. +// 3. A K3sServiceEndpointResource exposes the podinfo service: +// • Host processes reach it at http://localhost:{port} +// • DCP-network containers reach it at http://host.docker.internal:{port} +// 4. WithDataVolume keeps the cluster state alive across AppHost restarts. // ────────────────────────────────────────────────────────────────────────────── var builder = DistributedApplication.CreateBuilder(args); -var cluster = builder.AddK3sCluster("k8s", configure: opts => - { - opts.AgentCount = 2; - }) +var cluster = builder + .AddK3sCluster("k8s") + .WithDataVolume() .WithLifetime(ContainerLifetime.Persistent); -// cluster.AddHelmRelease( -// name: "podinfo", -// chart: "podinfo", -// repo: "https://stefanprodan.github.io/podinfo", -// version: "6.7.1", -// @namespace: "podinfo") -// .WithHelmValue("service.type", "NodePort") -// .WithEndpoint("podinfo", servicePort: 9898, name: "web"); +var podinfo = cluster.AddHelmRelease( + name: "podinfo", + chart: "podinfo", + repo: "https://stefanprodan.github.io/podinfo", + version: "6.7.1", + @namespace: "podinfo"); + +// Expose the podinfo service as an Aspire endpoint resource. +// WaitForCompletion waits for the helm install container to exit with code 0 +// before starting the port-forward — no NodePort required. +cluster.AddServiceEndpoint("podinfo-web", "podinfo", servicePort: 9898, @namespace: "podinfo") + .WaitForCompletion(podinfo); builder.Build().Run(); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs new file mode 100644 index 000000000..885c77724 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs @@ -0,0 +1,8 @@ +namespace CommunityToolkit.Aspire.Hosting; + +internal static class HelmContainerImageTags +{ + internal const string Registry = "docker.io"; + internal const string Image = "alpine/helm"; + internal const string Tag = "3.17.3"; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs deleted file mode 100644 index eb0845e2d..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Aspire.Hosting.ApplicationModel; -using Microsoft.Extensions.Diagnostics.HealthChecks; - -namespace CommunityToolkit.Aspire.Hosting; - -/// -/// Lightweight health check that gates WaitFor(helmRelease) on completion of -/// the helm install lifecycle. Returns only -/// after is set by RunReleaseAsync. -/// -internal sealed class HelmReleaseHealthCheck(HelmReleaseResource release) : IHealthCheck -{ - public Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default) - => Task.FromResult(release.IsReady - ? HealthCheckResult.Healthy("Helm release is running") - : HealthCheckResult.Unhealthy("Helm release not yet ready")); -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs index 247f26e22..b5de3d52b 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -2,8 +2,12 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents a Helm chart release deployed to a k3s cluster. -/// Appears as a distinct dashboard entry; transitions Starting → Running -/// when all pods reach the Ready state. +/// +/// Runs as an alpine/helm container on the DCP network. The container polls for the +/// cluster kubeconfig (written when the cluster health check first passes), executes +/// helm upgrade --install --wait, and exits with code 0 on success. Use +/// WaitForCompletion(helmRelease) on resources that depend on the release being installed. +/// /// /// The Aspire resource name (also used as the Helm release name). /// The Helm release name passed to helm upgrade --install. @@ -14,7 +18,7 @@ public sealed class HelmReleaseResource( string releaseName, string @namespace, K3sClusterResource cluster) - : Resource(name), IResourceWithParent, IResourceWithWaitSupport + : ContainerResource(name), IResourceWithParent { /// public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); @@ -29,17 +33,4 @@ public sealed class HelmReleaseResource( internal string? RepoUrl { get; set; } internal string? Version { get; set; } internal Dictionary HelmValues { get; } = new(StringComparer.Ordinal); - internal List EndpointDefinitions { get; } = []; - - /// - /// Set to by the lifecycle when the helm install completes and - /// all pods are ready. The WaitFor(helmRelease) health check polls this flag. - /// - internal volatile bool IsReady; } - -/// Describes a Kubernetes service endpoint to expose from a Helm release. -/// The Kubernetes service name. -/// The service port number. -/// A friendly name shown in the dashboard. -internal sealed record HelmEndpointDefinition(string ServiceName, int ServicePort, string EndpointName); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index 156c83cdb..bd8dbb780 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -1,14 +1,6 @@ -using System.Collections.Immutable; -using System.Diagnostics; -using System.Net; -using System.Net.Sockets; using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; -using k8s; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; #pragma warning disable ASPIREATS001 // AspireExport is experimental @@ -22,17 +14,17 @@ public static class K3sHelmBuilderExtensions /// /// Adds a Helm release as a child resource of the k3s cluster. /// - /// Follows the same pattern as PostgresServerResource.AddDatabase: the release is - /// registered on the parent cluster, and the cluster's own - /// handler drives the install lifecycle for all registered releases. Helm output streams to - /// each release's individual log tab in the dashboard. + /// The release runs as a bitnami/helm container on the DCP network, executing + /// helm upgrade --install --wait then exiting. No host-side helm binary + /// is required. Use WaitForCompletion(helmRelease) on resources that depend on + /// the release being fully installed. /// /// /// The k3s cluster resource builder. - /// Resource name — also the Helm release name. + /// Resource name — also used as the Helm release name. /// Chart name. Add for remote charts. - /// Optional Helm repository URL (passed as --repo). - /// Optional chart version (--version). + /// Optional Helm repository URL. + /// Optional chart version. /// Target namespace (created automatically). /// A builder for the . [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] @@ -57,22 +49,43 @@ public static IResourceBuilder AddHelmRelease( Version = version, }; - // Register the release on the parent cluster — mirrors PostgresServerResource.AddDatabase(). cluster.AddHelmRelease(release.Name, release.ReleaseName); - // Health check that satisfies WaitFor(helmRelease) on dependent resources. - // Returns Healthy only after the install lifecycle sets release.IsReady = true. - var healthCheckKey = $"helm_{name}_ready"; - builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( - healthCheckKey, - sp => new HelmReleaseHealthCheck(release), - failureStatus: HealthStatus.Unhealthy, - tags: null)); + // The helm installer container mounts container/kubeconfig.yaml so it can reach + // the k3s API via DCP DNS (https://{clusterName}:6443). The directory is created + // by AddK3sCluster; the kubeconfig file is written by K3sReadinessHealthCheck on + // first successful health check. WaitFor(cluster) guarantees the file exists. + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); + + var (helmRegistry, helmImage, helmTag) = cluster.HelmImageInfo; return builder.ApplicationBuilder .AddResource(release) + .WithImage(helmImage, helmTag) + .WithImageRegistry(helmRegistry) + .WithEntrypoint("/bin/sh") + + // The install script is injected as /helm-install.sh via WithContainerFiles. + // The callback fires when the container is being started (after WaitFor(cluster) + // is satisfied), so all WithHelmValue() calls have been made by then. + .WithContainerFiles("/", async (ctx, ct) => + { + var script = BuildHelmScript(release); + return [new ContainerFile + { + Name = "helm-install.sh", + Contents = script, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]; + }) + .WithArgs("/helm-install.sh") + .WithBindMount(containerKubeconfigDir, "/root/.kube") + .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + .WithIconName("Rocket") .ExcludeFromManifest() - .WithHealthCheck(healthCheckKey) .WithInitialState(new CustomResourceSnapshot { ResourceType = "Helm Release", @@ -103,324 +116,43 @@ public static IResourceBuilder WithHelmValue( return builder; } - /// - /// Exposes a Kubernetes service from this release as a clickable endpoint in the dashboard. - /// The NodePort is auto-discovered and forwarded in-process via the KubernetesClient WebSocket API. - /// - public static IResourceBuilder WithEndpoint( - this IResourceBuilder builder, - string serviceName, - int servicePort, - string name) - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentNullException.ThrowIfNull(serviceName); - ArgumentNullException.ThrowIfNull(name); - - builder.Resource.EndpointDefinitions.Add( - new HelmEndpointDefinition(serviceName, servicePort, name)); - - return builder; - } - - // ── Lifecycle (driven from the parent cluster's ResourceReadyEvent) ──────── + // ── Script generation ───────────────────────────────────────────────────── - /// - /// Runs the full install lifecycle for : - /// helm install → NodePort discovery → in-process port-forward → Running. - /// Called by the parent cluster's handler — - /// the same pattern Postgres uses for database creation in AddPostgres. - /// - internal static async Task RunReleaseAsync( - HelmReleaseResource release, - K3sClusterResource cluster, - ResourceNotificationService notifications, - ILogger logger, - CancellationToken ct) + // Visible for testing. + internal static string BuildHelmScript(HelmReleaseResource release) { - await notifications.PublishUpdateAsync(release, - state => state with { State = KnownResourceStates.Starting }) - .ConfigureAwait(false); - - try - { - var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); - - if (kubeconfigYaml is null) - { - throw new InvalidOperationException( - "k3s kubeconfig is not yet available. " + - "The cluster ResourceReadyEvent fired before the health check populated the kubeconfig."); - } + var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - await RunHelmAsync(release, kubeconfigYaml, logger, ct).ConfigureAwait(false); + // Poll until the k3s health check writes the kubeconfig — the file appears only + // after all nodes are Ready. This replaces WaitFor(cluster) since a container + // cannot WaitFor its IResourceWithParent. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + sb.AppendLine(" sleep 5"); + sb.AppendLine("done"); - var urls = release.EndpointDefinitions.Count > 0 - ? await DiscoverAndStartPortForwardAsync(release, kubeconfigYaml, logger, ct) - .ConfigureAwait(false) - : ImmutableArray.Empty; - - // Set before PublishUpdateAsync so the health check unblocks WaitFor callers - // as soon as the Running state notification is processed. - release.IsReady = true; - - await notifications.PublishUpdateAsync(release, state => state with - { - State = KnownResourceStates.Running, - Urls = urls, - // Merge: keep all existing properties (the orchestrator injects ParentName which - // drives parent-child display in the dashboard) and only update/add our own. - // Replacing the entire Properties array would wipe out ParentName, causing the - // release to lose its parent and appear at the top level after going Running. - Properties = - [ - .. state.Properties.Where(p => - p.Name is not ("ReleaseName" or "Chart" or "Namespace" or "Version" or "ChartVersion")), - new ResourcePropertySnapshot("ReleaseName", release.ReleaseName), - new ResourcePropertySnapshot("Chart", release.Chart!), - new ResourcePropertySnapshot("ChartVersion", release.Version ?? "latest"), - new ResourcePropertySnapshot("Namespace", release.Namespace), - ], - }).ConfigureAwait(false); - } - catch (Exception ex) when (!ct.IsCancellationRequested) + if (release.RepoUrl is not null) { - logger.LogError(ex, "Helm release '{Name}' failed.", release.ReleaseName); - - await notifications.PublishUpdateAsync(release, - state => state with { State = KnownResourceStates.FailedToStart }) - .ConfigureAwait(false); + var alias = $"aspire-k3s-{release.ReleaseName}"; + sb.AppendLine($"helm repo add --force-update \"{alias}\" \"{release.RepoUrl}\""); + sb.AppendLine($"helm repo update \"{alias}\""); } - } - // ── helm subprocess ─────────────────────────────────────────────────────── + var chartRef = release.RepoUrl is not null + ? $"aspire-k3s-{release.ReleaseName}/{release.Chart}" + : release.Chart!; - private static async Task RunHelmAsync( - HelmReleaseResource release, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var tempKubeconfig = Path.Combine( - Path.GetTempPath(), - $"aspire-k3s-helm-{Environment.ProcessId}-{release.ReleaseName}.yaml"); + sb.Append($"helm upgrade --install \"{release.ReleaseName}\" \"{chartRef}\""); + sb.Append($" --namespace \"{release.Namespace}\" --create-namespace"); + sb.Append(" --wait --timeout 10m"); - await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) - .ConfigureAwait(false); - - try - { - // When a repo URL is provided, add/update the repo first. - // The --repo shorthand in helm upgrade --install does not work reliably for - // all chart repositories (e.g. kubernetes-dashboard returns 404 via --repo). - // Official docs always show: helm repo add → helm repo update → helm install. - string? repoAlias = null; - if (release.RepoUrl is not null) - { - repoAlias = $"aspire-k3s-{release.ReleaseName}"; - await RunHelmCommandAsync( - logger, - ["repo", "add", "--force-update", repoAlias, release.RepoUrl], - ct).ConfigureAwait(false); + if (release.Version is not null) + sb.Append($" --version \"{release.Version}\""); - await RunHelmCommandAsync( - logger, - ["repo", "update", repoAlias], - ct).ConfigureAwait(false); - } + foreach (var (key, value) in release.HelmValues) + sb.Append($" --set \"{key}={value}\""); - var args = BuildHelmInstallArgs( - release.ReleaseName, release.Chart!, repoAlias, - release.Version, release.Namespace, release.HelmValues, - tempKubeconfig); - - logger.LogInformation("Running: helm {Args}", string.Join(' ', args)); - - await RunHelmCommandAsync(logger, args, ct).ConfigureAwait(false); - } - finally - { - try { File.Delete(tempKubeconfig); } catch { /* best effort */ } - } - } - - // ── Argument builder ────────────────────────────────────────────────────── - - /// Runs a helm subcommand, logging output and failing on non-zero exit. - private static async Task RunHelmCommandAsync( - ILogger logger, - IEnumerable args, - CancellationToken ct) - { - var psi = new ProcessStartInfo("helm") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - foreach (var arg in args) - { - psi.ArgumentList.Add(arg); - } - - using var process = Process.Start(psi) - ?? throw new InvalidOperationException("Failed to start helm process."); - - process.OutputDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogInformation("{Line}", e.Data); - }; - process.ErrorDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogWarning("{Line}", e.Data); - }; - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - - await process.WaitForExitAsync(ct).ConfigureAwait(false); - - if (process.ExitCode != 0) - { - throw new InvalidOperationException( - $"helm {string.Join(' ', psi.ArgumentList)} failed (exit code {process.ExitCode})."); - } - } - - internal static IReadOnlyList BuildHelmInstallArgs( - string releaseName, - string chart, - string? repoAlias, // null = no repo, non-null = "{alias}/{chart}" notation - string? version, - string @namespace, - IReadOnlyDictionary? values, - string kubeconfigPath) - { - // When a repo alias was registered via `helm repo add`, use "alias/chart" notation. - // Otherwise treat the chart as a path or OCI reference. - var chartRef = repoAlias is not null ? $"{repoAlias}/{chart}" : chart; - - var args = new List - { - "upgrade", "--install", - releaseName, - chartRef, - "--namespace", @namespace, - "--create-namespace", - "--wait", - "--timeout", "10m", - $"--kubeconfig={kubeconfigPath}", - }; - - if (version is not null) - { - args.Add("--version"); - args.Add(version); - } - - if (values is not null) - { - foreach (var (key, value) in values) - { - args.Add("--set"); - args.Add($"{key}={value}"); - } - } - - return args; - } - - // ── Port-forward ────────────────────────────────────────────────────────── - - private static async Task> DiscoverAndStartPortForwardAsync( - HelmReleaseResource release, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var urls = ImmutableArray.CreateBuilder(); - - using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); - using var k8sClient = new Kubernetes(config); - - foreach (var ep in release.EndpointDefinitions) - { - var nodePort = await DiscoverNodePortAsync( - k8sClient, release.ReleaseName, release.Namespace, - ep.ServiceName, ep.ServicePort, logger, ct).ConfigureAwait(false); - - var hostPort = AllocateHostPort(); - - var forwarder = new K3sInProcessPortForwarder( - kubeconfigYaml, - release.Namespace, - ep.ServiceName, // look up by service name, not release label - hostPort, - ep.ServicePort); - - _ = forwarder.RunAsync(logger, ct); - - var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; - urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); - - if (nodePort.HasValue) - { - urls.Add(new UrlSnapshot( - $"{ep.EndpointName} (container)", - $"{scheme}://{release.Parent.Name}:{nodePort.Value}", - IsInternal: true)); - } - } - - return urls.ToImmutable(); - } - - private static async Task DiscoverNodePortAsync( - Kubernetes k8sClient, - string releaseName, - string @namespace, - string serviceName, - int servicePort, - ILogger logger, - CancellationToken ct) - { - try - { - var services = await k8sClient.CoreV1.ListNamespacedServiceAsync( - @namespace, - labelSelector: $"app.kubernetes.io/instance={releaseName}", - cancellationToken: ct).ConfigureAwait(false); - - var port = services.Items - .FirstOrDefault(s => string.Equals( - s.Metadata.Name, serviceName, StringComparison.OrdinalIgnoreCase)) - ?.Spec.Ports - .FirstOrDefault(p => p.Port == servicePort); - - if (port?.NodePort is null) - { - logger.LogWarning( - "NodePort for {ServiceName}:{ServicePort} not found; container URL omitted.", - serviceName, servicePort); - } - - return port?.NodePort; - } - catch (Exception ex) - { - logger.LogWarning(ex, "NodePort discovery failed for service '{ServiceName}'.", serviceName); - return null; - } - } - - private static int AllocateHostPort() - { - using var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - listener.Stop(); - return port; + return sb.ToString(); } } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index f50f70ba4..40a911e55 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -1,15 +1,6 @@ -using System.Collections.Immutable; -using System.Diagnostics; -using System.Net; -using System.Net.Sockets; using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; -using k8s; -using k8s.Models; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; #pragma warning disable ASPIREATS001 // AspireExport is experimental @@ -22,15 +13,17 @@ public static class K3sManifestBuilderExtensions { /// /// Applies one or more Kubernetes YAML files to the cluster via - /// kubectl apply --server-side (Server-Side Apply). No bind-mount is required. + /// kubectl apply --server-side running inside a bitnami/kubectl container. + /// No host-side kubectl binary is required. + /// + /// After applying the manifests the container waits for any CRDs to reach the + /// Established condition, then exits with code 0. Use + /// WaitForCompletion(manifest) on dependent resources. + /// /// /// A single file: cluster.AddK8sManifest("crd", "./k8s/widget-crd.yaml") - /// A directory: all .yaml/.yml files applied lexicographically. - /// A glob: "./k8s/crds/*.yaml" + /// A directory: all .yaml/.yml files applied (kubectl handles ordering). /// - /// CRDs are detected automatically; the resource waits for the Established - /// condition via the KubernetesClient before transitioning to Running, so - /// WaitFor(manifest) correctly gates dependent operators. /// [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] public static IResourceBuilder AddK8sManifest( @@ -43,323 +36,91 @@ public static IResourceBuilder AddK8sManifest( ArgumentNullException.ThrowIfNull(path); var cluster = builder.Resource; - var manifest = new K8sManifestResource(name, path, cluster); - cluster.AddManifest(manifest.Name); - - var healthCheckKey = $"manifest_{name}_ready"; - builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( - healthCheckKey, - sp => new K8sManifestHealthCheck(manifest), - failureStatus: HealthStatus.Unhealthy, - tags: null)); - - return builder.ApplicationBuilder - .AddResource(manifest) - .ExcludeFromManifest() - .WithHealthCheck(healthCheckKey) - .WithInitialState(new CustomResourceSnapshot - { - ResourceType = "K8s Manifest", - State = KnownResourceStates.NotStarted, - Properties = [new ResourcePropertySnapshot("Path", path)], - }); - } - - /// - /// Exposes a Kubernetes service from this manifest as a clickable endpoint in the dashboard. - /// Traffic is forwarded in-process via the KubernetesClient WebSocket API. - /// - /// The manifest resource builder. - /// The Kubernetes service name. - /// The service port number. - /// Friendly name shown in the dashboard. - /// - /// The namespace containing the service. Defaults to "default". - /// For remote manifests (HTTP URLs) the namespace must be specified explicitly. - /// - public static IResourceBuilder WithEndpoint( - this IResourceBuilder builder, - string serviceName, - int servicePort, - string name, - string @namespace = "default") - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentNullException.ThrowIfNull(serviceName); - ArgumentNullException.ThrowIfNull(name); - - builder.Resource.EndpointDefinitions.Add( - new ManifestEndpointDefinition(serviceName, servicePort, name, @namespace)); - - return builder; - } + // Resolve to an absolute path so the bind-mount and container path are stable. + var absolutePath = System.IO.Path.IsPathRooted(path) + ? path + : System.IO.Path.GetFullPath( + System.IO.Path.Combine(builder.ApplicationBuilder.AppHostDirectory, path)); - // ── Lifecycle (called from the cluster's ResourceReadyEvent) ────────────── + string hostBindDir; + string containerManifestPath; - internal static async Task RunManifestAsync( - K8sManifestResource manifest, - K3sClusterResource cluster, - ResourceNotificationService notifications, - ILogger logger, - CancellationToken ct) - { - await notifications.PublishUpdateAsync(manifest, - state => state with { State = KnownResourceStates.Starting }) - .ConfigureAwait(false); - - try + if (Directory.Exists(absolutePath)) { - var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); - - if (kubeconfigYaml is null) - { - throw new InvalidOperationException( - "k3s kubeconfig is not yet available when applying manifest."); - } - - // Write a temp kubeconfig file — kubectl requires a path, same as helm. - var tempKubeconfig = Path.Combine( - Path.GetTempPath(), - $"aspire-k3s-manifest-{Environment.ProcessId}-{manifest.Name}.yaml"); - - await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) - .ConfigureAwait(false); - - try - { - var files = ResolveFiles(manifest.Path); - - logger.LogInformation( - "Applying {Count} manifest file(s) from '{Path}'", files.Count, manifest.Path); - - // kubectl apply --server-side (SSA) for each file. - foreach (var file in files) - { - await KubectlApplyAsync(file, tempKubeconfig, logger, ct) - .ConfigureAwait(false); - } - - // Wait for any CRDs to reach Established using the KubernetesClient. - await WaitForCrdsEstablishedAsync( - files, kubeconfigYaml, logger, ct).ConfigureAwait(false); - } - finally - { - try { File.Delete(tempKubeconfig); } catch { /* best effort */ } - } - - manifest.IsReady = true; - - // Start in-process port-forwards for any declared endpoints. - var urls = ImmutableArray.Empty; - if (manifest.EndpointDefinitions.Count > 0) - { - urls = await StartPortForwardAsync(manifest, kubeconfigYaml, logger, ct) - .ConfigureAwait(false); - } - - await notifications.PublishUpdateAsync(manifest, state => state with - { - State = KnownResourceStates.Running, - Urls = urls, - Properties = - [ - .. state.Properties.Where(p => p.Name is not "Path"), - new ResourcePropertySnapshot("Path", manifest.Path), - ], - }).ConfigureAwait(false); + hostBindDir = absolutePath; + containerManifestPath = "/k8s-manifests"; } - catch (Exception ex) when (!ct.IsCancellationRequested) + else { - logger.LogError(ex, "Failed to apply manifest '{Name}'.", manifest.Name); - - await notifications.PublishUpdateAsync(manifest, - state => state with { State = KnownResourceStates.FailedToStart }) - .ConfigureAwait(false); + hostBindDir = System.IO.Path.GetDirectoryName(absolutePath)!; + containerManifestPath = $"/k8s-manifests/{System.IO.Path.GetFileName(absolutePath)}"; } - } - - // ── kubectl apply ───────────────────────────────────────────────────────── - private static async Task KubectlApplyAsync( - string file, - string kubeconfigPath, - ILogger logger, - CancellationToken ct) - { - logger.LogInformation("Applying {File}", file); - - var psi = new ProcessStartInfo("kubectl") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - - psi.ArgumentList.Add("apply"); - psi.ArgumentList.Add("-f"); - psi.ArgumentList.Add(file); - psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); - psi.ArgumentList.Add("--server-side"); - psi.ArgumentList.Add("--field-manager=aspire-k3s"); - psi.ArgumentList.Add("--force-conflicts"); - - using var process = Process.Start(psi) - ?? throw new InvalidOperationException("Failed to start kubectl process."); - - process.OutputDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogInformation("{Line}", e.Data); - }; - process.ErrorDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogWarning("{Line}", e.Data); - }; - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - - await process.WaitForExitAsync(ct).ConfigureAwait(false); - - if (process.ExitCode != 0) - { - throw new InvalidOperationException( - $"kubectl apply failed for '{file}' (exit code {process.ExitCode})."); - } - } - - // ── Port-forward for manifest endpoints ────────────────────────────────── - - private static async Task> StartPortForwardAsync( - K8sManifestResource manifest, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var urls = ImmutableArray.CreateBuilder(); - - foreach (var ep in manifest.EndpointDefinitions) - { - var hostPort = AllocateHostPort(); - - var forwarder = new K3sInProcessPortForwarder( - kubeconfigYaml, - ep.Namespace, - ep.ServiceName, - hostPort, - ep.ServicePort); - - _ = forwarder.RunAsync(logger, ct); - - var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; - urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); - } - - return urls.ToImmutable(); - } - - private static int AllocateHostPort() - { - using var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - listener.Stop(); - return port; - } + var manifest = new K8sManifestResource(name, absolutePath, cluster); + cluster.AddManifest(manifest.Name); - // ── CRD readiness (in-process via KubernetesClient) ────────────────────── + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); - private static async Task WaitForCrdsEstablishedAsync( - IReadOnlyList files, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var crdNames = new List(); + var (kubectlRegistry, kubectlImage, kubectlTag) = cluster.KubectlImageInfo; - foreach (var file in files) - { - // Skip remote URLs — kubectl downloaded and applied them, but we can't parse - // them with KubernetesYaml locally. CRD detection for remote files is skipped; - // if you need CRD readiness gating, use a local file path instead. - if (file.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || - file.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var objects = await KubernetesYaml.LoadAllFromFileAsync(file) - .ConfigureAwait(false); - - foreach (var obj in objects) + return builder.ApplicationBuilder + .AddResource(manifest) + .WithImage(kubectlImage, kubectlTag) + .WithImageRegistry(kubectlRegistry) + .WithEntrypoint("/bin/sh") + .WithContainerFiles("/", async (ctx, ct) => { - if (obj is V1CustomResourceDefinition crd && crd.Metadata?.Name is { } crdName) + var script = BuildManifestScript(containerManifestPath); + return [new ContainerFile { - crdNames.Add(crdName); - } - } - } - - if (crdNames.Count == 0) - { - return; - } - - using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); - using var k8sClient = new Kubernetes(config); - - foreach (var crdName in crdNames) - { - await WaitForCrdEstablishedAsync(k8sClient, crdName, logger, ct) - .ConfigureAwait(false); - } - } - - private static async Task WaitForCrdEstablishedAsync( - Kubernetes k8sClient, - string crdName, - ILogger logger, - CancellationToken ct) - { - logger.LogInformation("Waiting for CRD '{Crd}' to reach Established...", crdName); - - while (!ct.IsCancellationRequested) - { - var crd = await k8sClient.ApiextensionsV1 - .ReadCustomResourceDefinitionAsync(crdName, cancellationToken: ct) - .ConfigureAwait(false); - - var established = crd.Status?.Conditions?.Any(c => - c.Type == "Established" && - string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true; - - if (established) + Name = "kubectl-apply.sh", + Contents = script, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]; + }) + .WithArgs("/kubectl-apply.sh") + .WithBindMount(hostBindDir, "/k8s-manifests") + .WithBindMount(containerKubeconfigDir, "/root/.kube") + .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + .WithIconName("Code") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot { - logger.LogInformation("CRD '{Crd}' is Established.", crdName); - return; - } - - await Task.Delay(TimeSpan.FromSeconds(2), ct).ConfigureAwait(false); - } + ResourceType = "K8s Manifest", + State = KnownResourceStates.NotStarted, + Properties = [new ResourcePropertySnapshot("Path", absolutePath)], + }); } - // ── File resolution ─────────────────────────────────────────────────────── - - // Exposed for unit tests via InternalsVisibleTo. - internal static IReadOnlyList ResolveFilesForTest(string path) => - ResolveFiles(path); + // ── Script generation ───────────────────────────────────────────────────── - private static IReadOnlyList ResolveFiles(string path) + internal static string BuildManifestScript(string containerManifestPath) { - // kubectl apply -f supports HTTPS URLs natively — pass through as-is. - if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || - path.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - { - return [path]; - } + var sb = new StringBuilder("#!/bin/sh\nset -e\n"); + + // Poll until the k3s health check writes the kubeconfig — same pattern as the + // helm installer. Replaces WaitFor(cluster) for child resources. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + sb.AppendLine(" sleep 5"); + sb.AppendLine("done"); + + sb.AppendLine($"kubectl apply -f \"{containerManifestPath}\" --server-side --field-manager=aspire-k3s --force-conflicts"); + // Wait for CRD Established condition if any CRDs are present. + // The check guard prevents failure when no CRDs were applied. + sb.AppendLine("if kubectl get crd --no-headers 2>/dev/null | grep -q .; then"); + sb.AppendLine(" kubectl wait --for=condition=Established crd --all --timeout=300s"); + sb.AppendLine("fi"); + return sb.ToString(); + } + // Keep for unit tests — file resolution logic is the same. + internal static IReadOnlyList ResolveFilesForTest(string path) + { if (Directory.Exists(path)) { return [ @@ -382,19 +143,4 @@ private static IReadOnlyList ResolveFiles(string path) } } -/// -/// Health check that satisfies WaitFor(manifest). -/// Returns once all files are applied -/// and any CRDs have reached the Established condition. -/// -internal sealed class K8sManifestHealthCheck(K8sManifestResource manifest) : IHealthCheck -{ - public Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default) - => Task.FromResult(manifest.IsReady - ? HealthCheckResult.Healthy("Manifests applied") - : HealthCheckResult.Unhealthy("Manifests not yet applied")); -} - #pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs new file mode 100644 index 000000000..dfbe5ecb8 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -0,0 +1,218 @@ +using System.Collections.Immutable; +using System.Net; +using System.Net.Sockets; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for exposing Kubernetes services from a k3s cluster into the Aspire network. +/// +public static class K3sServiceEndpointExtensions +{ + /// + /// Exposes a Kubernetes service as a first-class Aspire endpoint resource. + /// + /// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready, + /// binding to 0.0.0.0:{hostPort}. Use WaitFor to sequence after a + /// or that deploys the service. + /// + /// + /// + /// + /// var nginx = cluster.AddHelmRelease("nginx", "nginx", repo: "https://charts.bitnami.com/bitnami"); + /// var ui = cluster.AddServiceEndpoint("nginx-ui", "nginx", servicePort: 80) + /// .WaitFor(nginx); + /// builder.AddProject<Projects.Api>("api") + /// .WaitFor(ui) + /// .WithReference(ui); + /// + /// + [AspireExport("addServiceEndpoint", + Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] + public static IResourceBuilder AddServiceEndpoint( + this IResourceBuilder builder, + [ResourceName] string name, + string serviceName, + int servicePort, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(serviceName); + + var cluster = builder.Resource; + var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster); + + var healthCheckKey = $"k3s_endpoint_{name}_ready"; + builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + healthCheckKey, + sp => new K3sServiceEndpointHealthCheck(endpoint), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + return builder.ApplicationBuilder + .AddResource(endpoint) + .ExcludeFromManifest() + .WithHealthCheck(healthCheckKey) + .WithIconName("ArrowRouting") + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K3s Service Endpoint", + State = KnownResourceStates.NotStarted, + Properties = + [ + new ResourcePropertySnapshot("ServiceName", serviceName), + new ResourcePropertySnapshot("ServicePort", servicePort.ToString()), + new ResourcePropertySnapshot("Namespace", @namespace), + ], + }); + } + + /// + /// Injects the service URL exposed by into + /// using the Aspire services__{name}__url convention. + /// + /// Host processes receive https://localhost:{port}. + /// Container resources receive https://host.docker.internal:{port}. + /// On Linux without Docker Desktop, add + /// --add-host=host.docker.internal:host-gateway to the container runtime args. + /// + /// + [AspireExport("withReference", + Description = "Injects the k3s service URL into a dependent resource")] + public static IResourceBuilder WithReference( + this IResourceBuilder destination, + IResourceBuilder source) + where TDestination : IResourceWithEnvironment + { + ArgumentNullException.ThrowIfNull(destination); + ArgumentNullException.ThrowIfNull(source); + + var ep = source.Resource; + var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + var envKey = $"services__{ep.Name}__url"; + + if (destination.Resource is ContainerResource) + { + return destination.WithEnvironment(ctx => + { + if (ep.IsReady) + ctx.EnvironmentVariables[envKey] = $"{scheme}://host.docker.internal:{ep.HostPort}"; + }); + } + + return destination.WithEnvironment(ctx => + { + if (ep.IsReady) + ctx.EnvironmentVariables[envKey] = $"{scheme}://localhost:{ep.HostPort}"; + }); + } + + // ── Lifecycle ───────────────────────────────────────────────────────────── + + internal static async Task RunEndpointAsync( + K3sServiceEndpointResource endpoint, + K3sClusterResource cluster, + ResourceNotificationService notifications, + ILogger logger, + CancellationToken ct) + { + await notifications.PublishUpdateAsync(endpoint, + state => state with { State = KnownResourceStates.Starting }) + .ConfigureAwait(false); + + try + { + var kubeconfigPath = K3sBuilderExtensions.GetLocalKubeconfigPath(cluster); + if (kubeconfigPath is null || !File.Exists(kubeconfigPath)) + { + throw new InvalidOperationException( + "k3s local kubeconfig is not yet available for service endpoint."); + } + + // Allocate a host port by opening a listener, reading the OS-assigned port, + // then closing it before the forwarder binds — the port stays reserved in the + // kernel TIME_WAIT for long enough that the forwarder wins the race. + var hostPort = AllocatePort(); + endpoint.HostPort = hostPort; + + var scheme = endpoint.ServicePort is 443 or 8443 ? "https" : "http"; + + var forwarder = new K3sInProcessPortForwarder( + kubeconfigPath, + endpoint.Namespace, + endpoint.ServiceName, + hostPort, + endpoint.ServicePort, + isReady => + { + endpoint.IsReady = isReady; + var state = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy; + var urls = isReady + ? BuildUrls(scheme, endpoint.Name, hostPort, cluster.Name) + : ImmutableArray.Empty; + + _ = notifications.PublishUpdateAsync(endpoint, s => s with + { + State = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy, + Urls = urls, + }); + }); + + _ = Task.Run(() => forwarder.RunAsync(logger, ct), ct); + + // Wait for the forwarder to signal ready (IsReady set via callback above). + // The health check also reads IsReady, so WaitFor on dependent resources + // naturally blocks until the port-forward is accepting connections. + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + logger.LogError(ex, "Service endpoint '{Name}' failed to start.", endpoint.Name); + await notifications.PublishUpdateAsync(endpoint, + state => state with { State = KnownResourceStates.FailedToStart }) + .ConfigureAwait(false); + } + } + + private static ImmutableArray BuildUrls( + string scheme, string endpointName, int hostPort, string clusterName) + => [ + new UrlSnapshot(endpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false), + new UrlSnapshot( + $"{endpointName} (container)", + $"{scheme}://host.docker.internal:{hostPort}", + IsInternal: true), + ]; + + private static int AllocatePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } +} + +/// +/// Health check that satisfies WaitFor(serviceEndpoint). +/// Returns once the port-forward is active. +/// +internal sealed class K3sServiceEndpointHealthCheck(K3sServiceEndpointResource endpoint) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + => Task.FromResult(endpoint.IsReady + ? HealthCheckResult.Healthy("Port-forward is active") + : HealthCheckResult.Unhealthy("Port-forward not yet active")); +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index 3399474a0..6e87500dc 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -1,11 +1,8 @@ using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; -using k8s; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; -using System.Net; -using System.Net.Sockets; #pragma warning disable ASPIREATS001 // AspireExport is experimental #pragma warning disable ASPIRECERTIFICATES001 // WithHttpsDeveloperCertificate is experimental @@ -41,9 +38,24 @@ public static IResourceBuilder AddK3sCluster( var options = new K3sClusterOptions(); configure?.Invoke(options); - var resource = new K3sClusterResource(name); + var resource = new K3sClusterResource(name) + { + HelmImageInfo = (options.HelmRegistry, options.HelmImage, options.HelmTag), + KubectlImageInfo = (options.KubectlRegistry, options.KubectlImage, options.KubectlTag), + }; var tag = options.ImageTag ?? K3sContainerImageTags.Tag; + // ── Kubeconfig directory on the host ────────────────────────────────── + // AppHostDirectory/.k3s/{name}/ holds three sub-directories: + // cluster/ — bind-mounted into the k3s container; k3s writes kubeconfig.yaml here + // local/ — rewritten by the health check with server: https://localhost:{port} + // container/ — rewritten by the health check with server: https://{name}:6443 + var kubeconfigDir = Path.Combine(builder.AppHostDirectory, ".k3s", name); + var clusterDir = Path.Combine(kubeconfigDir, "cluster"); + Directory.CreateDirectory(clusterDir); + + resource.KubeconfigDirectory = kubeconfigDir; + var resourceBuilder = builder.AddResource(resource) .WithImage(K3sContainerImageTags.Image, tag) .WithImageRegistry(K3sContainerImageTags.Registry) @@ -93,11 +105,7 @@ public static IResourceBuilder AddK3sCluster( .WithArgs("--kube-apiserver-arg=v=0") .WithArgs("--kube-controller-manager-arg=v=0") .WithArgs("--kube-scheduler-arg=v=0") - // Suppress kubelet INFO-level noise including the harmless cgroupsv2 race warning - // "Failed to kill all the processes attached to cgroup / os: process not initialized". - // This is a known benign race condition in Docker-in-Docker: the kubelet tries to - // force-kill pod cgroup processes that are already dead. The cgroup is still cleaned - // up correctly; only the redundant kill attempt fails. + // Suppress kubelet INFO-level noise including the harmless cgroupsv2 race warning. .WithArgs("--kubelet-arg=v=0") // ── API server endpoint ─────────────────────────────────────────── @@ -107,33 +115,24 @@ public static IResourceBuilder AddK3sCluster( name: K3sClusterResource.ApiServerEndpointName) // ── Docker / container runtime flags (mirrors k3d) ──────────────── - // Privileged mode is mandatory for iptables, network namespaces, and cgroups. .WithContainerRuntimeArgs("--privileged") - // k3d uses Docker's --init (tini) so that k3s's child processes are properly - // reaped and signals are forwarded correctly. Without it, zombie processes - // accumulate and shutdown becomes unreliable. .WithContainerRuntimeArgs("--init") - // Use the host user namespace — required when Docker is configured with userns-remap; - // a no-op otherwise. k3d always passes this flag. .WithContainerRuntimeArgs("--userns=host") - // Share the host's (Docker Desktop VM's) cgroup namespace instead of creating - // a new one. Without this, the k3s kubelet fails to create the "kubepods" cgroup - // hierarchy because the new isolated namespace has domain controllers in an invalid - // state for cgroupsv2. k3d always passes --cgroupns=host for this reason. .WithContainerRuntimeArgs("--cgroupns=host") - // Bind-mount the cgroup filesystem from the Docker Desktop VM into the container - // as read-write. With --cgroupns=host the container sees the host's cgroup namespace, - // but the mount is still read-only by default; making it rw lets the kubelet create - // sub-cgroups for pods (kubepods/besteffort/...). k3d always mounts this as rw. .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") - // tmpfs mounts for runtime sockets and PIDs — same as k3d defaults. .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + // ── Kubeconfig bind-mount ───────────────────────────────────────── + // Mounts AppHostDirectory/.k3s/{name}/cluster/ into the container so k3s + // writes its kubeconfig into a host-accessible directory. + // K3S_KUBECONFIG_OUTPUT tells k3s where to write the kubeconfig file. + // The health check polls File.Exists on the host side — no docker exec needed. + .WithBindMount(clusterDir, "/tmp/k3s-kubeconfig") + // ── Environment ─────────────────────────────────────────────────── - // Set an explicit cluster token (k3d always sets K3S_TOKEN). .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") - // World-readable kubeconfig so docker exec can read it without root. .WithEnvironment("K3S_KUBECONFIG_MODE", "644") + .WithEnvironment("K3S_KUBECONFIG_OUTPUT", "/tmp/k3s-kubeconfig/kubeconfig.yaml") .WithIconName("Kubernetes") .WithHttpsDeveloperCertificate(); @@ -212,32 +211,23 @@ public static IResourceBuilder AddK3sCluster( failureStatus: HealthStatus.Unhealthy, tags: null)); - // Postgres pattern: the parent cluster's ResourceReadyEvent drives ALL registered - // child lifecycles — both HelmReleases and K8sManifests — in parallel. + // The cluster's ResourceReadyEvent drives service endpoint port-forwards. + // HelmReleaseResource and K8sManifestResource containers are managed directly + // by DCP — they WaitFor the cluster and exit when their work completes. builder.Eventing.Subscribe(resource, (@event, ct) => { var appModel = @event.Services.GetRequiredService(); var notifications = @event.Services.GetRequiredService(); var loggerService = @event.Services.GetRequiredService(); - // Start all Helm release installs concurrently. - foreach (var release in appModel.Resources - .OfType() - .Where(r => ReferenceEquals(r.Parent, resource))) + // Start all service endpoint forwarders concurrently. + foreach (var ep in appModel.Resources + .OfType() + .Where(e => ReferenceEquals(e.Parent, resource))) { - var logger = loggerService.GetLogger(release); - _ = Task.Run(() => K3sHelmBuilderExtensions.RunReleaseAsync( - release, resource, notifications, logger, ct), ct); - } - - // Start all manifest applies concurrently. - foreach (var manifest in appModel.Resources - .OfType() - .Where(m => ReferenceEquals(m.Parent, resource))) - { - var logger = loggerService.GetLogger(manifest); - _ = Task.Run(() => K3sManifestBuilderExtensions.RunManifestAsync( - manifest, resource, notifications, logger, ct), ct); + var logger = loggerService.GetLogger(ep); + _ = Task.Run(() => K3sServiceEndpointExtensions.RunEndpointAsync( + ep, resource, notifications, logger, ct), ct); } return Task.CompletedTask; @@ -310,13 +300,6 @@ public static IResourceBuilder WithExtraArg( /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. /// - /// The cluster resource builder. - /// - /// The volume name. When , an auto-generated name is used in the form - /// {appName}-{sha256}-{resourceName}-data — the same scheme used by - /// PostgresServerResource.WithDataVolume and all other Aspire hosting integrations. - /// - /// The resource builder for chaining. public static IResourceBuilder WithDataVolume( this IResourceBuilder builder, string? name = null) @@ -325,41 +308,30 @@ public static IResourceBuilder WithDataVolume( return builder .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s") - // Auto-restart on crash so a persistent cluster survives transient failures - // without requiring AppHost intervention. Docker will not restart the container - // on explicit stop (DCP shutdown), only on unexpected exits. .WithContainerRuntimeArgs("--restart=unless-stopped"); } /// - /// Injects the k3s kubeconfig into as a - /// KUBECONFIG_DATA environment variable (base-64-encoded YAML) — no files are written. + /// Injects the k3s kubeconfig into so it can authenticate + /// to the cluster. The injection method is selected automatically based on the resource type: /// /// - /// s (Auto or InlineData) receive the - /// container-network kubeconfig - /// (server: https://{resourceName}:6443). + /// s receive KUBECONFIG_DATA (base-64-encoded YAML + /// of the container-network kubeconfig) and KUBECONFIG=/var/k3s/kubeconfig.yaml + /// once the file is injected at container-start time. /// /// - /// Projects and executables (Auto or HostPath) receive the - /// host kubeconfig - /// (server: https://localhost:{allocatedPort}). + /// Projects and executables receive KUBECONFIG=<host path>/local/kubeconfig.yaml + /// pointing to a file that is accessible directly on the host filesystem. /// /// - /// Consuming code reads the variable and builds a client without touching disk: - /// - /// var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); - /// using var stream = new MemoryStream(bytes); - /// var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); - /// /// The variable is populated only after the cluster health check passes; use /// WaitFor(cluster) on the dependent resource to guarantee ordering. /// [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] public static IResourceBuilder WithReference( this IResourceBuilder destination, - IResourceBuilder source, - KubeconfigInjectionStrategy strategy = KubeconfigInjectionStrategy.Auto) + IResourceBuilder source) where TDestination : IResourceWithEnvironment { ArgumentNullException.ThrowIfNull(destination); @@ -367,43 +339,37 @@ public static IResourceBuilder WithReference( var cluster = source.Resource; - return destination.WithEnvironment(ctx => + if (destination.Resource is ContainerResource) { - // Select the right kubeconfig variant based on where the resource runs: - // • ContainerResource → container-network URL (reaches API server via DCP DNS) - // • Project/Executable → host URL (reaches API server via localhost port-mapping) - var useContainerVariant = - strategy == KubeconfigInjectionStrategy.ContainerNetwork - || (strategy == KubeconfigInjectionStrategy.Auto - && destination.Resource is ContainerResource); - - var cfg = useContainerVariant - ? cluster.ContainerKubeconfig - : cluster.AdminKubeconfig; - - if (cfg is not null) + // Containers receive KUBECONFIG_DATA (base64-encoded container-network kubeconfig). + // KubernetesClient reads this via: + // var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); + // using var stream = new MemoryStream(bytes); + // var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); + // For standard kubectl, use WithContainerFiles to inject a real file instead. + return destination.WithEnvironment(ctx => { - var yaml = KubernetesYaml.Serialize(cfg); + if (cluster.KubeconfigDirectory is null) return; + var path = Path.Combine(cluster.KubeconfigDirectory, "container", "kubeconfig.yaml"); + if (!File.Exists(path)) return; + var yaml = File.ReadAllText(path); ctx.EnvironmentVariables["KUBECONFIG_DATA"] = Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); - } + }); + } + + // Projects and executables: KUBECONFIG points to the host-accessible local kubeconfig. + // This file is regenerated on every AppHost start (port may change). + return destination.WithEnvironment(ctx => + { + if (cluster.KubeconfigDirectory is null) return; + var path = Path.Combine(cluster.KubeconfigDirectory, "local", "kubeconfig.yaml"); + ctx.EnvironmentVariables["KUBECONFIG"] = path; }); } - /// - /// Serialises the admin kubeconfig (server: https://localhost:{port}) to YAML. - /// Returns if the health check has not yet populated the config. - /// - internal static string? GetAdminKubeconfigYaml(K3sClusterResource cluster) => - cluster.AdminKubeconfig is null - ? null - : KubernetesYaml.Serialize(cluster.AdminKubeconfig); - /// /// Sets the container lifetime for the k3s cluster and all its agent nodes. - /// When is used the agents must also be - /// persistent — otherwise they are recreated on every AppHost restart while the server - /// retains its state, causing the node re-join sequence to fail. /// public static IResourceBuilder WithLifetime( this IResourceBuilder builder, @@ -411,13 +377,10 @@ public static IResourceBuilder WithLifetime( { ArgumentNullException.ThrowIfNull(builder); - // Apply to the cluster container (identical to the built-in generic WithLifetime). builder.WithAnnotation( new ContainerLifetimeAnnotation { Lifetime = lifetime }, ResourceAnnotationMutationBehavior.Replace); - // Propagate to every agent — they share the same cluster state volume and must - // have the same lifetime so nodes survive AppHost restarts in sync with the server. foreach (var agent in builder.Resource.AgentResources) { var existing = agent.Annotations.OfType().ToList(); @@ -432,6 +395,16 @@ public static IResourceBuilder WithLifetime( return builder; } + /// + /// Returns the path to the local/kubeconfig.yaml file for this cluster, + /// used by helm and manifest runners that invoke host-side tools. + /// Returns if the cluster directory is not yet configured. + /// + internal static string? GetLocalKubeconfigPath(K3sClusterResource cluster) => + cluster.KubeconfigDirectory is null + ? null + : Path.Combine(cluster.KubeconfigDirectory, "local", "kubeconfig.yaml"); + // cgroupsv2 fix adapted from moby/moby (Apache-2.0, used with permission by k3d). // See: https://github.com/k3d-io/k3d/blob/main/pkg/types/fixes/assets/k3d-entrypoint-cgroupv2.sh private const string K3sInitEntrypointScript = """ diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs index cb33f7aa7..ab67ca836 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -37,4 +37,44 @@ public sealed class K3sClusterOptions /// When , the default tag embedded in the package is used. /// public string? ImageTag { get; set; } + + // ── Helm installer image ────────────────────────────────────────────────── + + /// + /// Gets or sets the registry for the Helm installer container image. + /// Defaults to docker.io. + /// + public string HelmRegistry { get; set; } = HelmContainerImageTags.Registry; + + /// + /// Gets or sets the Helm installer container image name. + /// Defaults to alpine/helm. + /// + public string HelmImage { get; set; } = HelmContainerImageTags.Image; + + /// + /// Gets or sets the Helm installer container image tag. + /// Defaults to 3.17.3. + /// + public string HelmTag { get; set; } = HelmContainerImageTags.Tag; + + // ── kubectl image ───────────────────────────────────────────────────────── + + /// + /// Gets or sets the registry for the kubectl container image used by manifest applies. + /// Defaults to docker.io. + /// + public string KubectlRegistry { get; set; } = KubectlContainerImageTags.Registry; + + /// + /// Gets or sets the kubectl container image name used by manifest applies. + /// Defaults to alpine/k8s. + /// + public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; + + /// + /// Gets or sets the kubectl container image tag used by manifest applies. + /// Defaults to 1.32.3. + /// + public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index f7b983b5a..4e121d7f9 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -1,5 +1,3 @@ -using k8s.KubeConfigModels; - namespace Aspire.Hosting.ApplicationModel; /// @@ -10,20 +8,25 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) { internal const string ApiServerEndpointName = "api"; - /// - /// Gets the admin kubeconfig for host-side processes - /// (server: https://localhost:{allocatedPort}). - /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. - /// Serialise with KubernetesYaml.Serialize(AdminKubeconfig) when needed. - /// - internal K8SConfiguration? AdminKubeconfig { get; set; } + /// Container image settings for the Helm installer, resolved from cluster options. + internal (string Registry, string Image, string Tag) HelmImageInfo { get; set; } + = ("docker.io", "alpine/helm", "3.17.3"); + + /// Container image settings for the kubectl manifest applier, resolved from cluster options. + internal (string Registry, string Image, string Tag) KubectlImageInfo { get; set; } + = ("docker.io", "alpine/k8s", "1.32.3"); /// - /// Gets the kubeconfig for containers on the DCP Docker network - /// (server: https://{resourceName}:6443). - /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. + /// Host-side directory that holds all kubeconfig variants for this cluster. + /// Set by AddK3sCluster to AppHostDirectory/.k3s/{name}/. + /// Sub-directories: + /// + /// cluster/kubeconfig.yaml — raw file written by k3s (bind-mounted) + /// local/kubeconfig.yaml — server: https://localhost:{port} (host processes) + /// container/kubeconfig.yaml — server: https://{name}:6443 (DCP-network containers) + /// /// - internal K8SConfiguration? ContainerKubeconfig { get; set; } + internal string? KubeconfigDirectory { get; set; } private EndpointReference? _apiEndpoint; diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 1fa3eb074..1d704399b 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -1,108 +1,127 @@ -using System.Diagnostics; -using System.Text; +using System.Collections.Generic; +using System.Net; +using System.Net.Sockets; +using k8s; using Microsoft.Extensions.Logging; namespace CommunityToolkit.Aspire.Hosting; /// -/// Forwards a local TCP port to a Kubernetes service by running -/// kubectl port-forward service/{name} {localPort}:{servicePort} -n {namespace} -/// as a managed subprocess. +/// Forwards a local TCP port to a Kubernetes service using the KubernetesClient +/// WebSocket port-forward API — no kubectl binary required. /// -/// Mirrors what a developer types in a terminal — the most reliable approach for -/// k3s-in-Docker because kubectl handles WebSocket negotiation, kubelet routing, -/// and reconnect logic internally. +/// The listener binds to 0.0.0.0:{localPort} so both host processes +/// (localhost:{port}) and DCP-network containers +/// (host.docker.internal:{port}) can reach the service. /// /// internal sealed class K3sInProcessPortForwarder( - string kubeconfigYaml, + string kubeconfigPath, string @namespace, string serviceName, int localPort, - int servicePort) + int servicePort, + Action onReadyChanged) { public async Task RunAsync(ILogger logger, CancellationToken ct) { - logger.LogInformation( - "Port-forward: localhost:{Local} → svc/{Service}.{Ns}:{Port}", - localPort, serviceName, @namespace, servicePort); + var backoff = TimeSpan.FromSeconds(2); - var tempConfig = Path.Combine( - Path.GetTempPath(), - $"aspire-k3s-pf-{Environment.ProcessId}-{serviceName}.yaml"); - - await File.WriteAllTextAsync(tempConfig, kubeconfigYaml, Encoding.UTF8, ct) - .ConfigureAwait(false); - - try + while (!ct.IsCancellationRequested) { - while (!ct.IsCancellationRequested) + var listener = new TcpListener(IPAddress.Any, localPort); + try { - try - { - await RunKubectlAsync(tempConfig, logger, ct).ConfigureAwait(false); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - logger.LogWarning(ex, - "Port-forward for svc/{Service} exited; restarting in 5 s…", serviceName); + listener.Start(); + onReadyChanged(true); + + logger.LogInformation( + "Port-forward: 0.0.0.0:{Local} → svc/{Service}.{Ns}:{Port}", + localPort, serviceName, @namespace, servicePort); - await Task.Delay(TimeSpan.FromSeconds(5), ct).ConfigureAwait(false); + while (!ct.IsCancellationRequested) + { + var tcp = await listener.AcceptTcpClientAsync(ct).ConfigureAwait(false); + _ = Task.Run( + () => ForwardConnectionAsync(tcp, logger, ct), + CancellationToken.None); } } - } - finally - { - try { File.Delete(tempConfig); } catch { /* best-effort cleanup */ } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + logger.LogWarning(ex, + "Port-forward for svc/{Service} failed; retrying in {Delay}s…", + serviceName, backoff.TotalSeconds); + onReadyChanged(false); + } + finally + { + listener.Stop(); + } + + if (ct.IsCancellationRequested) break; + + try { await Task.Delay(backoff, ct).ConfigureAwait(false); } catch (OperationCanceledException) { break; } + backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 30)); } } - private async Task RunKubectlAsync(string kubeconfigPath, ILogger logger, CancellationToken ct) + private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, CancellationToken ct) { - var psi = new ProcessStartInfo("kubectl") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - - psi.ArgumentList.Add("port-forward"); - psi.ArgumentList.Add($"service/{serviceName}"); - psi.ArgumentList.Add($"{localPort}:{servicePort}"); - psi.ArgumentList.Add("-n"); - psi.ArgumentList.Add(@namespace); - psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); - - using var process = Process.Start(psi) - ?? throw new InvalidOperationException("Failed to start kubectl port-forward."); - - using var reg = ct.Register(() => + using var _ = tcp; + try { - try { process.Kill(entireProcessTree: true); } catch { /* already exited */ } - }); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); + using var k8sClient = new Kubernetes(config); - process.OutputDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogDebug("{Line}", e.Data); - }; - process.ErrorDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogDebug("{Line}", e.Data); - }; - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); + // Resolve the service to a running pod. + var svc = await k8sClient.CoreV1 + .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) + .ConfigureAwait(false); + + var selector = string.Join(",", + (svc.Spec.Selector ?? new Dictionary()).Select(kv => $"{kv.Key}={kv.Value}")); - await process.WaitForExitAsync(ct).ConfigureAwait(false); + var pods = await k8sClient.CoreV1 + .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ConfigureAwait(false); - if (process.ExitCode != 0 && !ct.IsCancellationRequested) + var pod = pods.Items.FirstOrDefault(p => + p.Status?.Phase == "Running" && + p.Status?.ContainerStatuses?.All(c => c.Ready) == true); + + if (pod is null) + { + logger.LogWarning( + "No ready pod found for service {Service}/{Ns} — connection dropped.", + serviceName, @namespace); + return; + } + + // Open WebSocket port-forward to the pod. + using var ws = await k8sClient.WebSocketNamespacedPodPortForwardAsync( + pod.Metadata.Name, @namespace, [servicePort], + cancellationToken: ct).ConfigureAwait(false); + + using var demuxer = new StreamDemuxer(ws, StreamType.PortForward); + demuxer.Start(); + + using var k8sStream = demuxer.GetStream((byte?)0, (byte?)0); + using var tcpStream = tcp.GetStream(); + + // Bidirectional byte pump until either side closes. + await Task.WhenAny( + tcpStream.CopyToAsync(k8sStream, ct), + k8sStream.CopyToAsync(tcpStream, ct)).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { } + catch (Exception ex) { - throw new InvalidOperationException( - $"kubectl port-forward exited with code {process.ExitCode}."); + logger.LogDebug(ex, "Port-forward connection for svc/{Service} closed.", serviceName); } } } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index f0cc8f1fd..a51a9e2e2 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using Aspire.Hosting.ApplicationModel; using k8s; using k8s.KubeConfigModels; @@ -9,27 +8,23 @@ namespace CommunityToolkit.Aspire.Hosting; /// /// Health check for . /// -/// Instead of probing GET /healthz (which requires authentication in Kubernetes 1.28+ -/// because anonymous-auth defaults to false), this check runs -/// docker exec {container} kubectl get nodes inside the k3s container. -/// kubectl inside the container uses the default in-cluster kubeconfig, so no external -/// credentials are needed and the result is authoritative: a node in Ready state -/// proves the API server, scheduler, and kubelet are all functional. -/// -/// -/// On first success the kubeconfig is read from the container via docker exec cat, -/// parsed into two variants, and stored on the resource: +/// Polls for the kubeconfig file written by k3s into the bind-mounted +/// AppHostDirectory/.k3s/{name}/cluster/kubeconfig.yaml. On first appearance +/// it rewrites the server URL for two variants: /// -/// — server: https://localhost:{port} -/// — server: https://{name}:6443 +/// local/kubeconfig.yaml — server: https://localhost:{allocatedPort} (host processes) +/// container/kubeconfig.yaml — server: https://{name}:6443 (DCP-network containers) /// +/// Then uses a cached client to call ListNodeAsync, +/// confirming that all expected nodes (server + agents) are in Ready state. +/// No docker exec is involved — works with any container runtime. /// /// internal sealed class K3sReadinessHealthCheck : IHealthCheck { private readonly K3sClusterResource _resource; private readonly EndpointReference _endpoint; - private bool _kubeconfigRead; + private Kubernetes? _cachedClient; internal K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) { @@ -43,148 +38,102 @@ public async Task CheckHealthAsync( CancellationToken cancellationToken = default) { if (!_endpoint.IsAllocated) - { return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); - } - var port = _endpoint.Port; + var dir = _resource.KubeconfigDirectory; + if (dir is null) + return HealthCheckResult.Unhealthy("Kubeconfig directory not configured on resource"); + + var rawPath = Path.Combine(dir, "cluster", "kubeconfig.yaml"); + if (!File.Exists(rawPath)) + return HealthCheckResult.Unhealthy("Waiting for k3s to write kubeconfig"); try { - var containerId = await FindContainerIdAsync(cancellationToken); - - if (containerId is null) - { - return HealthCheckResult.Unhealthy("k3s container not yet found via docker ps"); - } - - // Run kubectl get nodes inside the container where the default kubeconfig is - // already configured — avoids any authentication issue from the outside. - var nodesOutput = await RunDockerAsync( - ["exec", containerId, - "kubectl", "get", "nodes", - "--kubeconfig", "/etc/rancher/k3s/k3s.yaml", - "--no-headers"], - cancellationToken); - - if (nodesOutput is null) - { - return HealthCheckResult.Unhealthy( - "kubectl get nodes failed — k3s API server not yet ready"); - } - - // Count nodes actually in Ready state (excluding NotReady ones). - var readyNodeLines = nodesOutput - .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Where(line => line.Contains("Ready") && !line.Contains("NotReady")) - .ToArray(); - - // For multi-node clusters (WithAgentNodes), wait for server + all agents. - // For single-node, 1 Ready node is sufficient. - var expectedNodes = 1 + _resource.AgentCount; - - if (readyNodeLines.Length < expectedNodes) - { - return HealthCheckResult.Unhealthy( - $"k3s cluster: {readyNodeLines.Length}/{expectedNodes} nodes Ready"); - } - - if (!_kubeconfigRead) - { - var rawYaml = await RunDockerAsync( - ["exec", containerId, "cat", "/etc/rancher/k3s/k3s.yaml"], - cancellationToken); - - if (rawYaml is null) - { - return HealthCheckResult.Unhealthy( - "k3s kubeconfig not yet available inside the container"); - } - - var parsed = KubernetesYaml.Deserialize(rawYaml); - - _resource.AdminKubeconfig = - BuildConfig(parsed, $"https://localhost:{port}"); - _resource.ContainerKubeconfig = - BuildConfig(parsed, $"https://{_resource.Name}:6443"); - _kubeconfigRead = true; - } + var client = await EnsureClientAsync(rawPath, cancellationToken).ConfigureAwait(false); + + var nodes = await client.CoreV1 + .ListNodeAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var readyCount = nodes.Items.Count(n => + n.Status?.Conditions?.Any(c => + c.Type == "Ready" && + string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true); + + var expected = 1 + _resource.AgentCount; + if (readyCount < expected) + return HealthCheckResult.Unhealthy($"k3s cluster: {readyCount}/{expected} nodes Ready"); return HealthCheckResult.Healthy("k3s cluster is ready"); } + catch (Exception ex) when (IsTlsOrAuthFailure(ex)) + { + // Stale kubeconfig — cluster was recreated with new certs (e.g. data volume wiped). + // Invalidate everything; k3s will overwrite cluster/kubeconfig.yaml on next start. + _cachedClient?.Dispose(); + _cachedClient = null; + TryDelete(rawPath); + TryDelete(Path.Combine(dir, "local", "kubeconfig.yaml")); + TryDelete(Path.Combine(dir, "container", "kubeconfig.yaml")); + return HealthCheckResult.Unhealthy("k3s kubeconfig is stale — waiting for cluster refresh"); + } catch (Exception ex) { return HealthCheckResult.Unhealthy(ex.Message, ex); } } - private static K8SConfiguration BuildConfig(K8SConfiguration source, string serverUrl) + private async Task EnsureClientAsync(string rawPath, CancellationToken ct) + { + if (_cachedClient is not null) + return _cachedClient; + + var port = _endpoint.Port; + var dir = _resource.KubeconfigDirectory!; + + var rawYaml = await File.ReadAllTextAsync(rawPath, ct).ConfigureAwait(false); + var parsed = KubernetesYaml.Deserialize(rawYaml); + + var localDir = Path.Combine(dir, "local"); + Directory.CreateDirectory(localDir); + var localPath = Path.Combine(localDir, "kubeconfig.yaml"); + await File.WriteAllTextAsync(localPath, BuildConfigYaml(parsed, $"https://localhost:{port}"), ct) + .ConfigureAwait(false); + + var containerDir = Path.Combine(dir, "container"); + Directory.CreateDirectory(containerDir); + await File.WriteAllTextAsync( + Path.Combine(containerDir, "kubeconfig.yaml"), + BuildConfigYaml(parsed, $"https://{_resource.Name}:6443"), + ct).ConfigureAwait(false); + + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(localPath); + _cachedClient = new Kubernetes(config); + return _cachedClient; + } + + private static string BuildConfigYaml(K8SConfiguration source, string serverUrl) { var yaml = KubernetesYaml.Serialize(source); var copy = KubernetesYaml.Deserialize(yaml); - foreach (var cluster in copy.Clusters ?? []) { if (cluster.ClusterEndpoint is not null) - { cluster.ClusterEndpoint.Server = serverUrl; - } } - return copy; + return KubernetesYaml.Serialize(copy); } - private async Task FindContainerIdAsync(CancellationToken ct) - { - // docker ps --filter name=VALUE uses substring matching: "name=k8s" also matches - // "k8s-agent-0", "k8s-agent-1", etc. Use --format to get names alongside IDs and - // exclude agent containers whose names contain "-agent-". - var output = await RunDockerAsync( - ["ps", - "--filter", $"name={_resource.Name}", - "--format", "{{.ID}}\t{{.Names}}", - "--no-trunc"], - ct); - - if (output is null) - { - return null; - } - - return output - .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Select(line => line.Split('\t', 2)) - .Where(parts => parts.Length == 2 && !parts[1].Contains("-agent-")) - .Select(parts => parts[0].Trim()) - .FirstOrDefault(id => !string.IsNullOrWhiteSpace(id)); - } + private static bool IsTlsOrAuthFailure(Exception ex) => + ex is System.Security.Authentication.AuthenticationException + || ex.InnerException is System.Security.Authentication.AuthenticationException + || (ex is k8s.Autorest.HttpOperationException op && + op.Response?.StatusCode == System.Net.HttpStatusCode.Unauthorized); - private static async Task RunDockerAsync(string[] args, CancellationToken ct) + private static void TryDelete(string path) { - var psi = new ProcessStartInfo("docker") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - - foreach (var arg in args) - { - psi.ArgumentList.Add(arg); - } - - using var process = Process.Start(psi); - if (process is null) - { - return null; - } - - var output = await process.StandardOutput.ReadToEndAsync(ct); - await process.WaitForExitAsync(ct); - - return process.ExitCode == 0 && !string.IsNullOrWhiteSpace(output) - ? output - : null; + try { File.Delete(path); } catch { /* best effort */ } } } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs new file mode 100644 index 000000000..95d4c9f9a --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -0,0 +1,47 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Exposes a Kubernetes service running inside a k3s cluster as an Aspire endpoint resource. +/// +/// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready. +/// The forwarder binds to 0.0.0.0:{hostPort} so both host processes and DCP-network +/// containers can reach the service. +/// +/// +/// Host consumers receive services__{name}__url=https://localhost:{port}. +/// Container consumers receive services__{name}__url=https://host.docker.internal:{port}. +/// +/// +public sealed class K3sServiceEndpointResource( + string name, + string serviceName, + int servicePort, + string @namespace, + K3sClusterResource cluster) + : Resource(name), IResourceWithParent, IResourceWithWaitSupport +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the Kubernetes service name. + public string ServiceName { get; } = serviceName ?? throw new ArgumentNullException(nameof(serviceName)); + + /// Gets the service port number. + public int ServicePort { get; } = servicePort; + + /// Gets the Kubernetes namespace containing the service. + public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); + + /// + /// The host port allocated for the port-forward listener. + /// Set by RunEndpointAsync before the resource transitions to Running. + /// Consumers can use this to construct the service URL directly when needed. + /// + public int HostPort { get; internal set; } + + /// + /// when the port-forward is active and accepting connections. + /// Set by K3sInProcessPortForwarder; read by the health check. + /// + internal volatile bool IsReady; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index f0a6a361b..a87a687ee 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -2,46 +2,22 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via -/// Server-Side Apply. This is a child resource of , following -/// the same parent-child pattern as . +/// kubectl apply --server-side running inside a bitnami/kubectl container. /// -/// No kubectl binary is required — the KubernetesClient library handles the apply. -/// CRDs reach the Established condition before the resource transitions to -/// Running, so dependent resources can safely WaitFor the manifest. +/// The container polls for the cluster kubeconfig (written when the cluster health check +/// first passes), applies the manifests, waits for any CRDs to reach Established, +/// then exits with code 0. Use WaitForCompletion(manifest) on dependent resources. /// /// /// The Aspire resource name. -/// -/// Path to a single YAML file, a directory, or a glob pattern (*.yaml). -/// Directories and globs are expanded lexicographically. -/// +/// Absolute path to a single YAML file or a directory. /// The parent k3s cluster resource. public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) - : Resource(name), IResourceWithParent, IResourceWithWaitSupport + : ContainerResource(name), IResourceWithParent { /// public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); - /// Gets the manifest path, directory, or glob. + /// Gets the manifest path or directory on the host. public string Path { get; } = path ?? throw new ArgumentNullException(nameof(path)); - - /// - /// Set to by the lifecycle after all objects are applied and - /// (for CRDs) the Established condition is confirmed. - /// - internal volatile bool IsReady; - - /// Services to expose via in-process port-forward after the manifest is applied. - internal List EndpointDefinitions { get; } = []; } - -/// Describes a service endpoint to expose from a . -/// Kubernetes service name. -/// Service port number. -/// Friendly name shown in the dashboard. -/// Kubernetes namespace where the service lives. -internal sealed record ManifestEndpointDefinition( - string ServiceName, - int ServicePort, - string EndpointName, - string Namespace); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs deleted file mode 100644 index b747206b1..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs +++ /dev/null @@ -1,33 +0,0 @@ -namespace CommunityToolkit.Aspire.Hosting; - -/// -/// Controls which kubeconfig server-URL variant is injected via KUBECONFIG_DATA. -/// All variants are delivered as base-64-encoded YAML without writing any file. -/// -public enum KubeconfigInjectionStrategy -{ - /// - /// Selects the server URL automatically based on the resource type: - /// - /// Container resources receive the - /// container-network URL (https://{resourceName}:6443). - /// Projects and executables receive the host URL - /// (https://localhost:{allocatedPort}). - /// - /// - Auto, - - /// - /// Always inject the host-network kubeconfig (server: https://localhost:{port}). - /// Use when a container is launched with --network=host or when the caller - /// explicitly needs host-side connectivity. - /// - HostNetwork, - - /// - /// Always inject the DCP-network kubeconfig - /// (server: https://{resourceName}:6443). - /// Use when a host process needs to reach the cluster the same way containers do. - /// - ContainerNetwork, -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs new file mode 100644 index 000000000..eb6eedb68 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs @@ -0,0 +1,11 @@ +namespace CommunityToolkit.Aspire.Hosting; + +internal static class KubectlContainerImageTags +{ + internal const string Registry = "docker.io"; + // alpine/k8s: lightweight Alpine-based image that includes kubectl and other k8s tools. + // Same organisation as alpine/helm — consistent image family. + internal const string Image = "alpine/k8s"; + // Matches the Kubernetes version shipped by the default k3s tag. + internal const string Tag = "1.32.3"; +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj new file mode 100644 index 000000000..95df55b8f --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs new file mode 100644 index 000000000..f08fa0588 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -0,0 +1,158 @@ +using Aspire.Components.Common.Tests; +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Utils; +using CommunityToolkit.Aspire.Testing; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; + +/// +/// End-to-end integration tests that spin up a real k3s cluster inside Docker. +/// +/// Requirements: +/// +/// Linux with Docker (privileged containers required by k3s). +/// helm on PATH — used by AddHelmRelease. +/// kubectl on PATH — used by AddK8sManifest. +/// +/// The tests are gated by [RequiresDocker] and intended for the +/// ubuntu-latest-only CI job in tests.yaml. +/// +/// +[RequiresDocker] +public class K3sIntegrationTests : IAsyncLifetime +{ + private DistributedApplication? _app; + private IDistributedApplicationTestingBuilder? _builder; + + public async ValueTask InitializeAsync() + { + _builder = TestDistributedApplicationBuilder.Create(); + await ValueTask.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + if (_app is not null) + { + await _app.StopAsync(); + await _app.DisposeAsync(); + } + + _builder?.Dispose(); + } + + [Fact] + public async Task ClusterReachesRunningAndKubeconfigIsValid() + { + var cluster = _builder!.AddK3sCluster("k8s"); + _app = _builder.Build(); + + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + // local/kubeconfig.yaml must exist on the host. + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + Assert.True(File.Exists(kubeconfigPath), + $"Expected local kubeconfig at {kubeconfigPath}"); + + // container/kubeconfig.yaml must also exist. + var containerKubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "container", "kubeconfig.yaml"); + Assert.True(File.Exists(containerKubeconfigPath), + $"Expected container kubeconfig at {containerKubeconfigPath}"); + } + + [Fact] + public async Task HelmReleaseReachesRunning() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + cluster.AddHelmRelease( + name: "nginx", + chart: "nginx", + repo: "https://charts.bitnami.com/bitnami", + version: "18.3.6", + @namespace: "nginx"); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + await rns.WaitForResourceAsync("nginx", + s => s.Snapshot.State?.Text == KnownResourceStates.Running, cts.Token); + } + + [Fact] + public async Task ServiceEndpointExposesHttpPort() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + var nginx = cluster.AddHelmRelease( + name: "nginx", + chart: "nginx", + repo: "https://charts.bitnami.com/bitnami", + version: "18.3.6", + @namespace: "nginx"); + + cluster.AddServiceEndpoint("nginx-web", "nginx", servicePort: 80, @namespace: "nginx") + .WaitForCompletion(nginx); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + // nginx is a run-to-completion container — wait for it to exit (Exited state) + await rns.WaitForResourceAsync("nginx", + s => s.Snapshot.State?.Text == "Exited", cts.Token); + await rns.WaitForResourceHealthyAsync("nginx-web", cts.Token); + + // Find the allocated port from the endpoint resource. + var model = _app.Services.GetRequiredService(); + var ep = model.Resources.OfType().Single(); + + Assert.True(ep.HostPort > 0, "HostPort should be allocated"); + + using var http = new HttpClient(); + var response = await http.GetAsync($"http://localhost:{ep.HostPort}", cts.Token); + Assert.True(response.IsSuccessStatusCode, + $"Expected HTTP 200 from nginx at localhost:{ep.HostPort}, got {response.StatusCode}"); + } + + [Fact] + public async Task WithReferenceInjectsKubeconfigForProject() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + // WithReference on a project injects KUBECONFIG pointing to local/kubeconfig.yaml. + // We verify the env var would be set by checking the cluster state. + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + var yaml = await File.ReadAllTextAsync(kubeconfigPath, cts.Token); + Assert.Contains("localhost", yaml); + Assert.DoesNotContain("127.0.0.1:6443", yaml); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 1269e0a34..02d3fc3c4 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -21,10 +21,10 @@ public void AddHelmReleaseAddsHelmReleaseResourceWithCorrectName() } [Fact] - public void AddHelmReleaseDoesNotCreateSeparateInstallResource() + public void HelmReleaseResourceIsContainerResource() { - // helm upgrade --install runs internally inside the HelmReleaseResource lifecycle; - // no separate ExecutableResource is added to the application model. + // HelmReleaseResource extends ContainerResource — it runs bitnami/helm in Docker. + // No host-side helm binary required. WaitForCompletion waits for exit code 0. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); @@ -33,7 +33,8 @@ public void AddHelmReleaseDoesNotCreateSeparateInstallResource() using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - Assert.Empty(model.Resources.OfType()); + var resource = Assert.Single(model.Resources.OfType()); + Assert.IsAssignableFrom(resource); } [Fact] @@ -118,7 +119,6 @@ public void HelmReleaseParentIsCluster() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - cluster.AddHelmRelease("argocd", "argo-cd"); using var app = appBuilder.Build(); @@ -126,27 +126,7 @@ public void HelmReleaseParentIsCluster() var resource = Assert.Single(model.Resources.OfType()); Assert.Same(cluster.Resource, resource.Parent); - } - - [Fact] - public void HelmReleaseImplementsNonGenericIResourceWithParent() - { - // The Aspire dashboard uses the non-generic IResourceWithParent to group - // child resources under their parent. Verify both the generic and non-generic - // interfaces are satisfied and point to the same cluster resource. - var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s"); - cluster.AddHelmRelease("argocd", "argo-cd"); - - using var app = appBuilder.Build(); - var model = app.Services.GetRequiredService(); - - var resource = Assert.Single(model.Resources.OfType()); - - // Non-generic IResourceWithParent (used by the dashboard) - var nonGeneric = resource as IResourceWithParent; - Assert.NotNull(nonGeneric); - Assert.Same(cluster.Resource, nonGeneric.Parent); + Assert.IsAssignableFrom>(resource); } [Fact] @@ -168,39 +148,36 @@ public void WithHelmValueAccumulatesValues() } [Fact] - public void WithEndpointAccumulatesEndpoints() + public void AddServiceEndpointAddsEndpointResource() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); - cluster.AddHelmRelease("argocd", "argo-cd") - .WithEndpoint("argocd-server", servicePort: 443, name: "ui"); + cluster.AddServiceEndpoint("argocd-ui", "argocd-server", servicePort: 443, @namespace: "argocd"); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var resource = Assert.Single(model.Resources.OfType()); - var ep = Assert.Single(resource.EndpointDefinitions); + var ep = Assert.Single(model.Resources.OfType()); Assert.Equal("argocd-server", ep.ServiceName); Assert.Equal(443, ep.ServicePort); - Assert.Equal("ui", ep.EndpointName); + Assert.Equal("argocd", ep.Namespace); } [Fact] - public void WithEndpointMultipleEndpoints() + public void AddServiceEndpointMultipleEndpointsAllRegistered() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - cluster.AddHelmRelease("argocd", "argo-cd") - .WithEndpoint("argocd-server", 443, "ui") - .WithEndpoint("argocd-server", 80, "http"); + cluster.AddServiceEndpoint("ui", "argocd-server", 443); + cluster.AddServiceEndpoint("http", "argocd-server", 80); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var resource = Assert.Single(model.Resources.OfType()); - Assert.Equal(2, resource.EndpointDefinitions.Count); + Assert.Equal(2, model.Resources.OfType().Count()); } [Fact] @@ -218,127 +195,106 @@ public void HelmReleaseIsExcludedFromManifest() Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); } - // ── BuildHelmInstallArgs tests (pure logic, no DI needed) ───────────────── - - [Fact] - public void BuildHelmInstallArgsIncludesUpgradeInstall() - { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "argocd", "argo-cd", null, null, "argocd", null, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.Contains("upgrade", list); - Assert.Contains("--install", list); - Assert.Contains("argocd", list); - Assert.Contains("argo-cd", list); - } + // ── BuildHelmScript tests (pure logic, no DI needed) ────────────────────── - [Fact] - public void BuildHelmInstallArgsIncludesKubeconfig() + private static HelmReleaseResource MakeRelease( + string releaseName, string chart, string? repo, string? version, + string @namespace, Dictionary? values = null) { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); - - Assert.Contains("--kubeconfig=/tmp/admin.yaml", args); + var cluster = new K3sClusterResource("k8s"); + var r = new HelmReleaseResource(releaseName, releaseName, @namespace, cluster) + { + Chart = chart, + RepoUrl = repo, + Version = version, + }; + foreach (var kv in values ?? []) + r.HelmValues[kv.Key] = kv.Value; + return r; } [Fact] - public void BuildHelmInstallArgsIncludesWait() + public void BuildHelmScriptIncludesUpgradeInstall() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("argocd", "argo-cd", null, null, "argocd")); - Assert.Contains("--wait", args); + Assert.Contains("helm upgrade --install", script); + Assert.Contains("\"argocd\"", script); + Assert.Contains("\"argo-cd\"", script); } [Fact] - public void BuildHelmInstallArgsIncludesNamespace() + public void BuildHelmScriptIncludesWaitAndNamespace() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "my-ns", null, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.Contains("--namespace", list); - Assert.Contains("my-ns", list); - Assert.Contains("--create-namespace", list); - } + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "my-ns")); - [Fact] - public void BuildHelmInstallArgsWithRepoAliasUsesPrefixedChartRef() - { - // When a repo alias is pre-registered via `helm repo add`, BuildHelmInstallArgs - // uses "{alias}/{chart}" notation — NOT the --repo flag (which is unreliable). - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", "my-repo-alias", null, "default", null, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.DoesNotContain("--repo", list); - Assert.Contains("my-repo-alias/chart", list); + Assert.Contains("--wait", script); + Assert.Contains("--namespace \"my-ns\"", script); + Assert.Contains("--create-namespace", script); } [Fact] - public void BuildHelmInstallArgsWithNullAliasUsesChartDirectly() + public void BuildHelmScriptWithRepoAddsRepoSteps() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "oci://registry/chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", "https://my-repo.example.com", null, "default")); - Assert.DoesNotContain("--repo", args); - Assert.Contains("oci://registry/chart", args); + Assert.Contains("helm repo add", script); + Assert.Contains("helm repo update", script); + Assert.Contains("aspire-k3s-r/chart", script); } [Fact] - public void BuildHelmInstallArgsIncludesVersion() + public void BuildHelmScriptWithoutRepoSkipsRepoSteps() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, "7.8.0", "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "oci://registry/chart", null, null, "default")); - var list = args.ToArray(); - Assert.Contains("--version", list); - Assert.Contains("7.8.0", list); + Assert.DoesNotContain("helm repo add", script); + Assert.Contains("\"oci://registry/chart\"", script); } [Fact] - public void BuildHelmInstallArgsOmitsRepoWhenNull() + public void BuildHelmScriptIncludesVersion() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, "7.8.0", "default")); - Assert.DoesNotContain("--repo", args); + Assert.Contains("--version \"7.8.0\"", script); } [Fact] - public void BuildHelmInstallArgsOmitsVersionWhenNull() + public void BuildHelmScriptOmitsVersionWhenNull() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "default")); - Assert.DoesNotContain("--version", args); + Assert.DoesNotContain("--version", script); } [Fact] - public void BuildHelmInstallArgsIncludesSetValues() + public void BuildHelmScriptIncludesSetValues() { - var values = new Dictionary - { - ["service.type"] = "NodePort", - ["replicaCount"] = "2", - }; - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", values, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.Contains("--set", list); - Assert.Contains("service.type=NodePort", list); - Assert.Contains("replicaCount=2", list); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "default", new() + { + ["service.type"] = "NodePort", + ["replicaCount"] = "2", + })); + + Assert.Contains("--set \"service.type=NodePort\"", script); + Assert.Contains("--set \"replicaCount=2\"", script); } - // ── WaitFor support ─────────────────────────────────────────────────────── + // ── WaitForCompletion support ───────────────────────────────────────────── [Fact] - public void HelmReleaseHasHealthCheckForWaitForSupport() + public void HelmReleaseHasNoHealthCheckAnnotation() { - // WaitFor(helmRelease) is satisfied by the HelmReleaseHealthCheck, - // which flips IsReady once RunReleaseAsync completes. + // HelmReleaseResource is a run-to-completion container — consumers use + // WaitForCompletion(helmRelease) rather than WaitFor. No health check needed. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); cluster.AddHelmRelease("argocd", "argo-cd"); @@ -347,17 +303,7 @@ public void HelmReleaseHasHealthCheckForWaitForSupport() var model = app.Services.GetRequiredService(); var resource = Assert.Single(model.Resources.OfType()); - Assert.Contains(resource.Annotations.OfType(), a => - a.Key == "helm_argocd_ready"); - } - - [Fact] - public void HelmReleaseIsReadyFlagStartsFalse() - { - var resource = new HelmReleaseResource( - "argocd", "argocd", "default", new K3sClusterResource("k8s")); - - Assert.False(resource.IsReady); + Assert.Empty(resource.Annotations.OfType()); } // ── Public API null-guard tests ─────────────────────────────────────────── @@ -397,10 +343,10 @@ public void WithHelmValueShouldThrowWhenBuilderIsNull() } [Fact] - public void WithEndpointShouldThrowWhenBuilderIsNull() + public void AddServiceEndpointShouldThrowWhenBuilderIsNull() { - IResourceBuilder builder = null!; - var action = () => builder.WithEndpoint("svc", 443, "ui"); + IResourceBuilder builder = null!; + var action = () => builder.AddServiceEndpoint("ui", "svc", 443); Assert.Throws(action); } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 9d1811255..0ca9b8bf9 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -211,13 +211,14 @@ public void WithReferenceSetsKubeconfigDataEnvForContainer() } [Fact] - public void K3sClusterResourceHasNoKubeconfigDirectoryByDefault() + public void AddK3sClusterSetsKubeconfigDirectory() { - // Kubeconfig is now stored in-memory (K8SConfiguration objects) and - // never written to disk by the resource itself — docker exec reads it. - var resource = new K3sClusterResource("k8s"); - Assert.Null(resource.AdminKubeconfig); - Assert.Null(resource.ContainerKubeconfig); + var appBuilder = DistributedApplication.CreateBuilder(); + var clusterBuilder = appBuilder.AddK3sCluster("k8s"); + + // KubeconfigDirectory is set by AddK3sCluster under AppHostDirectory/.k3s/{name}/ + Assert.NotNull(clusterBuilder.Resource.KubeconfigDirectory); + Assert.EndsWith(Path.Combine(".k3s", "k8s"), clusterBuilder.Resource.KubeconfigDirectory); } [Fact] diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs index 906d590fd..c657655fc 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -23,7 +23,7 @@ public void AddK8sManifestAddsResourceWithCorrectName() } [Fact] - public void AddK8sManifestStoresPath() + public void AddK8sManifestStoresAbsolutePath() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); @@ -34,7 +34,11 @@ public void AddK8sManifestStoresPath() var model = app.Services.GetRequiredService(); var resource = Assert.Single(model.Resources.OfType()); - Assert.Equal("./k8s/crds/", resource.Path); + // Path is resolved to absolute at registration time. + Assert.True(System.IO.Path.IsPathRooted(resource.Path)); + Assert.EndsWith(System.IO.Path.Combine("k8s", "crds"), + resource.Path.TrimEnd(System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar), + StringComparison.OrdinalIgnoreCase); } [Fact] @@ -50,26 +54,18 @@ public void AddK8sManifestParentIsCluster() var resource = Assert.Single(model.Resources.OfType()); Assert.Same(cluster.Resource, resource.Parent); + Assert.IsAssignableFrom>(resource); } [Fact] - public void AddK8sManifestImplementsIResourceWithParent() + public void K8sManifestResourceIsContainerResource() { + // K8sManifestResource extends ContainerResource — it runs bitnami/kubectl in Docker. + // No host-side kubectl binary required. WaitForCompletion waits for exit code 0. var resource = new K8sManifestResource( "crd", "./crd.yaml", new K3sClusterResource("k8s")); - var nonGeneric = resource as IResourceWithParent; - Assert.NotNull(nonGeneric); - Assert.Same(resource.Parent, nonGeneric.Parent); - } - - [Fact] - public void AddK8sManifestImplementsIResourceWithWaitSupport() - { - var resource = new K8sManifestResource( - "crd", "./crd.yaml", new K3sClusterResource("k8s")); - - Assert.IsAssignableFrom(resource); + Assert.IsAssignableFrom(resource); } [Fact] From a2ce723c786a23c93a6e05ec5c5e77af0d82b4eb Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Thu, 14 May 2026 01:45:39 +0200 Subject: [PATCH 03/29] feat: support files for helm/manifest --- .../HelmReleaseResource.cs | 7 ++ .../K3sBuilderExtensions.Helm.cs | 43 +++++++ .../K3sBuilderExtensions.Manifest.cs | 105 +++++++++++------- .../HelmReleaseResourceTests.cs | 52 +++++++++ .../K8sManifestResourceTests.cs | 60 ++++++++++ 5 files changed, 226 insertions(+), 41 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs index b5de3d52b..65740bea8 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -33,4 +33,11 @@ public sealed class HelmReleaseResource( internal string? RepoUrl { get; set; } internal string? Version { get; set; } internal Dictionary HelmValues { get; } = new(StringComparer.Ordinal); + + /// + /// Absolute host paths of values files to inject into the helm container via + /// --values /helm-values/{filename}. + /// Populated by WithHelmValuesFile. + /// + internal List ValuesFiles { get; } = []; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index bd8dbb780..6799c8f8e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -82,6 +82,18 @@ public static IResourceBuilder AddHelmRelease( }]; }) .WithArgs("/helm-install.sh") + // Inject host-side values files declared via WithHelmValuesFile using + // Aspire's built-in WithContainerFiles(destinationPath, hostSourcePath). + // One call per file; each copies the file into /helm-values/ in the container. + // The callback wraps all files so it fires after all WithHelmValuesFile() calls. + .WithContainerFiles("/helm-values", async (ctx, ct) => + release.ValuesFiles + .Select(hostPath => (ContainerFileSystemItem)new ContainerFile + { + Name = System.IO.Path.GetFileName(hostPath), + SourcePath = hostPath, + }) + .ToList()) .WithBindMount(containerKubeconfigDir, "/root/.kube") .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") .WithIconName("Rocket") @@ -100,6 +112,33 @@ public static IResourceBuilder AddHelmRelease( }); } + /// + /// Injects a host-side YAML values file into the Helm installer container and + /// passes it as --values /helm-values/{filename} to helm upgrade --install. + /// Multiple files are applied in the order they are declared (last wins for overlapping keys). + /// + /// The Helm release resource builder. + /// + /// Path to the values YAML file on the host. Relative paths are resolved against + /// AppHostDirectory. + /// + public static IResourceBuilder WithHelmValuesFile( + this IResourceBuilder builder, + string path) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var absolutePath = System.IO.Path.IsPathRooted(path) + ? path + : System.IO.Path.GetFullPath( + System.IO.Path.Combine( + builder.ApplicationBuilder.AppHostDirectory, path)); + + builder.Resource.ValuesFiles.Add(absolutePath); + return builder; + } + /// /// Adds a Helm --set key=value argument to this release. /// @@ -149,6 +188,10 @@ internal static string BuildHelmScript(HelmReleaseResource release) if (release.Version is not null) sb.Append($" --version \"{release.Version}\""); + // Values files are injected into /helm-values/ by WithContainerFiles. + foreach (var hostPath in release.ValuesFiles) + sb.Append($" --values \"/helm-values/{System.IO.Path.GetFileName(hostPath)}\""); + foreach (var (key, value) in release.HelmValues) sb.Append($" --set \"{key}={value}\""); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 40a911e55..6cb79b838 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -12,18 +12,23 @@ namespace Aspire.Hosting; public static class K3sManifestBuilderExtensions { /// - /// Applies one or more Kubernetes YAML files to the cluster via - /// kubectl apply --server-side running inside a bitnami/kubectl container. - /// No host-side kubectl binary is required. + /// Applies one or more Kubernetes YAML files — or a Kustomize overlay — to the cluster + /// via a bitnami/kubectl container. No host-side kubectl binary is required. /// - /// After applying the manifests the container waits for any CRDs to reach the - /// Established condition, then exits with code 0. Use - /// WaitForCompletion(manifest) on dependent resources. + /// The container exits with code 0 after manifests are applied and any CRDs reach the + /// Established condition. Use WaitForCompletion(manifest) on dependent resources. /// + /// + /// Three modes, selected automatically based on : /// - /// A single file: cluster.AddK8sManifest("crd", "./k8s/widget-crd.yaml") - /// A directory: all .yaml/.yml files applied (kubectl handles ordering). + /// Single file — injected via WithContainerFiles, applied with kubectl apply -f. + /// Directory without kustomization.yaml — all .yaml/.yml files + /// injected via WithContainerFiles, applied with kubectl apply -f. + /// Kustomize overlay (directory contains kustomization.yaml or + /// kustomization.yml) — directory bind-mounted (preserving relative base references), + /// applied with kubectl apply -k. /// + /// /// [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] public static IResourceBuilder AddK8sManifest( @@ -37,25 +42,13 @@ public static IResourceBuilder AddK8sManifest( var cluster = builder.Resource; - // Resolve to an absolute path so the bind-mount and container path are stable. var absolutePath = System.IO.Path.IsPathRooted(path) ? path : System.IO.Path.GetFullPath( System.IO.Path.Combine(builder.ApplicationBuilder.AppHostDirectory, path)); - string hostBindDir; - string containerManifestPath; - - if (Directory.Exists(absolutePath)) - { - hostBindDir = absolutePath; - containerManifestPath = "/k8s-manifests"; - } - else - { - hostBindDir = System.IO.Path.GetDirectoryName(absolutePath)!; - containerManifestPath = $"/k8s-manifests/{System.IO.Path.GetFileName(absolutePath)}"; - } + bool isDirectory = Directory.Exists(absolutePath); + bool isKustomize = isDirectory && IsKustomizeDirectory(absolutePath); var manifest = new K8sManifestResource(name, absolutePath, cluster); cluster.AddManifest(manifest.Name); @@ -65,60 +58,92 @@ public static IResourceBuilder AddK8sManifest( var (kubectlRegistry, kubectlImage, kubectlTag) = cluster.KubectlImageInfo; - return builder.ApplicationBuilder + var resourceBuilder = builder.ApplicationBuilder .AddResource(manifest) .WithImage(kubectlImage, kubectlTag) .WithImageRegistry(kubectlRegistry) .WithEntrypoint("/bin/sh") + // Script is injected at "/kubectl-apply.sh". The script auto-detects whether + // /k8s-manifests contains a kustomization.yaml and uses -k or -f accordingly. .WithContainerFiles("/", async (ctx, ct) => - { - var script = BuildManifestScript(containerManifestPath); - return [new ContainerFile + [new ContainerFile { Name = "kubectl-apply.sh", - Contents = script, + Contents = BuildManifestScript(), Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, - }]; - }) + }]) .WithArgs("/kubectl-apply.sh") - .WithBindMount(hostBindDir, "/k8s-manifests") - .WithBindMount(containerKubeconfigDir, "/root/.kube") + .WithBindMount(containerKubeconfigDir, "/root/.kube"); + + if (isKustomize) + { + // Bind-mount the overlay directory so kubectl kustomize can resolve relative + // references to base manifests (e.g. ../../base). WithContainerFiles copies + // files, not directory structure, so it would break cross-directory references. + resourceBuilder.WithBindMount(absolutePath, "/k8s-manifests"); + } + else + { + // Single file or regular directory — use Aspire's built-in + // WithContainerFiles(destinationPath, hostSourcePath) which copies the + // file or all files in the directory into the container without a bind-mount. + resourceBuilder.WithContainerFiles("/k8s-manifests", absolutePath); + } + + return resourceBuilder .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") .WithIconName("Code") .ExcludeFromManifest() .WithInitialState(new CustomResourceSnapshot { - ResourceType = "K8s Manifest", + ResourceType = isKustomize ? "K8s Kustomize" : "K8s Manifest", State = KnownResourceStates.NotStarted, - Properties = [new ResourcePropertySnapshot("Path", absolutePath)], + Properties = + [ + new ResourcePropertySnapshot("Path", absolutePath), + new ResourcePropertySnapshot("Mode", isKustomize ? "kustomize" : "apply"), + ], }); } // ── Script generation ───────────────────────────────────────────────────── - internal static string BuildManifestScript(string containerManifestPath) + internal static string BuildManifestScript() { var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - // Poll until the k3s health check writes the kubeconfig — same pattern as the - // helm installer. Replaces WaitFor(cluster) for child resources. + // Poll until the k3s health check writes the kubeconfig. sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); - sb.AppendLine($"kubectl apply -f \"{containerManifestPath}\" --server-side --field-manager=aspire-k3s --force-conflicts"); + // Auto-detect kustomize: if a kustomization file is present, use -k. + // Otherwise use -f with server-side apply. + sb.AppendLine("if [ -f /k8s-manifests/kustomization.yaml ] || [ -f /k8s-manifests/kustomization.yml ]; then"); + sb.AppendLine(" echo 'Detected kustomization — using kubectl apply -k'"); + sb.AppendLine(" kubectl apply -k /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine("else"); + sb.AppendLine(" kubectl apply -f /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine("fi"); + // Wait for CRD Established condition if any CRDs are present. - // The check guard prevents failure when no CRDs were applied. sb.AppendLine("if kubectl get crd --no-headers 2>/dev/null | grep -q .; then"); sb.AppendLine(" kubectl wait --for=condition=Established crd --all --timeout=300s"); sb.AppendLine("fi"); + return sb.ToString(); } - // Keep for unit tests — file resolution logic is the same. + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static bool IsKustomizeDirectory(string directory) => + File.Exists(System.IO.Path.Combine(directory, "kustomization.yaml")) || + File.Exists(System.IO.Path.Combine(directory, "kustomization.yml")); + + // Exposed for unit tests. internal static IReadOnlyList ResolveFilesForTest(string path) { if (Directory.Exists(path)) @@ -135,9 +160,7 @@ internal static IReadOnlyList ResolveFilesForTest(string path) var pattern = System.IO.Path.GetFileName(path); if (pattern.Contains('*') || pattern.Contains('?')) - { return [..Directory.GetFiles(dir, pattern).Order(StringComparer.OrdinalIgnoreCase)]; - } return [path]; } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 02d3fc3c4..9d940fe58 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -349,4 +349,56 @@ public void AddServiceEndpointShouldThrowWhenBuilderIsNull() var action = () => builder.AddServiceEndpoint("ui", "svc", 443); Assert.Throws(action); } + + // ── WithHelmValuesFile tests ────────────────────────────────────────────── + + [Fact] + public void BuildHelmScriptIncludesValuesFiles() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("argocd", "argocd", "argocd", cluster) + { + Chart = "argo-cd", + }; + release.ValuesFiles.Add("/tmp/values.yaml"); + release.ValuesFiles.Add("/tmp/values-prod.yaml"); + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + Assert.Contains("--values \"/helm-values/values.yaml\"", script); + Assert.Contains("--values \"/helm-values/values-prod.yaml\"", script); + // Values files are applied before --set overrides (last wins). + var valuesIndex = script.IndexOf("--values", StringComparison.Ordinal); + var setIndex = script.IndexOf("--set", StringComparison.Ordinal); + Assert.True(valuesIndex < setIndex || setIndex == -1, + "--values flags should appear before --set flags"); + } + + [Fact] + public void WithHelmValuesFileAccumulatesAbsolutePaths() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + // Use a temp file so the path resolution succeeds. + var tempFile = Path.Combine(appBuilder.AppHostDirectory, "values.yaml"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValuesFile("values.yaml"); // relative to AppHostDirectory + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Single(resource.ValuesFiles); + Assert.Equal(tempFile, resource.ValuesFiles[0]); + } + + [Fact] + public void WithHelmValuesFileShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithHelmValuesFile("values.yaml"); + Assert.Throws(action); + } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs index c657655fc..a286f79ea 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -184,4 +184,64 @@ public void ResolveFilesDirectory() Directory.Delete(dir, recursive: true); } } + + // ── Script generation ───────────────────────────────────────────────────── + + [Fact] + public void BuildManifestScriptIncludesKubectlApply() + { + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + Assert.Contains("kubectl apply -f /k8s-manifests", script); + Assert.Contains("--server-side", script); + } + + [Fact] + public void BuildManifestScriptAutoDetectsKustomize() + { + // The script checks for kustomization.yaml at runtime — no path argument needed. + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + Assert.Contains("kustomization.yaml", script); + Assert.Contains("kubectl apply -k /k8s-manifests", script); + } + + [Fact] + public void BuildManifestScriptWaitsForKubeconfigBeforeApplying() + { + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + var kubeconfigWaitIndex = script.IndexOf("/root/.kube/kubeconfig.yaml", StringComparison.Ordinal); + var applyIndex = script.IndexOf("kubectl apply", StringComparison.Ordinal); + Assert.True(kubeconfigWaitIndex < applyIndex, "Kubeconfig wait must precede kubectl apply"); + } + + // ── Kustomize detection ─────────────────────────────────────────────────── + + [Fact] + public void AddK8sManifestKustomizeDirectoryShowsKustomizeResourceType() + { + var dir = Path.Combine(Path.GetTempPath(), $"kustomize-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "kustomization.yaml"), "resources: []"); + + try + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddK8sManifest("kustom", dir); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var typeSnapshot = resource.Annotations + .OfType() + .Select(a => a.InitialSnapshot.ResourceType) + .FirstOrDefault(); + + Assert.Equal("K8s Kustomize", typeSnapshot); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } } From d343ee3a65321600ec7a851efd1c51884b4f79c7 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 14:50:19 +0200 Subject: [PATCH 04/29] compliting left work --- .../Program.cs | 7 +- .../ValidationAppHost/apphost.ts | 82 ++++++++ .../ValidationAppHost/aspire.config.json | 26 +++ .../ValidationAppHost/package.json | 19 ++ .../ValidationAppHost/tsconfig.json | 20 ++ .../Annotations/HelmReleaseAnnotation.cs | 35 ---- .../KubernetesDashboardAnnotation.cs | 11 - .../Annotations/KustomizeAnnotation.cs | 11 - ...CommunityToolkit.Aspire.Hosting.K3s.csproj | 2 +- .../HelmReleaseResource.cs | 3 + .../K3sBuilderExtensions.Helm.cs | 17 +- .../K3sBuilderExtensions.cs | 2 + .../K3sClusterResource.cs | 5 + .../K3sServiceEndpointResource.cs | 3 + .../K8sManifestResource.cs | 3 + .../README.md | 192 +++++++++++++++--- .../CommunityToolkit.Aspire.Hosting.K3s.cs | 150 +++++++++++--- .../K3sIntegrationTests.cs | 160 ++++++++++++++- .../HelmReleaseResourceTests.cs | 50 ++++- 19 files changed, 651 insertions(+), 147 deletions(-) create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs index 527bf16d0..a25dce592 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -1,9 +1,10 @@ // K3s hosting example // ────────────────────────────────────────────────────────────────────────────── // Prerequisites (host machine): -// • Docker with --privileged support (Linux or Docker Desktop on Mac/Windows) -// • helm → https://helm.sh/docs/intro/install/ -// • kubectl → https://kubernetes.io/docs/tasks/tools/ +// • A container runtime that supports privileged Linux containers: +// - Linux: Docker Engine 20.10+ or rootful Podman 4.0+ +// - macOS / Windows: Docker Desktop (WSL2 / Hyper-V) +// No host-side helm or kubectl required — both run as containers. // // What this demonstrates: // 1. A k3s cluster starts inside a Docker container. diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts new file mode 100644 index 000000000..c9830b8f3 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts @@ -0,0 +1,82 @@ +import { createBuilder, ContainerLifetime } from './.modules/aspire.js'; + +const builder = await createBuilder(); + +// ── Runtime path (actually executed) ───────────────────────────────────────── +// Minimal cluster startup — validates that the core add/build/run path works. +const cluster = builder.addK3sCluster('k8s'); +const clusterResource = await cluster; +const _apiEndpoint = await clusterResource.apiEndpoint.get(); + +// ── Compile-time coverage ───────────────────────────────────────────────────── +// Guards with false so these are type-checked but never executed. +// Covers the full exported API surface without requiring Docker/k3s in CI. +const includeCompileOnlyScenarios = false; + +if (includeCompileOnlyScenarios) { + + // ── Cluster configuration ──────────────────────────────────────────────── + const configuredCluster = builder.addK3sCluster('k8s-configured') + .withK3sVersion('v1.32.3-k3s1') + .withPodSubnet('10.42.0.0/16') + .withServiceSubnet('10.43.0.0/16') + .withDisabledComponent('traefik') + .withExtraArg('--write-kubeconfig-mode=644') + .withDataVolume({ name: 'k8s-data' }) + .withLifetime(ContainerLifetime.Persistent); + + const configuredClusterResource = await configuredCluster; + const _configuredApiEndpoint = await configuredClusterResource.apiEndpoint.get(); + + // ── Helm release ───────────────────────────────────────────────────────── + const argocd = configuredCluster.addHelmRelease('argocd', 'argo-cd', { + repo: 'https://argoproj.github.io/argo-helm', + version: '7.8.0', + namespace: 'argocd', + }) + .withHelmValue('server.insecure', 'true') + .withHelmValuesFile('./deploy/argocd-values.yaml'); + + const argocdResource = await argocd; + const _argocdParent = await argocdResource.parent.get(); + const _argocdReleaseName = await argocdResource.releaseName.get(); + const _argocdNamespace = await argocdResource.namespace.get(); + + // ── K8s manifest / Kustomize overlay ───────────────────────────────────── + const widgetCrd = configuredCluster.addK8sManifest('widget-crd', './k8s/crds/'); + + const widgetCrdResource = await widgetCrd; + const _crdParent = await widgetCrdResource.parent.get(); + const _crdPath = await widgetCrdResource.path.get(); + + // ── Service endpoint ───────────────────────────────────────────────────── + const ui = configuredCluster.addServiceEndpoint('argocd-ui', 'argocd-server', 443, { + namespace: 'argocd', + }); + + const uiResource = await ui; + const _uiParent = await uiResource.parent.get(); + const _uiServiceName = await uiResource.serviceName.get(); + const _uiServicePort = await uiResource.servicePort.get(); + const _uiNamespace = await uiResource.namespace.get(); + const _uiHostPort = await uiResource.hostPort.get(); + + // ── WithReference — cluster kubeconfig injection ────────────────────────── + // Project: receives KUBECONFIG=.../.k3s/k8s/local/kubeconfig.yaml + const _projectRef = builder + .addProject('operator', { projectPath: '../WidgetOperator/WidgetOperator.csproj' }) + .withReference(configuredCluster); + + // Container: receives KUBECONFIG_DATA= + const _containerRef = builder + .addContainer('sidecar', 'myorg/sidecar') + .withReference(configuredCluster); + + // ── WithReference — service endpoint URL injection ──────────────────────── + // Host: receives services__argocd-ui__url=https://localhost:{port} + const _endpointRef = builder + .addProject('api', { projectPath: '../WidgetApi/WidgetApi.csproj' }) + .withReference(ui); +} + +await builder.build().run(); diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json new file mode 100644 index 000000000..80fbafe65 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json @@ -0,0 +1,26 @@ +{ + "appHost": { + "path": "apphost.ts", + "language": "typescript/nodejs" + }, + "profiles": { + "https": { + "applicationUrl": "https://localhost:17149;http://localhost:15243", + "environmentVariables": { + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21065", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22170" + } + }, + "http": { + "applicationUrl": "http://localhost:15243", + "environmentVariables": { + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19027", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20141", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + } + }, + "packages": { + "CommunityToolkit.Aspire.Hosting.K3s": "" + } +} diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json new file mode 100644 index 000000000..54f1c98e3 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json @@ -0,0 +1,19 @@ +{ + "name": "validationapphost", + "version": "1.0.0", + "type": "module", + "scripts": { + "start": "aspire run", + "build": "tsc", + "dev": "tsc --watch" + }, + "dependencies": { + "vscode-jsonrpc": "^8.2.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "nodemon": "^3.1.11", + "tsx": "^4.19.0", + "typescript": "^5.3.0" + } +} diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json new file mode 100644 index 000000000..05c7cf2f8 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "outDir": "./dist", + "rootDir": "." + }, + "include": [ + "apphost.ts", + ".modules/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs deleted file mode 100644 index b1d44bf9c..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace Aspire.Hosting.ApplicationModel; - -/// -/// Annotation that describes a Helm chart release to install into the cluster. -/// -/// The Helm release name. -/// The chart name or local path. -/// The Kubernetes namespace to install into. -/// Optional Helm repository URL containing the chart. -/// Optional chart version. -public sealed class HelmReleaseAnnotation( - string releaseName, - string chart, - string @namespace, - string? repoUrl, - string? version) : IResourceAnnotation -{ - /// Gets the Helm release name. - public string ReleaseName { get; } = releaseName; - - /// Gets the chart name or local path. - public string Chart { get; } = chart; - - /// Gets the target Kubernetes namespace. - public string Namespace { get; } = @namespace; - - /// Gets the optional Helm repository URL. - public string? RepoUrl { get; } = repoUrl; - - /// Gets the optional chart version. - public string? Version { get; } = version; - - /// Gets the extra --set values passed to helm install. - public IDictionary Values { get; } = new Dictionary(); -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs deleted file mode 100644 index 635eebd5e..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Aspire.Hosting.ApplicationModel; - -/// -/// Annotation that signals the Kubernetes Dashboard should be installed alongside the cluster. -/// -/// The Kubernetes Dashboard chart version to install. -public sealed class KubernetesDashboardAnnotation(string? version = null) : IResourceAnnotation -{ - /// Gets the dashboard chart version, or to use the latest. - public string? Version { get; } = version; -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs deleted file mode 100644 index 4525fa5a1..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Aspire.Hosting.ApplicationModel; - -/// -/// Annotation that describes a Kustomize overlay to apply into the cluster. -/// -/// Path to the kustomization directory or remote URL. -public sealed class KustomizeAnnotation(string path) : IResourceAnnotation -{ - /// Gets the kustomization path. - public string Path { get; } = path; -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj index 1800d1555..67a2f57a5 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -2,7 +2,7 @@ kubernetes k3s hosting cluster - An Aspire hosting integration for k3s — a lightweight Kubernetes distribution. + An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via alpine/k8s), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs index 65740bea8..e288b9ab4 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -1,3 +1,5 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// @@ -13,6 +15,7 @@ namespace Aspire.Hosting.ApplicationModel; /// The Helm release name passed to helm upgrade --install. /// The Kubernetes namespace to install into. /// The parent k3s cluster resource. +[AspireExport(ExposeProperties = true)] public sealed class HelmReleaseResource( string name, string releaseName, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index 6799c8f8e..9782bfd71 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -88,9 +88,12 @@ public static IResourceBuilder AddHelmRelease( // The callback wraps all files so it fires after all WithHelmValuesFile() calls. .WithContainerFiles("/helm-values", async (ctx, ct) => release.ValuesFiles - .Select(hostPath => (ContainerFileSystemItem)new ContainerFile + .Select((hostPath, i) => (ContainerFileSystemItem)new ContainerFile { - Name = System.IO.Path.GetFileName(hostPath), + // Prefix with index so files are unique even if basenames collide + // (e.g. prod/values.yaml + base/values.yaml → 0-values.yaml, 1-values.yaml) + // and order is explicit on the filesystem as well as in the script. + Name = $"{i}-{System.IO.Path.GetFileName(hostPath)}", SourcePath = hostPath, }) .ToList()) @@ -122,6 +125,7 @@ public static IResourceBuilder AddHelmRelease( /// Path to the values YAML file on the host. Relative paths are resolved against /// AppHostDirectory. /// + [AspireExport("withHelmValuesFile", Description = "Injects a host-side YAML values file into the Helm installer container")] public static IResourceBuilder WithHelmValuesFile( this IResourceBuilder builder, string path) @@ -142,6 +146,7 @@ public static IResourceBuilder WithHelmValuesFile( /// /// Adds a Helm --set key=value argument to this release. /// + [AspireExport("withHelmValue", Description = "Adds a --set key=value argument to the Helm release")] public static IResourceBuilder WithHelmValue( this IResourceBuilder builder, string key, @@ -188,10 +193,12 @@ internal static string BuildHelmScript(HelmReleaseResource release) if (release.Version is not null) sb.Append($" --version \"{release.Version}\""); - // Values files are injected into /helm-values/ by WithContainerFiles. - foreach (var hostPath in release.ValuesFiles) - sb.Append($" --values \"/helm-values/{System.IO.Path.GetFileName(hostPath)}\""); + // Values files: injected as {index}-{filename} to guarantee uniqueness and order. + // Applied first so --set flags below can override individual keys. + for (var i = 0; i < release.ValuesFiles.Count; i++) + sb.Append($" --values \"/helm-values/{i}-{System.IO.Path.GetFileName(release.ValuesFiles[i])}\""); + // --set flags override everything above (highest Helm precedence). foreach (var (key, value) in release.HelmValues) sb.Append($" --set \"{key}={value}\""); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index 6e87500dc..b630b613b 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -300,6 +300,7 @@ public static IResourceBuilder WithExtraArg( /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. /// + [AspireExport("withDataVolume", Description = "Adds a named volume for the k3s cluster data directory so state survives AppHost restarts")] public static IResourceBuilder WithDataVolume( this IResourceBuilder builder, string? name = null) @@ -371,6 +372,7 @@ public static IResourceBuilder WithReference( /// /// Sets the container lifetime for the k3s cluster and all its agent nodes. /// + [AspireExport("withLifetime", Description = "Sets the container lifetime for the k3s cluster and all its agent nodes")] public static IResourceBuilder WithLifetime( this IResourceBuilder builder, ContainerLifetime lifetime) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index 4e121d7f9..48baf3907 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -1,9 +1,12 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// /// Represents a k3s Kubernetes cluster running as a privileged container resource. /// /// The resource name. +[AspireExport(ExposeProperties = true)] public sealed class K3sClusterResource(string name) : ContainerResource(name) { internal const string ApiServerEndpointName = "api"; @@ -52,6 +55,7 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) new(StringComparer.OrdinalIgnoreCase); /// A dictionary of registered Helm releases keyed by resource name. + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] public IReadOnlyDictionary HelmReleases => _helmReleases; internal void AddHelmRelease(string resourceName, string releaseName) => @@ -60,6 +64,7 @@ internal void AddHelmRelease(string resourceName, string releaseName) => private readonly List _manifests = []; /// Names of registered children. + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] public IReadOnlyList Manifests => _manifests; internal void AddManifest(string resourceName) => _manifests.Add(resourceName); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs index 95d4c9f9a..0af117b1e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -1,3 +1,5 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// @@ -12,6 +14,7 @@ namespace Aspire.Hosting.ApplicationModel; /// Container consumers receive services__{name}__url=https://host.docker.internal:{port}. /// /// +[AspireExport(ExposeProperties = true)] public sealed class K3sServiceEndpointResource( string name, string serviceName, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index a87a687ee..2610f033c 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -1,3 +1,5 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// @@ -12,6 +14,7 @@ namespace Aspire.Hosting.ApplicationModel; /// The Aspire resource name. /// Absolute path to a single YAML file or a directory. /// The parent k3s cluster resource. +[AspireExport(ExposeProperties = true)] public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) : ContainerResource(name), IResourceWithParent { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index f6a5d8150..cc830bcbe 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -1,67 +1,197 @@ # CommunityToolkit.Aspire.Hosting.K3s -An Aspire hosting integration for [k3s](https://k3s.io/) — a lightweight, certified Kubernetes distribution by Rancher/SUSE. +Provides extension methods and resource definitions for the .NET Aspire AppHost to run a +[k3s](https://k3s.io/) lightweight Kubernetes cluster as part of the local development +inner loop. The cluster, Helm chart installs, manifest applies, and service endpoint +exposures all appear as first-class resources in the Aspire dashboard — no external +tooling beyond Docker is required. -## Getting started +## Getting Started ### Prerequisites -- Docker with support for `--privileged` containers (Linux host or Docker Desktop on macOS/Windows) +- A container runtime that supports privileged Linux containers: + - **Docker Engine 20.10+** (Linux) or **Docker Desktop** (macOS / Windows) + - **Podman 4.0+** (Linux) — works with rootful Podman; rootless requires cgroup v2 delegation -### Installation +> **Note:** k3s requires `--privileged`, `--cgroupns=host`, and a writable cgroup +> filesystem mount. These flags are passed automatically by the integration. Whether your +> runtime honours them depends on your system configuration. + +### Install the package + +In your AppHost project: ```sh dotnet add package CommunityToolkit.Aspire.Hosting.K3s ``` -## Usage +### Quick start ```csharp -var builder = DistributedApplication.CreateBuilder(args); +var cluster = builder.AddK3sCluster("k8s"); + +// Inject kubeconfig into a project — KUBECONFIG env var points to local/kubeconfig.yaml +builder.AddProject("operator") + .WaitFor(cluster) + .WithReference(cluster); +``` + +## Deploying Helm charts + +`AddHelmRelease` runs `helm upgrade --install` inside an `alpine/helm` container on the +DCP network. No host-side `helm` binary is required. The container exits with code 0 on +success, so use `WaitForCompletion` on any resource that depends on the chart being installed. -var cluster = builder.AddK3sCluster("k8s") - .WithPersistentState(); +```csharp +var argocd = cluster.AddHelmRelease("argocd", "argo-cd", + repo: "https://argoproj.github.io/argo-helm", + version: "7.8.0", + @namespace: "argocd") + .WithHelmValue("server.insecure", "true") + .WithHelmValuesFile("./deploy/argocd-values.yaml"); builder.AddProject("api") - .WithReference(cluster) - .WaitFor(cluster); + .WaitForCompletion(argocd) // wait for the chart install to finish + .WithReference(cluster); +``` + +### Helm override precedence + +Values are applied in this order (last wins): + +1. `WithHelmValuesFile` calls — in the order they are declared (`0-`, `1-`, … prefix) +2. `WithHelmValue` `--set` flags — always override values files + +## Applying Kubernetes manifests -builder.Build().Run(); +`AddK8sManifest` runs `kubectl apply --server-side` inside an `alpine/k8s` container. +No host-side `kubectl` binary is required. Kustomize overlays are auto-detected. + +```csharp +// Plain YAML file or directory +var crd = cluster.AddK8sManifest("widget-crd", "./k8s/crds/"); + +// Kustomize overlay — detected automatically when kustomization.yaml is present +var overlay = cluster.AddK8sManifest("prod-overlay", "./k8s/overlays/local"); + +// Gate dependent resources until the CRD is Established +builder.AddProject("operator") + .WaitForCompletion(crd) + .WithReference(cluster); +``` + +For Kustomize overlays that reference base directories outside the overlay path, point +`AddK8sManifest` to the common root and specify the overlay path in `kustomization.yaml`. + +## Exposing k8s services to the Aspire network + +`AddServiceEndpoint` starts an in-process KubernetesClient WebSocket port-forward bound +to `0.0.0.0:{allocatedPort}`. No NodePort configuration is required. + +```csharp +var ui = cluster.AddServiceEndpoint("argocd-ui", + serviceName: "argocd-server", + servicePort: 443, + @namespace: "argocd") + .WaitForCompletion(argocd); // wait for chart install before port-forwarding + +// Host processes receive services__argocd-ui__url=https://localhost:{port} +builder.AddProject("consumer") + .WaitFor(ui) + .WithReference(ui); + +// DCP containers receive https://host.docker.internal:{port} +// --add-host=host.docker.internal:host-gateway is injected automatically on Linux +builder.AddContainer("sidecar", "myorg/sidecar") + .WaitFor(ui) + .WithReference(ui); ``` -### Kubeconfig injection +## Kubeconfig injection -`WithReference(cluster)` automatically selects the injection mode: +`WithReference(cluster)` selects the injection mode automatically: -| Resource type | Environment variable set | +| Consumer type | What is injected | |---|---| -| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=/tmp/aspire-k3s-k8s/admin.yaml` | -| `ContainerResource` | `KUBECONFIG_DATA=` | +| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=…/.k3s/k8s/local/kubeconfig.yaml` | +| `ContainerResource` | `KUBECONFIG_DATA=` | -### Configuration options +Reading in .NET: +```csharp +// Project / executable — standard kubectl convention +var config = KubernetesClientConfiguration.BuildConfigFromConfigFile( + Environment.GetEnvironmentVariable("KUBECONFIG")); + +// Container — decode from env var +var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); +using var stream = new MemoryStream(bytes); +var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); +``` + +## Persistent cluster state + +```csharp +builder.AddK3sCluster("k8s") + .WithDataVolume() // persists /var/lib/rancher/k3s across AppHost restarts + .WithLifetime(ContainerLifetime.Persistent); +``` + +Without `WithDataVolume` the cluster state is ephemeral — each AppHost start produces a +fresh cluster. With it, subsequent starts reuse the existing cluster and skip +reinitialisation, making startup significantly faster. + +## Multi-node clusters ```csharp builder.AddK3sCluster("k8s", configure: opts => { - opts.ClusterCidr = "10.42.0.0/16"; - opts.ServiceCidr = "10.43.0.0/16"; - opts.DisabledComponents.Add("traefik"); + opts.AgentCount = 2; // 1 server + 2 agents }); ``` -Or use the fluent API: +The health check waits for all nodes to reach `Ready` before the cluster is marked healthy. + +## Image overrides + +The `alpine/helm` and `alpine/k8s` images are pinned but configurable: ```csharp -builder.AddK3sCluster("k8s") - .WithK3sVersion("v1.32.3-k3s1") - .WithPodSubnet("10.42.0.0/16") - .WithServiceSubnet("10.43.0.0/16") - .WithDisabledComponent("traefik") - .WithPersistentState(); +builder.AddK3sCluster("k8s", configure: opts => +{ + opts.HelmImage = "my-registry/helm"; + opts.HelmTag = "3.18.0"; + opts.KubectlImage = "my-registry/k8s"; + opts.KubectlTag = "1.33.0"; +}); ``` -## Known limitations +## Reaching Aspire services from k3s pods + +k3s pods run on the internal pod network (`10.42.0.0/16`). k3s's Flannel CNI masquerades +pod traffic through the k3s container's DCP network IP, so pods can reach DCP services by +their host-mapped port — but not by their Aspire DNS name. Use Helm values or a ConfigMap +to inject the host-accessible address: + +```csharp +var postgres = builder.AddPostgres("db"); + +// Resolve the host-mapped port at configuration time +var dbPort = postgres.GetEndpoint("tcp"); + +cluster.AddHelmRelease("my-operator", "my-operator-chart") + .WithHelmValue("database.host", "host.docker.internal") + .WithHelmValue("database.port", dbPort.Property(EndpointProperty.Port)); +``` + +Inside the pod, `host.docker.internal` resolves to the Docker host because k3s runs as a +privileged container on the DCP network and Flannel masquerades outbound pod traffic +through it. + +## Additional information + +https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-k3s + +## Feedback & contributing -- Requires a privileged Docker runtime; `--privileged` is passed automatically. -- On Linux hosts the `/lib/modules` directory should be present for CNI networking. -- The first cluster start can take 30–60 seconds while container images and CNI plugins are initialised. +https://github.com/CommunityToolkit/Aspire diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs index 50701b0d7..770655906 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -8,47 +8,125 @@ //------------------------------------------------------------------------------ namespace Aspire.Hosting { + public static partial class K3sBuilderExtensions + { + [AspireExport("addK3sCluster", Description = "Adds a k3s Kubernetes cluster resource")] + public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } + + [AspireExport("withDataVolume", Description = "Adds a named volume for the k3s cluster data directory so state survives AppHost restarts")] + public static ApplicationModel.IResourceBuilder WithDataVolume(this ApplicationModel.IResourceBuilder builder, string? name = null) { throw null; } + + [AspireExport("withDisabledComponent", Description = "Disables a built-in k3s component")] + public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } + + [AspireExport("withExtraArg", Description = "Appends a raw argument to the k3s server command")] + public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } + + [AspireExport("withK3sVersion", Description = "Overrides the k3s server image version")] + public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } + + [AspireExport("withLifetime", Description = "Sets the container lifetime for the k3s cluster and all its agent nodes")] + public static ApplicationModel.IResourceBuilder WithLifetime(this ApplicationModel.IResourceBuilder builder, ApplicationModel.ContainerLifetime lifetime) { throw null; } + + [AspireExport("withPodSubnet", Description = "Sets the pod subnet CIDR for the k3s cluster")] + public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + + [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] + public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source) + where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } + + [AspireExport("withServiceSubnet", Description = "Sets the service subnet CIDR for the k3s cluster")] + public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + } + public static partial class K3sHelmBuilderExtensions { + [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] public static ApplicationModel.IResourceBuilder AddHelmRelease(this ApplicationModel.IResourceBuilder builder, string name, string chart, string? repo = null, string? version = null, string @namespace = "default") { throw null; } + + [AspireExport("withHelmValue", Description = "Adds a --set key=value argument to the Helm release")] public static ApplicationModel.IResourceBuilder WithHelmValue(this ApplicationModel.IResourceBuilder builder, string key, string value) { throw null; } - public static ApplicationModel.IResourceBuilder WithEndpoint(this ApplicationModel.IResourceBuilder builder, string serviceName, int servicePort, string name) { throw null; } + + [AspireExport("withHelmValuesFile", Description = "Injects a host-side YAML values file into the Helm installer container")] + public static ApplicationModel.IResourceBuilder WithHelmValuesFile(this ApplicationModel.IResourceBuilder builder, string path) { throw null; } } - public static partial class K3sBuilderExtensions + public static partial class K3sManifestBuilderExtensions { - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source, CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy strategy = CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy.Auto) where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } - public static ApplicationModel.IResourceBuilder WithPersistentState(this ApplicationModel.IResourceBuilder builder, string? volumeName = null) { throw null; } + [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] + public static ApplicationModel.IResourceBuilder AddK8sManifest(this ApplicationModel.IResourceBuilder builder, string name, string path) { throw null; } + } + + public static partial class K3sServiceEndpointExtensions + { + [AspireExport("addServiceEndpoint", Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] + public static ApplicationModel.IResourceBuilder AddServiceEndpoint(this ApplicationModel.IResourceBuilder builder, string name, string serviceName, int servicePort, string @namespace = "default") { throw null; } + + [AspireExport("withReference", Description = "Injects the k3s service URL into a dependent resource")] + public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source) + where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } } } namespace Aspire.Hosting.ApplicationModel { - public sealed partial class HelmReleaseResource : Resource, IResourceWithParent, IResourceWithWaitSupport + [AspireExport(ExposeProperties = true)] + public sealed partial class HelmReleaseResource : ContainerResource, IResourceWithParent, IResourceWithParent, IResource { - public HelmReleaseResource(string name, string releaseName, string @namespace, K3sClusterResource cluster) : base(default!) { } + public HelmReleaseResource(string name, string releaseName, string @namespace, K3sClusterResource cluster) : base(default!, default) { } + + public string Namespace { get { throw null; } } + public K3sClusterResource Parent { get { throw null; } } + public string ReleaseName { get { throw null; } } - public string Namespace { get { throw null; } } } + public sealed partial class K3sAgentResource : ContainerResource, IResourceWithParent, IResourceWithParent, IResource + { + public K3sAgentResource(string name, K3sClusterResource cluster) : base(default!, default) { } + + public K3sClusterResource Parent { get { throw null; } } + } + + [AspireExport(ExposeProperties = true)] public sealed partial class K3sClusterResource : ContainerResource { - public K3sClusterResource(string name) : base(default!) { } + public K3sClusterResource(string name) : base(default!, default) { } + public EndpointReference ApiEndpoint { get { throw null; } } + + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] + public System.Collections.Generic.IReadOnlyDictionary HelmReleases { get { throw null; } } + + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] + public System.Collections.Generic.IReadOnlyList Manifests { get { throw null; } } + } + + [AspireExport(ExposeProperties = true)] + public sealed partial class K3sServiceEndpointResource : Resource, IResourceWithParent, IResourceWithParent, IResource, IResourceWithWaitSupport + { + public K3sServiceEndpointResource(string name, string serviceName, int servicePort, string @namespace, K3sClusterResource cluster) : base(default!) { } + + public int HostPort { get { throw null; } } + + public string Namespace { get { throw null; } } + + public K3sClusterResource Parent { get { throw null; } } + + public string ServiceName { get { throw null; } } + + public int ServicePort { get { throw null; } } + } + + [AspireExport(ExposeProperties = true)] + public sealed partial class K8sManifestResource : ContainerResource, IResourceWithParent, IResourceWithParent, IResource + { + public K8sManifestResource(string name, string path, K3sClusterResource cluster) : base(default!, default) { } + + public K3sClusterResource Parent { get { throw null; } } + + public string Path { get { throw null; } } } } @@ -56,18 +134,28 @@ namespace CommunityToolkit.Aspire.Hosting { public sealed partial class K3sClusterOptions { - public K3sClusterOptions() { } + public int AgentCount { get { throw null; } set { } } + public string? ClusterCidr { get { throw null; } set { } } - public string? ImageTag { get { throw null; } set { } } - public string? ServiceCidr { get { throw null; } set { } } + public System.Collections.Generic.IList DisabledComponents { get { throw null; } } + public System.Collections.Generic.IList ExtraArgs { get { throw null; } } - } - public enum KubeconfigInjectionStrategy - { - Auto = 0, - HostNetwork = 1, - ContainerNetwork = 2, + public string HelmImage { get { throw null; } set { } } + + public string HelmRegistry { get { throw null; } set { } } + + public string HelmTag { get { throw null; } set { } } + + public string? ImageTag { get { throw null; } set { } } + + public string KubectlImage { get { throw null; } set { } } + + public string KubectlRegistry { get { throw null; } set { } } + + public string KubectlTag { get { throw null; } set { } } + + public string? ServiceCidr { get { throw null; } set { } } } -} +} \ No newline at end of file diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs index f08fa0588..226335163 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -3,6 +3,8 @@ using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Utils; using CommunityToolkit.Aspire.Testing; +using k8s; +using k8s.Models; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -13,12 +15,14 @@ namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; /// /// Requirements: /// -/// Linux with Docker (privileged containers required by k3s). -/// helm on PATH — used by AddHelmRelease. -/// kubectl on PATH — used by AddK8sManifest. +/// A container runtime that supports privileged Linux containers. +/// Linux: Docker Engine 20.10+ or rootful Podman 4.0+. +/// macOS / Windows: Docker Desktop (containers run inside WSL2 / Hyper-V VM). /// -/// The tests are gated by [RequiresDocker] and intended for the -/// ubuntu-latest-only CI job in tests.yaml. +/// No host-side helm or kubectl is needed — both run as Docker containers +/// (alpine/helm and alpine/k8s). +/// Tests are gated by [RequiresDocker] and run on both ubuntu-latest +/// and windows-latest CI jobs since privileged Linux containers work on Docker Desktop. /// /// [RequiresDocker] @@ -72,7 +76,7 @@ public async Task ClusterReachesRunningAndKubeconfigIsValid() } [Fact] - public async Task HelmReleaseReachesRunning() + public async Task HelmReleaseExitsSuccessfully() { var cluster = _builder!.AddK3sCluster("k8s"); @@ -90,8 +94,11 @@ public async Task HelmReleaseReachesRunning() using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + // HelmReleaseResource is a run-to-completion container — it exits with code 0 + // when helm upgrade --install --wait completes successfully. await rns.WaitForResourceAsync("nginx", - s => s.Snapshot.State?.Text == KnownResourceStates.Running, cts.Token); + s => s.Snapshot.State?.Text == "Exited", cts.Token); } [Fact] @@ -116,12 +123,10 @@ public async Task ServiceEndpointExposesHttpPort() using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); await rns.WaitForResourceHealthyAsync("k8s", cts.Token); - // nginx is a run-to-completion container — wait for it to exit (Exited state) await rns.WaitForResourceAsync("nginx", s => s.Snapshot.State?.Text == "Exited", cts.Token); await rns.WaitForResourceHealthyAsync("nginx-web", cts.Token); - // Find the allocated port from the endpoint resource. var model = _app.Services.GetRequiredService(); var ep = model.Resources.OfType().Single(); @@ -138,8 +143,6 @@ public async Task WithReferenceInjectsKubeconfigForProject() { var cluster = _builder!.AddK3sCluster("k8s"); - // WithReference on a project injects KUBECONFIG pointing to local/kubeconfig.yaml. - // We verify the env var would be set by checking the cluster state. _app = _builder.Build(); await _app.StartAsync(); @@ -155,4 +158,139 @@ public async Task WithReferenceInjectsKubeconfigForProject() Assert.Contains("localhost", yaml); Assert.DoesNotContain("127.0.0.1:6443", yaml); } + + [Fact] + public async Task ManifestAppliesCrdAndReachesEstablished() + { + // Write a minimal CRD manifest to a temp file so AddK8sManifest can find it. + var crdYaml = """ + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: widgets.example.com + spec: + group: example.com + scope: Namespaced + names: + plural: widgets + singular: widget + kind: Widget + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + color: + type: string + """; + + var manifestDir = Path.Combine(_builder!.AppHostDirectory, "k8s-test-crds"); + Directory.CreateDirectory(manifestDir); + var crdPath = Path.Combine(manifestDir, "widget-crd.yaml"); + await File.WriteAllTextAsync(crdPath, crdYaml); + + try + { + var cluster = _builder.AddK3sCluster("k8s"); + var crd = cluster.AddK8sManifest("widget-crd", crdPath); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + // K8sManifestResource is run-to-completion — exits 0 after CRD reaches Established. + await rns.WaitForResourceAsync("widget-crd", + s => s.Snapshot.State?.Text == "Exited", cts.Token); + + // Independently verify the CRD is Established via KubernetesClient. + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); + using var k8sClient = new Kubernetes(config); + + var widgetCrd = await k8sClient.ApiextensionsV1 + .ReadCustomResourceDefinitionAsync("widgets.example.com", cancellationToken: cts.Token); + + var established = widgetCrd.Status?.Conditions?.Any(c => + c.Type == "Established" && + string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true; + + Assert.True(established, "CRD 'widgets.example.com' should be Established"); + } + finally + { + Directory.Delete(manifestDir, recursive: true); + } + } + + [Fact] + public async Task WithDataVolumePreservesStateAcrossRestarts() + { + // Use an explicit volume name shared between both app instances. + var volumeName = $"aspire-k3s-persist-{Guid.NewGuid():N}"; + + // ── First run ───────────────────────────────────────────────────── + _builder!.AddK3sCluster("k8s").WithDataVolume(volumeName); + _app = _builder.Build(); + await _app.StartAsync(); + + var rns1 = _app.Services.GetRequiredService(); + using var cts1 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await rns1.WaitForResourceHealthyAsync("k8s", cts1.Token); + + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + using (var k8sClient = new Kubernetes( + KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath))) + { + await k8sClient.CoreV1.CreateNamespacedConfigMapAsync( + new V1ConfigMap + { + Metadata = new V1ObjectMeta { Name = "persist-check" }, + Data = new Dictionary { ["run"] = "first" }, + }, + "default", + cancellationToken: cts1.Token); + } + + // Stop the first app. The volume is retained; the container is removed by DCP. + await _app.StopAsync(); + await _app.DisposeAsync(); + _app = null; + + // ── Second run with the same named volume ───────────────────────── + var builder2 = TestDistributedApplicationBuilder.Create(); + builder2.AddK3sCluster("k8s").WithDataVolume(volumeName); + + await using var app2 = builder2.Build(); + await app2.StartAsync(); + + var rns2 = app2.Services.GetRequiredService(); + using var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await rns2.WaitForResourceHealthyAsync("k8s", cts2.Token); + + var kubeconfigPath2 = Path.Combine( + builder2.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + using var k8sClient2 = new Kubernetes( + KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath2)); + + var cm = await k8sClient2.CoreV1.ReadNamespacedConfigMapAsync( + "persist-check", "default", cancellationToken: cts2.Token); + + Assert.Equal("first", cm.Data["run"]); + + await app2.StopAsync(); + } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 9d940fe58..7677ad7aa 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -353,7 +353,7 @@ public void AddServiceEndpointShouldThrowWhenBuilderIsNull() // ── WithHelmValuesFile tests ────────────────────────────────────────────── [Fact] - public void BuildHelmScriptIncludesValuesFiles() + public void BuildHelmScriptIncludesValuesFilesWithIndexPrefix() { var cluster = new K3sClusterResource("k8s"); var release = new HelmReleaseResource("argocd", "argocd", "argocd", cluster) @@ -365,13 +365,47 @@ public void BuildHelmScriptIncludesValuesFiles() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); - Assert.Contains("--values \"/helm-values/values.yaml\"", script); - Assert.Contains("--values \"/helm-values/values-prod.yaml\"", script); - // Values files are applied before --set overrides (last wins). - var valuesIndex = script.IndexOf("--values", StringComparison.Ordinal); - var setIndex = script.IndexOf("--set", StringComparison.Ordinal); - Assert.True(valuesIndex < setIndex || setIndex == -1, - "--values flags should appear before --set flags"); + // Index prefix guarantees uniqueness even when basenames collide. + Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); + Assert.Contains("--values \"/helm-values/1-values-prod.yaml\"", script); + } + + [Fact] + public void BuildHelmScriptValuesFilesOrderedBeforeSetFlags() + { + // Helm precedence: --values (ascending index) → --set (highest, always wins). + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("argocd", "argocd", "argocd", cluster) + { + Chart = "argo-cd", + }; + release.ValuesFiles.Add("/tmp/base.yaml"); + release.ValuesFiles.Add("/tmp/prod.yaml"); + release.HelmValues["key"] = "override"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + var firstValuesIdx = script.IndexOf("--values \"/helm-values/0-", StringComparison.Ordinal); + var secondValuesIdx = script.IndexOf("--values \"/helm-values/1-", StringComparison.Ordinal); + var setIdx = script.IndexOf("--set", StringComparison.Ordinal); + + Assert.True(firstValuesIdx < secondValuesIdx, "0-base.yaml must precede 1-prod.yaml"); + Assert.True(secondValuesIdx < setIdx, "--values flags must precede --set flags"); + } + + [Fact] + public void BuildHelmScriptCollisionSafeWithSameBasename() + { + // Two files from different directories with the same name must not collide. + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.ValuesFiles.Add("/prod/values.yaml"); + release.ValuesFiles.Add("/dev/values.yaml"); + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); + Assert.Contains("--values \"/helm-values/1-values.yaml\"", script); } [Fact] From 0a4d13a3a07f66516205be4551e8bf31930d6533 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 14:59:29 +0200 Subject: [PATCH 05/29] upgrade example aspire version Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index 65e9770a0..387601982 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe From f9a50d13942c7a37c46ebb75a7d458dcc9c6c45f Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 16:47:11 +0200 Subject: [PATCH 06/29] fix: copilot reviews --- .../K3sBuilderExtensions.Helm.cs | 30 ++++++-- .../K3sBuilderExtensions.Manifest.cs | 32 +++++++- .../K3sBuilderExtensions.ServiceEndpoint.cs | 48 ++++++------ .../K3sBuilderExtensions.cs | 71 +++++++++++------ .../K3sContainerImageTags.cs | 4 +- .../K3sInProcessPortForwarder.cs | 77 ++++++++++++++++++- .../HelmReleaseResourceTests.cs | 26 +++---- .../K3sClusterResourceTests.cs | 34 ++++++-- 8 files changed, 239 insertions(+), 83 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index 9782bfd71..c5ade685f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -178,32 +178,48 @@ internal static string BuildHelmScript(HelmReleaseResource release) if (release.RepoUrl is not null) { var alias = $"aspire-k3s-{release.ReleaseName}"; - sb.AppendLine($"helm repo add --force-update \"{alias}\" \"{release.RepoUrl}\""); - sb.AppendLine($"helm repo update \"{alias}\""); + sb.AppendLine($"helm repo add --force-update {ShellEscape(alias)} {ShellEscape(release.RepoUrl)}"); + sb.AppendLine($"helm repo update {ShellEscape(alias)}"); } var chartRef = release.RepoUrl is not null ? $"aspire-k3s-{release.ReleaseName}/{release.Chart}" : release.Chart!; - sb.Append($"helm upgrade --install \"{release.ReleaseName}\" \"{chartRef}\""); - sb.Append($" --namespace \"{release.Namespace}\" --create-namespace"); + sb.Append($"helm upgrade --install {ShellEscape(release.ReleaseName)} {ShellEscape(chartRef)}"); + sb.Append($" --namespace {ShellEscape(release.Namespace)} --create-namespace"); sb.Append(" --wait --timeout 10m"); if (release.Version is not null) - sb.Append($" --version \"{release.Version}\""); + sb.Append($" --version {ShellEscape(release.Version)}"); // Values files: injected as {index}-{filename} to guarantee uniqueness and order. // Applied first so --set flags below can override individual keys. + // Paths are single-quoted so spaces or special characters in filenames are safe. for (var i = 0; i < release.ValuesFiles.Count; i++) - sb.Append($" --values \"/helm-values/{i}-{System.IO.Path.GetFileName(release.ValuesFiles[i])}\""); + { + var filename = $"{i}-{System.IO.Path.GetFileName(release.ValuesFiles[i])}"; + sb.Append($" --values {ShellEscape($"/helm-values/{filename}")}"); + } // --set flags override everything above (highest Helm precedence). + // Both key and value are single-quoted so quotes, spaces, or $ in values are safe. foreach (var (key, value) in release.HelmValues) - sb.Append($" --set \"{key}={value}\""); + sb.Append($" --set {ShellEscape($"{key}={value}")}"); return sb.ToString(); } + + /// + /// Wraps in POSIX single quotes so that all shell + /// metacharacters ($, `, \, ", ;, + /// &, |, space, etc.) are treated as literals in Dockerfile + /// RUN shell-form commands and Delve's --build-flags parser. + /// Embedded single quotes are escaped with the standard POSIX technique: + /// ' → '\''. + /// + private static string ShellEscape(string value) => + $"'{value.Replace("'", "'\\''")}'"; } #pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 6cb79b838..33c7ac128 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -86,10 +86,34 @@ [new ContainerFile } else { - // Single file or regular directory — use Aspire's built-in - // WithContainerFiles(destinationPath, hostSourcePath) which copies the - // file or all files in the directory into the container without a bind-mount. - resourceBuilder.WithContainerFiles("/k8s-manifests", absolutePath); + // Single file or regular directory — copy via async callback so the file(s) + // need not exist when the AppHost is built (only when the container starts). + // This mirrors WithContainerFiles(path, hostPath) semantics but without the + // build-time path validation that Aspire's string overload performs. + resourceBuilder.WithContainerFiles("/k8s-manifests", async (ctx, ct) => + { + if (Directory.Exists(absolutePath)) + { + var files = Directory + .GetFiles(absolutePath, "*.yaml", SearchOption.TopDirectoryOnly) + .Concat(Directory.GetFiles(absolutePath, "*.yml", SearchOption.TopDirectoryOnly)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase); + + return [.. files + .Select(f => (ContainerFileSystemItem)new ContainerFile + { + Name = System.IO.Path.GetFileName(f), + SourcePath = f, + })]; + } + + return [(ContainerFileSystemItem)new ContainerFile + { + Name = System.IO.Path.GetFileName(absolutePath), + SourcePath = absolutePath, + }]; + }); } return resourceBuilder diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs index dfbe5ecb8..0a2a12eb1 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -3,6 +3,7 @@ using System.Net.Sockets; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; +using k8s; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -20,20 +21,12 @@ public static class K3sServiceEndpointExtensions /// Exposes a Kubernetes service as a first-class Aspire endpoint resource. /// /// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready, - /// binding to 0.0.0.0:{hostPort}. Use WaitFor to sequence after a - /// or that deploys the service. + /// binding to 0.0.0.0:{hostPort}. The endpoint only becomes healthy after the + /// Kubernetes service has a ready pod — use WaitForCompletion on a + /// or to sequence the + /// install before starting the port-forward. /// /// - /// - /// - /// var nginx = cluster.AddHelmRelease("nginx", "nginx", repo: "https://charts.bitnami.com/bitnami"); - /// var ui = cluster.AddServiceEndpoint("nginx-ui", "nginx", servicePort: 80) - /// .WaitFor(nginx); - /// builder.AddProject<Projects.Api>("api") - /// .WaitFor(ui) - /// .WithReference(ui); - /// - /// [AspireExport("addServiceEndpoint", Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] public static IResourceBuilder AddServiceEndpoint( @@ -47,6 +40,10 @@ public static IResourceBuilder AddServiceEndpoint( ArgumentNullException.ThrowIfNull(name); ArgumentNullException.ThrowIfNull(serviceName); + if (servicePort is < 1 or > 65535) + throw new ArgumentOutOfRangeException(nameof(servicePort), + servicePort, "Service port must be in the range 1–65535."); + var cluster = builder.Resource; var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster); @@ -79,10 +76,10 @@ public static IResourceBuilder AddServiceEndpoint( /// Injects the service URL exposed by into /// using the Aspire services__{name}__url convention. /// - /// Host processes receive https://localhost:{port}. - /// Container resources receive https://host.docker.internal:{port}. - /// On Linux without Docker Desktop, add - /// --add-host=host.docker.internal:host-gateway to the container runtime args. + /// Host processes receive http(s)://localhost:{port}. + /// Container resources receive http(s)://host.docker.internal:{port}. + /// The --add-host=host.docker.internal:host-gateway runtime arg is injected + /// automatically so the hostname resolves on Linux Docker Engine. /// /// [AspireExport("withReference", @@ -101,6 +98,14 @@ public static IResourceBuilder WithReference( if (destination.Resource is ContainerResource) { + // Inject --add-host so host.docker.internal resolves inside Linux containers. + // DCP does not inject this automatically; Docker Desktop on Mac/Windows resolves + // it natively, but Docker Engine on Linux requires the explicit mapping. + // ContainerRuntimeArgsCallbackAnnotation receives IList directly. + destination.Resource.Annotations.Add( + new ContainerRuntimeArgsCallbackAnnotation( + args => args.Add("--add-host=host.docker.internal:host-gateway"))); + return destination.WithEnvironment(ctx => { if (ep.IsReady) @@ -137,9 +142,6 @@ await notifications.PublishUpdateAsync(endpoint, "k3s local kubeconfig is not yet available for service endpoint."); } - // Allocate a host port by opening a listener, reading the OS-assigned port, - // then closing it before the forwarder binds — the port stays reserved in the - // kernel TIME_WAIT for long enough that the forwarder wins the race. var hostPort = AllocatePort(); endpoint.HostPort = hostPort; @@ -154,7 +156,6 @@ await notifications.PublishUpdateAsync(endpoint, isReady => { endpoint.IsReady = isReady; - var state = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy; var urls = isReady ? BuildUrls(scheme, endpoint.Name, hostPort, cluster.Name) : ImmutableArray.Empty; @@ -167,10 +168,6 @@ await notifications.PublishUpdateAsync(endpoint, }); _ = Task.Run(() => forwarder.RunAsync(logger, ct), ct); - - // Wait for the forwarder to signal ready (IsReady set via callback above). - // The health check also reads IsReady, so WaitFor on dependent resources - // naturally blocks until the port-forward is accepting connections. } catch (Exception ex) when (!ct.IsCancellationRequested) { @@ -203,7 +200,8 @@ private static int AllocatePort() /// /// Health check that satisfies WaitFor(serviceEndpoint). -/// Returns once the port-forward is active. +/// Returns once the port-forward has a confirmed +/// connection to a ready pod. /// internal sealed class K3sServiceEndpointHealthCheck(K3sServiceEndpointResource endpoint) : IHealthCheck { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index b630b613b..3f6a11865 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -1,4 +1,3 @@ -using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; using Microsoft.Extensions.DependencyInjection; @@ -245,7 +244,27 @@ public static IResourceBuilder WithK3sVersion( ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(tag); - return builder.WithImageTag(tag); + // Update the server image tag. + builder.WithImageTag(tag); + + // Sync all agent nodes to the same tag — mismatched server/agent versions can + // break node joins and exceed Kubernetes' supported ±1 minor version skew. + foreach (var agent in builder.Resource.AgentResources) + { + var existing = agent.Annotations.OfType().FirstOrDefault(); + if (existing is not null) + { + agent.Annotations.Remove(existing); + agent.Annotations.Add(new ContainerImageAnnotation + { + Image = existing.Image, + Tag = tag, + Registry = existing.Registry, + }); + } + } + + return builder; } /// Sets the pod subnet CIDR (--cluster-cidr). @@ -317,17 +336,20 @@ public static IResourceBuilder WithDataVolume( /// to the cluster. The injection method is selected automatically based on the resource type: /// /// - /// s receive KUBECONFIG_DATA (base-64-encoded YAML - /// of the container-network kubeconfig) and KUBECONFIG=/var/k3s/kubeconfig.yaml - /// once the file is injected at container-start time. + /// s receive a physical kubeconfig file copied to + /// /var/k3s/kubeconfig.yaml (container-network variant, + /// server: https://{resourceName}:6443). KUBECONFIG=/var/k3s/kubeconfig.yaml + /// is set automatically so all standard Kubernetes tooling (kubectl, helm, + /// KubernetesClient SDK) works without any custom bootstrap code. /// /// /// Projects and executables receive KUBECONFIG=<host path>/local/kubeconfig.yaml - /// pointing to a file that is accessible directly on the host filesystem. + /// pointing directly to a file on the host filesystem. /// /// - /// The variable is populated only after the cluster health check passes; use - /// WaitFor(cluster) on the dependent resource to guarantee ordering. + /// Both files are written by the health check after all nodes reach Ready state. + /// Use WaitFor(cluster) on the dependent resource to guarantee the files exist + /// before the resource starts. /// [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] public static IResourceBuilder WithReference( @@ -342,21 +364,24 @@ public static IResourceBuilder WithReference( if (destination.Resource is ContainerResource) { - // Containers receive KUBECONFIG_DATA (base64-encoded container-network kubeconfig). - // KubernetesClient reads this via: - // var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); - // using var stream = new MemoryStream(bytes); - // var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); - // For standard kubectl, use WithContainerFiles to inject a real file instead. - return destination.WithEnvironment(ctx => - { - if (cluster.KubeconfigDirectory is null) return; - var path = Path.Combine(cluster.KubeconfigDirectory, "container", "kubeconfig.yaml"); - if (!File.Exists(path)) return; - var yaml = File.ReadAllText(path); - ctx.EnvironmentVariables["KUBECONFIG_DATA"] = - Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); - }); + // Containers get a bind-mount of the container/ kubeconfig directory at /var/k3s. + // ContainerMountAnnotation is added directly to bypass the T : ContainerResource + // constraint on WithBindMount — the annotation is equivalent. + // + // Bind-mount (not file copy) is used for the same reason as in the helm and + // kubectl installer containers: if the cluster is recreated while a container is + // running, the new kubeconfig appears automatically without restarting the container. + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); + + destination.Resource.Annotations.Add( + new ContainerMountAnnotation( + containerKubeconfigDir, + "/var/k3s", + ContainerMountType.BindMount, + isReadOnly: true)); + + return destination.WithEnvironment("KUBECONFIG", "/var/k3s/kubeconfig.yaml"); } // Projects and executables: KUBECONFIG points to the host-accessible local kubeconfig. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs index 605e5c65c..a312b2e25 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs @@ -8,6 +8,6 @@ internal static class K3sContainerImageTags /// rancher/k3s public const string Image = "rancher/k3s"; - /// v1.31.4-k3s1 - public const string Tag = "v1.31.4-k3s1"; + /// v1.32.3-k3s1 + public const string Tag = "v1.32.3-k3s1"; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 1d704399b..64ceceacb 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -14,6 +14,12 @@ namespace CommunityToolkit.Aspire.Hosting; /// (localhost:{port}) and DCP-network containers /// (host.docker.internal:{port}) can reach the service. /// +/// +/// The callback is invoked with +/// only after a ready pod is confirmed via ListNamespacedPodAsync — not when the +/// TCP listener starts. This ensures WaitFor(endpoint) on dependent resources +/// correctly waits until the k8s service has a reachable pod. +/// /// internal sealed class K3sInProcessPortForwarder( string kubeconfigPath, @@ -33,12 +39,17 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) try { listener.Start(); - onReadyChanged(true); logger.LogInformation( "Port-forward: 0.0.0.0:{Local} → svc/{Service}.{Ns}:{Port}", localPort, serviceName, @namespace, servicePort); + // Probe the service before signalling ready — the Kubernetes service and + // a ready pod must exist before any connection can succeed. + // This makes the ready signal meaningful for WaitFor(endpoint) consumers. + await WaitForServiceReadyAsync(logger, ct).ConfigureAwait(false); + onReadyChanged(true); + while (!ct.IsCancellationRequested) { var tcp = await listener.AcceptTcpClientAsync(ct).ConfigureAwait(false); @@ -70,6 +81,57 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) } } + /// + /// Polls until the named service has at least one fully-ready pod. + /// This ensures the ready signal is only emitted when connections can actually succeed. + /// + private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); + using var k8sClient = new Kubernetes(config); + + var svc = await k8sClient.CoreV1 + .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) + .ConfigureAwait(false); + + var selector = string.Join(",", + (svc.Spec.Selector ?? new Dictionary()) + .Select(kv => $"{kv.Key}={kv.Value}")); + + var pods = await k8sClient.CoreV1 + .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ConfigureAwait(false); + + var hasReadyPod = pods.Items.Any(p => + p.Status?.Phase == "Running" && + p.Status?.ContainerStatuses?.All(c => c.Ready) == true); + + if (hasReadyPod) + { + logger.LogDebug( + "Service {Service}/{Ns} has a ready pod — port-forward is ready.", + serviceName, @namespace); + return; + } + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + logger.LogDebug(ex, + "Service {Service}/{Ns} not yet ready; retrying…", serviceName, @namespace); + } + + await Task.Delay(TimeSpan.FromSeconds(3), ct).ConfigureAwait(false); + } + } + private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, CancellationToken ct) { using var _ = tcp; @@ -102,9 +164,20 @@ private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, Cancell return; } + // Resolve the pod container port from the service's targetPort. + // WebSocketNamespacedPodPortForwardAsync requires the pod/container port, + // NOT the service port. For services where port != targetPort the wrong + // port would be forwarded otherwise. + var svcPort = svc.Spec.Ports.FirstOrDefault(p => p.Port == servicePort); + // TargetPort is IntOrString — its Value property is always a string. + // Parse it as an integer; if it's a named port or unset, fall back to the service port. + var podPort = svcPort?.TargetPort?.Value is { } tp && int.TryParse(tp, out var tpInt) + ? tpInt + : servicePort; + // Open WebSocket port-forward to the pod. using var ws = await k8sClient.WebSocketNamespacedPodPortForwardAsync( - pod.Metadata.Name, @namespace, [servicePort], + pod.Metadata.Name, @namespace, [podPort], cancellationToken: ct).ConfigureAwait(false); using var demuxer = new StreamDemuxer(ws, StreamType.PortForward); diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 7677ad7aa..d5f6feb5b 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -220,8 +220,8 @@ public void BuildHelmScriptIncludesUpgradeInstall() MakeRelease("argocd", "argo-cd", null, null, "argocd")); Assert.Contains("helm upgrade --install", script); - Assert.Contains("\"argocd\"", script); - Assert.Contains("\"argo-cd\"", script); + Assert.Contains("'argocd'", script); + Assert.Contains("'argo-cd'", script); } [Fact] @@ -231,7 +231,7 @@ public void BuildHelmScriptIncludesWaitAndNamespace() MakeRelease("r", "chart", null, null, "my-ns")); Assert.Contains("--wait", script); - Assert.Contains("--namespace \"my-ns\"", script); + Assert.Contains("--namespace 'my-ns'", script); Assert.Contains("--create-namespace", script); } @@ -253,7 +253,7 @@ public void BuildHelmScriptWithoutRepoSkipsRepoSteps() MakeRelease("r", "oci://registry/chart", null, null, "default")); Assert.DoesNotContain("helm repo add", script); - Assert.Contains("\"oci://registry/chart\"", script); + Assert.Contains("'oci://registry/chart'", script); } [Fact] @@ -262,7 +262,7 @@ public void BuildHelmScriptIncludesVersion() var script = K3sHelmBuilderExtensions.BuildHelmScript( MakeRelease("r", "chart", null, "7.8.0", "default")); - Assert.Contains("--version \"7.8.0\"", script); + Assert.Contains("--version '7.8.0'", script); } [Fact] @@ -284,8 +284,8 @@ public void BuildHelmScriptIncludesSetValues() ["replicaCount"] = "2", })); - Assert.Contains("--set \"service.type=NodePort\"", script); - Assert.Contains("--set \"replicaCount=2\"", script); + Assert.Contains("--set 'service.type=NodePort'", script); + Assert.Contains("--set 'replicaCount=2'", script); } // ── WaitForCompletion support ───────────────────────────────────────────── @@ -366,8 +366,8 @@ public void BuildHelmScriptIncludesValuesFilesWithIndexPrefix() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); // Index prefix guarantees uniqueness even when basenames collide. - Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); - Assert.Contains("--values \"/helm-values/1-values-prod.yaml\"", script); + Assert.Contains("--values '/helm-values/0-values.yaml'", script); + Assert.Contains("--values '/helm-values/1-values-prod.yaml'", script); } [Fact] @@ -385,8 +385,8 @@ public void BuildHelmScriptValuesFilesOrderedBeforeSetFlags() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); - var firstValuesIdx = script.IndexOf("--values \"/helm-values/0-", StringComparison.Ordinal); - var secondValuesIdx = script.IndexOf("--values \"/helm-values/1-", StringComparison.Ordinal); + var firstValuesIdx = script.IndexOf("--values '/helm-values/0-", StringComparison.Ordinal); + var secondValuesIdx = script.IndexOf("--values '/helm-values/1-", StringComparison.Ordinal); var setIdx = script.IndexOf("--set", StringComparison.Ordinal); Assert.True(firstValuesIdx < secondValuesIdx, "0-base.yaml must precede 1-prod.yaml"); @@ -404,8 +404,8 @@ public void BuildHelmScriptCollisionSafeWithSameBasename() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); - Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); - Assert.Contains("--values \"/helm-values/1-values.yaml\"", script); + Assert.Contains("--values '/helm-values/0-values.yaml'", script); + Assert.Contains("--values '/helm-values/1-values.yaml'", script); } [Fact] diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 0ca9b8bf9..19fd52758 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -185,29 +185,49 @@ public void AddK3sClusterWithClusterCidrViaOptions() public void WithReferenceSetsKubeconfigEnvForProject() { var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s"); - // ProjectResource would need a project file; use ExecutableResource as a proxy + // ProjectResource would need a project file; use ExecutableResource as a proxy. var exe = appBuilder.AddExecutable("myapp", "myapp", "."); exe.WithReference(cluster); - // Verify the environment callback was added (no exception thrown) using var app = appBuilder.Build(); - Assert.NotNull(app); + var model = app.Services.GetRequiredService(); + + var exeResource = Assert.Single(model.Resources.OfType()); + + // Executables receive KUBECONFIG pointing to local/kubeconfig.yaml on the host. + Assert.Contains( + exeResource.Annotations.OfType(), + a => a.Callback is not null); } [Fact] - public void WithReferenceSetsKubeconfigDataEnvForContainer() + public void WithReferenceMountsKubeconfigDirForContainer() { var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s"); var container = appBuilder.AddContainer("operator", "myorg/operator"); container.WithReference(cluster); using var app = appBuilder.Build(); - Assert.NotNull(app); + var model = app.Services.GetRequiredService(); + + var containerResource = model.Resources + .OfType() + .Single(r => r.Name == "operator"); + + // All containers (user containers, helm, and kubectl installers) receive a bind-mount + // of the container/ kubeconfig directory. Bind-mount is used so the kubeconfig + // updates automatically if the cluster is recreated without restarting the container. + var mount = containerResource.Annotations + .OfType() + .FirstOrDefault(m => m.Target == "/var/k3s"); + + Assert.NotNull(mount); + Assert.Equal(ContainerMountType.BindMount, mount.Type); + Assert.True(mount.IsReadOnly); + Assert.EndsWith(Path.Combine(".k3s", "k8s", "container"), mount.Source); } [Fact] From da4bf9dfdadb2728b97353a7751e9c637a5322ca Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 16:58:59 +0200 Subject: [PATCH 07/29] chore: exluding integration tests for win runners --- .github/workflows/tests.yaml | 5 +++++ .../K3sIntegrationTests.cs | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c4cc48db9..a08ff774c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -83,6 +83,11 @@ jobs: Sftp.Tests, SurrealDb.Tests, ] + exclude: + # k3s integration tests require privileged Linux containers. + # GitHub-hosted Windows runners do not support this reliably. + - os: windows-latest + name: Hosting.K3s.IntegrationTests steps: - name: Checkout code diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs index 226335163..62887833a 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -21,8 +21,8 @@ namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; /// /// No host-side helm or kubectl is needed — both run as Docker containers /// (alpine/helm and alpine/k8s). -/// Tests are gated by [RequiresDocker] and run on both ubuntu-latest -/// and windows-latest CI jobs since privileged Linux containers work on Docker Desktop. +/// Tests are gated by [RequiresDocker] and run on ubuntu-latest only — +/// GitHub-hosted Windows runners do not support privileged Linux containers reliably. /// /// [RequiresDocker] From ff66be4a4d9d032192d45f7cde9bd8dddf07bf64 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 17:10:09 +0200 Subject: [PATCH 08/29] chore: update example comment Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../ValidationAppHost/apphost.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts index c9830b8f3..1085187e0 100644 --- a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts @@ -67,7 +67,7 @@ if (includeCompileOnlyScenarios) { .addProject('operator', { projectPath: '../WidgetOperator/WidgetOperator.csproj' }) .withReference(configuredCluster); - // Container: receives KUBECONFIG_DATA= + // Container: receives a bind-mounted kubeconfig and KUBECONFIG=/var/k3s/kubeconfig.yaml const _containerRef = builder .addContainer('sidecar', 'myorg/sidecar') .withReference(configuredCluster); From 37c11e32d384dbf7a7e7e0c3dfa06bf945f80495 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 19:56:11 +0200 Subject: [PATCH 09/29] fix: copilot reviews --- .../K3sBuilderExtensions.Helm.cs | 39 ++++++------ .../K3sBuilderExtensions.Manifest.cs | 49 +++++++++------ .../K3sClusterOptions.cs | 4 +- .../K3sClusterResource.cs | 2 +- .../K3sInProcessPortForwarder.cs | 59 ++++++++++++++----- .../K3sReadinessHealthCheck.cs | 12 ++-- .../K8sManifestResource.cs | 2 +- .../KubectlContainerImageTags.cs | 9 ++- .../README.md | 16 +++-- .../K3sIntegrationTests.cs | 45 ++++++++++---- .../K3sClusterResourceTests.cs | 26 ++++++-- 11 files changed, 171 insertions(+), 92 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index c5ade685f..cccf24087 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -69,10 +69,10 @@ public static IResourceBuilder AddHelmRelease( // The install script is injected as /helm-install.sh via WithContainerFiles. // The callback fires when the container is being started (after WaitFor(cluster) // is satisfied), so all WithHelmValue() calls have been made by then. - .WithContainerFiles("/", async (ctx, ct) => + .WithContainerFiles("/", (ctx, ct) => { var script = BuildHelmScript(release); - return [new ContainerFile + IEnumerable items = [new ContainerFile { Name = "helm-install.sh", Contents = script, @@ -80,23 +80,22 @@ public static IResourceBuilder AddHelmRelease( | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, }]; + return Task.FromResult(items); }) .WithArgs("/helm-install.sh") - // Inject host-side values files declared via WithHelmValuesFile using - // Aspire's built-in WithContainerFiles(destinationPath, hostSourcePath). - // One call per file; each copies the file into /helm-values/ in the container. - // The callback wraps all files so it fires after all WithHelmValuesFile() calls. - .WithContainerFiles("/helm-values", async (ctx, ct) => - release.ValuesFiles + // Inject host-side values files declared via WithHelmValuesFile. + // The callback fires at container-start time so all WithHelmValuesFile() calls + // have been made and ValuesFiles is fully populated. + .WithContainerFiles("/helm-values", (ctx, ct) => + { + IEnumerable items = [.. release.ValuesFiles .Select((hostPath, i) => (ContainerFileSystemItem)new ContainerFile { - // Prefix with index so files are unique even if basenames collide - // (e.g. prod/values.yaml + base/values.yaml → 0-values.yaml, 1-values.yaml) - // and order is explicit on the filesystem as well as in the script. - Name = $"{i}-{System.IO.Path.GetFileName(hostPath)}", + Name = $"{i}-{Path.GetFileName(hostPath)}", SourcePath = hostPath, - }) - .ToList()) + })]; + return Task.FromResult(items); + }) .WithBindMount(containerKubeconfigDir, "/root/.kube") .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") .WithIconName("Rocket") @@ -167,11 +166,13 @@ internal static string BuildHelmScript(HelmReleaseResource release) { var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - // Poll until the k3s health check writes the kubeconfig — the file appears only - // after all nodes are Ready. This replaces WaitFor(cluster) since a container - // cannot WaitFor its IResourceWithParent. - sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); - sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + // Poll until the kubeconfig exists AND the k3s API server is reachable. + // DCP sets up container network aliases asynchronously, so the kubeconfig file + // can appear in the bind-mount before the k8s hostname resolves in the helm + // container. Using `helm list` (which calls the k8s API) verifies both the + // file and the network path before proceeding. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ] && helm list --kubeconfig /root/.kube/kubeconfig.yaml > /dev/null 2>&1; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready and reachable...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 33c7ac128..997131dcb 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -13,7 +13,7 @@ public static class K3sManifestBuilderExtensions { /// /// Applies one or more Kubernetes YAML files — or a Kustomize overlay — to the cluster - /// via a bitnami/kubectl container. No host-side kubectl binary is required. + /// via a rancher/kubectl container. No host-side kubectl binary is required. /// /// The container exits with code 0 after manifests are applied and any CRDs reach the /// Established condition. Use WaitForCompletion(manifest) on dependent resources. @@ -65,15 +65,18 @@ public static IResourceBuilder AddK8sManifest( .WithEntrypoint("/bin/sh") // Script is injected at "/kubectl-apply.sh". The script auto-detects whether // /k8s-manifests contains a kustomization.yaml and uses -k or -f accordingly. - .WithContainerFiles("/", async (ctx, ct) => - [new ContainerFile + .WithContainerFiles("/", (ctx, ct) => + { + IEnumerable items = [new ContainerFile { Name = "kubectl-apply.sh", Contents = BuildManifestScript(), Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, - }]) + }]; + return Task.FromResult(items); + }) .WithArgs("/kubectl-apply.sh") .WithBindMount(containerKubeconfigDir, "/root/.kube"); @@ -90,8 +93,10 @@ [new ContainerFile // need not exist when the AppHost is built (only when the container starts). // This mirrors WithContainerFiles(path, hostPath) semantics but without the // build-time path validation that Aspire's string overload performs. - resourceBuilder.WithContainerFiles("/k8s-manifests", async (ctx, ct) => + resourceBuilder.WithContainerFiles("/k8s-manifests", (ctx, ct) => { + IEnumerable items; + if (Directory.Exists(absolutePath)) { var files = Directory @@ -100,19 +105,22 @@ [new ContainerFile .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase); - return [.. files - .Select(f => (ContainerFileSystemItem)new ContainerFile - { - Name = System.IO.Path.GetFileName(f), - SourcePath = f, - })]; + items = [.. files.Select(f => new ContainerFile + { + Name = Path.GetFileName(f), + SourcePath = f, + })]; } - - return [(ContainerFileSystemItem)new ContainerFile + else { - Name = System.IO.Path.GetFileName(absolutePath), - SourcePath = absolutePath, - }]; + items = [new ContainerFile + { + Name = Path.GetFileName(absolutePath), + SourcePath = absolutePath, + }]; + } + + return Task.FromResult(items); }); } @@ -138,9 +146,12 @@ internal static string BuildManifestScript() { var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - // Poll until the k3s health check writes the kubeconfig. - sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); - sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + // Poll until the kubeconfig exists AND the k3s API server is reachable. + // DCP sets up container network aliases asynchronously, so the kubeconfig file + // can appear in the bind-mount before the k8s hostname resolves in the kubectl + // container. Using `kubectl cluster-info` verifies both the file and the network. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ] && kubectl cluster-info --kubeconfig /root/.kube/kubeconfig.yaml > /dev/null 2>&1; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready and reachable...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs index ab67ca836..bc161e80f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -68,13 +68,13 @@ public sealed class K3sClusterOptions /// /// Gets or sets the kubectl container image name used by manifest applies. - /// Defaults to alpine/k8s. + /// Defaults to rancher/kubectl — maintained by the same team as k3s. /// public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; /// /// Gets or sets the kubectl container image tag used by manifest applies. - /// Defaults to 1.32.3. + /// Defaults to v1.32.3, matching the default k3s server version. /// public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index 48baf3907..9390de879 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -17,7 +17,7 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) /// Container image settings for the kubectl manifest applier, resolved from cluster options. internal (string Registry, string Image, string Tag) KubectlImageInfo { get; set; } - = ("docker.io", "alpine/k8s", "1.32.3"); + = ("docker.io", "rancher/kubectl", "v1.32.3"); /// /// Host-side directory that holds all kubeconfig variants for this cluster. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 64ceceacb..82a285587 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -98,12 +98,19 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) .ConfigureAwait(false); - var selector = string.Join(",", - (svc.Spec.Selector ?? new Dictionary()) - .Select(kv => $"{kv.Key}={kv.Value}")); + if (svc.Spec.Selector is null or { Count: 0 }) + { + logger.LogWarning( + "Service {Service}/{Ns} has no pod selector — cannot determine readiness.", + serviceName, @namespace); + return; + } + + var labelSelector = string.Join(",", + svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); var pods = await k8sClient.CoreV1 - .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) .ConfigureAwait(false); var hasReadyPod = pods.Items.Any(p => @@ -145,11 +152,19 @@ private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, Cancell .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) .ConfigureAwait(false); - var selector = string.Join(",", - (svc.Spec.Selector ?? new Dictionary()).Select(kv => $"{kv.Key}={kv.Value}")); + if (svc.Spec.Selector is null or { Count: 0 }) + { + logger.LogWarning( + "Service {Service}/{Ns} has no pod selector — connection dropped.", + serviceName, @namespace); + return; + } + + var labelSelector = string.Join(",", + svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); var pods = await k8sClient.CoreV1 - .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) .ConfigureAwait(false); var pod = pods.Items.FirstOrDefault(p => @@ -165,15 +180,29 @@ private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, Cancell } // Resolve the pod container port from the service's targetPort. - // WebSocketNamespacedPodPortForwardAsync requires the pod/container port, - // NOT the service port. For services where port != targetPort the wrong - // port would be forwarded otherwise. + // WebSocketNamespacedPodPortForwardAsync requires the container port, not the + // service port. targetPort can be a numeric string or a named port string. var svcPort = svc.Spec.Ports.FirstOrDefault(p => p.Port == servicePort); - // TargetPort is IntOrString — its Value property is always a string. - // Parse it as an integer; if it's a named port or unset, fall back to the service port. - var podPort = svcPort?.TargetPort?.Value is { } tp && int.TryParse(tp, out var tpInt) - ? tpInt - : servicePort; + int podPort; + if (svcPort?.TargetPort?.Value is { } tp) + { + if (int.TryParse(tp, out var numeric)) + { + podPort = numeric; + } + else + { + // Named targetPort — resolve against the selected pod's container ports. + podPort = pod.Spec.Containers + .SelectMany(c => c.Ports ?? []) + .FirstOrDefault(p => p.Name == tp) + ?.ContainerPort ?? servicePort; + } + } + else + { + podPort = servicePort; + } // Open WebSocket port-forward to the pod. using var ws = await k8sClient.WebSocketNamespacedPodPortForwardAsync( diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index a51a9e2e2..22851041f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -69,14 +69,18 @@ public async Task CheckHealthAsync( } catch (Exception ex) when (IsTlsOrAuthFailure(ex)) { - // Stale kubeconfig — cluster was recreated with new certs (e.g. data volume wiped). - // Invalidate everything; k3s will overwrite cluster/kubeconfig.yaml on next start. + // Stale cached client — the cluster was recreated with new certs while the + // health check held an old IKubernetes instance. k3s has already written a fresh + // kubeconfig to rawPath (it writes once at startup, not continuously), so we must + // NOT delete rawPath — that would remove the fresh file and leave the health check + // waiting forever for a file that k3s will never rewrite. + // Instead, discard only the cached client and the derived variants so they are + // regenerated from the fresh raw file on the next check cycle. _cachedClient?.Dispose(); _cachedClient = null; - TryDelete(rawPath); TryDelete(Path.Combine(dir, "local", "kubeconfig.yaml")); TryDelete(Path.Combine(dir, "container", "kubeconfig.yaml")); - return HealthCheckResult.Unhealthy("k3s kubeconfig is stale — waiting for cluster refresh"); + return HealthCheckResult.Unhealthy("k3s kubeconfig is stale — retrying with fresh credentials"); } catch (Exception ex) { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index 2610f033c..9893def84 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -4,7 +4,7 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via -/// kubectl apply --server-side running inside a bitnami/kubectl container. +/// kubectl apply --server-side running inside a rancher/kubectl container. /// /// The container polls for the cluster kubeconfig (written when the cluster health check /// first passes), applies the manifests, waits for any CRDs to reach Established, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs index eb6eedb68..5ead02f04 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs @@ -3,9 +3,8 @@ namespace CommunityToolkit.Aspire.Hosting; internal static class KubectlContainerImageTags { internal const string Registry = "docker.io"; - // alpine/k8s: lightweight Alpine-based image that includes kubectl and other k8s tools. - // Same organisation as alpine/helm — consistent image family. - internal const string Image = "alpine/k8s"; - // Matches the Kubernetes version shipped by the default k3s tag. - internal const string Tag = "1.32.3"; + // rancher/kubectl: maintained by the same team as k3s. Version tags mirror + // the Kubernetes version, so v1.32.x pairs correctly with the default k3s tag. + internal const string Image = "rancher/kubectl"; + internal const string Tag = "v1.32.3"; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index cc830bcbe..cef56ce7a 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -65,7 +65,7 @@ Values are applied in this order (last wins): ## Applying Kubernetes manifests -`AddK8sManifest` runs `kubectl apply --server-side` inside an `alpine/k8s` container. +`AddK8sManifest` runs `kubectl apply --server-side` inside an `rancher/kubectl` container. No host-side `kubectl` binary is required. Kustomize overlays are auto-detected. ```csharp @@ -115,18 +115,16 @@ builder.AddContainer("sidecar", "myorg/sidecar") | Consumer type | What is injected | |---|---| | `ProjectResource` / `ExecutableResource` | `KUBECONFIG=…/.k3s/k8s/local/kubeconfig.yaml` | -| `ContainerResource` | `KUBECONFIG_DATA=` | +| `ContainerResource` | Bind-mount of `container/kubeconfig.yaml` at `/var/k3s/` + `KUBECONFIG=/var/k3s/kubeconfig.yaml` | + +All standard Kubernetes tooling (`kubectl`, `helm`, KubernetesClient SDK) reads `KUBECONFIG` automatically — no custom bootstrap code required. Reading in .NET: ```csharp -// Project / executable — standard kubectl convention +// Works identically for both projects and containers — the SDK reads KUBECONFIG automatically. var config = KubernetesClientConfiguration.BuildConfigFromConfigFile( Environment.GetEnvironmentVariable("KUBECONFIG")); - -// Container — decode from env var -var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); -using var stream = new MemoryStream(bytes); -var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); +using var client = new Kubernetes(config); ``` ## Persistent cluster state @@ -154,7 +152,7 @@ The health check waits for all nodes to reach `Ready` before the cluster is mark ## Image overrides -The `alpine/helm` and `alpine/k8s` images are pinned but configurable: +The `alpine/helm` and `rancher/kubectl` images are pinned but configurable: ```csharp builder.AddK3sCluster("k8s", configure: opts => diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs index 62887833a..3861c4c16 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -270,27 +270,46 @@ await k8sClient.CoreV1.CreateNamespacedConfigMapAsync( _app = null; // ── Second run with the same named volume ───────────────────────── - var builder2 = TestDistributedApplicationBuilder.Create(); + using var builder2 = TestDistributedApplicationBuilder.Create(); builder2.AddK3sCluster("k8s").WithDataVolume(volumeName); await using var app2 = builder2.Build(); - await app2.StartAsync(); + try + { + await app2.StartAsync(); - var rns2 = app2.Services.GetRequiredService(); - using var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - await rns2.WaitForResourceHealthyAsync("k8s", cts2.Token); + var rns2 = app2.Services.GetRequiredService(); + using var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await rns2.WaitForResourceHealthyAsync("k8s", cts2.Token); - var kubeconfigPath2 = Path.Combine( - builder2.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + var kubeconfigPath2 = Path.Combine( + builder2.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); - using var k8sClient2 = new Kubernetes( - KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath2)); + using var k8sClient2 = new Kubernetes( + KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath2)); - var cm = await k8sClient2.CoreV1.ReadNamespacedConfigMapAsync( - "persist-check", "default", cancellationToken: cts2.Token); + var cm = await k8sClient2.CoreV1.ReadNamespacedConfigMapAsync( + "persist-check", "default", cancellationToken: cts2.Token); - Assert.Equal("first", cm.Data["run"]); + Assert.Equal("first", cm.Data["run"]); + } + finally + { + await app2.StopAsync(); - await app2.StopAsync(); + // Remove the named volume so it does not accumulate on CI runners. + try + { + using var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo("docker") + { + ArgumentList = { "volume", "rm", "--force", volumeName }, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }); + if (process is not null) await process.WaitForExitAsync(); + } + catch { /* best effort */ } + } } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 19fd52758..2e8b98d5b 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -251,7 +251,13 @@ public void WithPodSubnetAddsClusterCidrArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var args = resource.Annotations.OfType(); + Assert.Contains(args, a => a is not null); // arg callbacks registered + // Verify the actual arg value by evaluating the callbacks. + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in args) a.Callback(ctx); + Assert.Contains("--cluster-cidr=10.88.0.0/16", ctx.Args); } [Fact] @@ -264,7 +270,11 @@ public void WithServiceSubnetAddsServiceCidrArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + Assert.Contains("--service-cidr=10.89.0.0/16", ctx.Args); } [Fact] @@ -277,7 +287,11 @@ public void WithDisabledComponentAddsDisableArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + Assert.Contains("--disable=traefik", ctx.Args); } [Fact] @@ -290,7 +304,11 @@ public void WithExtraArgAddsRawArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + Assert.Contains("--write-kubeconfig-mode=644", ctx.Args); } [Fact] From 99297cdfd52f885dd89e7e6b8991417b6bc40411 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:15:10 +0200 Subject: [PATCH 10/29] fix: WaitForServiceReadyAsync service health check Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../K3sInProcessPortForwarder.cs | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 82a285587..014d21a34 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -98,31 +98,40 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) .ConfigureAwait(false); - if (svc.Spec.Selector is null or { Count: 0 }) + var exposesRequestedPort = svc.Spec?.Ports?.Any(p => p.Port == servicePort) == true; + if (!exposesRequestedPort) + { + logger.LogDebug( + "Service {Service}/{Ns} does not expose requested port {ServicePort}; retrying…", + serviceName, @namespace, servicePort); + } + else if (svc.Spec?.Selector is null or { Count: 0 }) { logger.LogWarning( "Service {Service}/{Ns} has no pod selector — cannot determine readiness.", serviceName, @namespace); return; } - - var labelSelector = string.Join(",", - svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); - - var pods = await k8sClient.CoreV1 - .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) - .ConfigureAwait(false); - - var hasReadyPod = pods.Items.Any(p => - p.Status?.Phase == "Running" && - p.Status?.ContainerStatuses?.All(c => c.Ready) == true); - - if (hasReadyPod) + else { - logger.LogDebug( - "Service {Service}/{Ns} has a ready pod — port-forward is ready.", - serviceName, @namespace); - return; + var labelSelector = string.Join(",", + svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); + + var pods = await k8sClient.CoreV1 + .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) + .ConfigureAwait(false); + + var hasReadyPod = pods.Items.Any(p => + p.Status?.Phase == "Running" && + p.Status?.ContainerStatuses?.All(c => c.Ready) == true); + + if (hasReadyPod) + { + logger.LogDebug( + "Service {Service}/{Ns} exposes requested port {ServicePort} and has a ready pod — port-forward is ready.", + serviceName, @namespace, servicePort); + return; + } } } catch (OperationCanceledException) when (ct.IsCancellationRequested) From 7d34ddc3724d9c080fcc69d63b0728634198e2cb Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:16:23 +0200 Subject: [PATCH 11/29] chore: addressing reviews --- .../CommunityToolkit.Aspire.Hosting.K3s.csproj | 2 +- .../K3sBuilderExtensions.cs | 3 +-- src/CommunityToolkit.Aspire.Hosting.K3s/README.md | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj index 67a2f57a5..1ef4e1af1 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -2,7 +2,7 @@ kubernetes k3s hosting cluster - An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via alpine/k8s), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. + An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via rancher/kubectl), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index 3f6a11865..c5908b7dc 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -327,8 +327,7 @@ public static IResourceBuilder WithDataVolume( ArgumentNullException.ThrowIfNull(builder); return builder - .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s") - .WithContainerRuntimeArgs("--restart=unless-stopped"); + .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s"); } /// diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index cef56ce7a..23112e81e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -4,7 +4,7 @@ Provides extension methods and resource definitions for the .NET Aspire AppHost [k3s](https://k3s.io/) lightweight Kubernetes cluster as part of the local development inner loop. The cluster, Helm chart installs, manifest applies, and service endpoint exposures all appear as first-class resources in the Aspire dashboard — no external -tooling beyond Docker is required. +tooling beyond a supported container runtime is required. ## Getting Started From 7473170af228b0a5d7f15ffb8adb3b2c9415ff31 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:24:56 +0200 Subject: [PATCH 12/29] fix: copilot reviews --- .../K3sBuilderExtensions.ServiceEndpoint.cs | 17 +++++++++++++---- .../K3sInProcessPortForwarder.cs | 15 +++++++++++---- .../K3sServiceEndpointResource.cs | 7 +++++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs index 0a2a12eb1..f84c99e00 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -34,7 +34,8 @@ public static IResourceBuilder AddServiceEndpoint( [ResourceName] string name, string serviceName, int servicePort, - string @namespace = "default") + string @namespace = "default", + string? scheme = null) { ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(name); @@ -44,8 +45,16 @@ public static IResourceBuilder AddServiceEndpoint( throw new ArgumentOutOfRangeException(nameof(servicePort), servicePort, "Service port must be in the range 1–65535."); + // Infer scheme from the port when not explicitly provided. + // Callers should pass an explicit scheme whenever the Kubernetes service port + // does not reliably indicate the application protocol (e.g. HTTPS on port 80). + var resolvedScheme = scheme ?? (servicePort is 443 or 8443 ? "https" : "http"); + var cluster = builder.Resource; - var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster); + var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster) + { + Scheme = resolvedScheme, + }; var healthCheckKey = $"k3s_endpoint_{name}_ready"; builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( @@ -93,7 +102,7 @@ public static IResourceBuilder WithReference( ArgumentNullException.ThrowIfNull(source); var ep = source.Resource; - var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + var scheme = ep.Scheme; var envKey = $"services__{ep.Name}__url"; if (destination.Resource is ContainerResource) @@ -145,7 +154,7 @@ await notifications.PublishUpdateAsync(endpoint, var hostPort = AllocatePort(); endpoint.HostPort = hostPort; - var scheme = endpoint.ServicePort is 443 or 8443 ? "https" : "http"; + var scheme = endpoint.Scheme; var forwarder = new K3sInProcessPortForwarder( kubeconfigPath, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 014d21a34..96ce3b79a 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -62,6 +62,15 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) { break; } + catch (InvalidOperationException ioe) when (!ct.IsCancellationRequested) + { + // Non-retryable configuration error (e.g. service has no pod selector). + // Log and stop — retrying will never succeed. + logger.LogError(ioe, "Port-forward for svc/{Service}/{Ns} cannot be established.", + serviceName, @namespace); + onReadyChanged(false); + break; + } catch (Exception ex) { logger.LogWarning(ex, @@ -107,10 +116,8 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct } else if (svc.Spec?.Selector is null or { Count: 0 }) { - logger.LogWarning( - "Service {Service}/{Ns} has no pod selector — cannot determine readiness.", - serviceName, @namespace); - return; + throw new InvalidOperationException( + $"Service {serviceName}/{@namespace} has no pod selector and cannot be port-forwarded by {nameof(K3sInProcessPortForwarder)}."); } else { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs index 0af117b1e..d2a039de0 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -42,6 +42,13 @@ public sealed class K3sServiceEndpointResource( /// public int HostPort { get; internal set; } + /// + /// The URL scheme used for services__{name}__url injection and dashboard URLs. + /// Set by AddServiceEndpoint; callers can override the default port-based inference + /// via the scheme parameter. + /// + internal string Scheme { get; init; } = "http"; + /// /// when the port-forward is active and accepting connections. /// Set by K3sInProcessPortForwarder; read by the health check. From 98bf92164fc5c3b383ed8f025a166de5343ebce2 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:39:40 +0200 Subject: [PATCH 13/29] fix: copilot reviews --- .../K3sBuilderExtensions.Helm.cs | 27 ++++++++++++++----- .../K3sBuilderExtensions.Manifest.cs | 11 +++++--- .../K3sBuilderExtensions.cs | 6 ++++- .../K3sInProcessPortForwarder.cs | 5 ++++ .../CommunityToolkit.Aspire.Hosting.K3s.cs | 2 +- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index cccf24087..d0d159bfc 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -204,20 +204,33 @@ internal static string BuildHelmScript(HelmReleaseResource release) } // --set flags override everything above (highest Helm precedence). - // Both key and value are single-quoted so quotes, spaces, or $ in values are safe. + // Values are Helm-escaped then shell-escaped: + // 1. HelmEscape: escapes Helm's --set parser metacharacters (`,`, `{`, `}`, `\`) + // so Helm treats them as literals rather than array/map/list syntax. + // 2. ShellEscape: wraps in POSIX single quotes so the shell passes the value + // to Helm without any shell interpretation. + // For values containing commas, braces, or backslashes that Helm --set cannot + // represent safely (e.g. multi-line strings), use WithHelmValuesFile instead. foreach (var (key, value) in release.HelmValues) - sb.Append($" --set {ShellEscape($"{key}={value}")}"); + sb.Append($" --set {ShellEscape($"{key}={HelmEscape(value)}")}"); return sb.ToString(); } + /// + /// Escapes Helm --set value metacharacters so that Helm's own parser treats + /// them as literals rather than as array/map/list syntax delimiters. + /// + private static string HelmEscape(string value) => + value.Replace("\\", "\\\\") // backslash first to avoid double-escaping + .Replace(",", "\\,") // comma separates multiple assignments + .Replace("{", "\\{") // brace opens a map/list literal + .Replace("}", "\\}"); + /// /// Wraps in POSIX single quotes so that all shell - /// metacharacters ($, `, \, ", ;, - /// &, |, space, etc.) are treated as literals in Dockerfile - /// RUN shell-form commands and Delve's --build-flags parser. - /// Embedded single quotes are escaped with the standard POSIX technique: - /// ' → '\''. + /// metacharacters are treated as literals. Embedded single quotes are escaped + /// with the standard POSIX technique: ' → '\''. /// private static string ShellEscape(string value) => $"'{value.Replace("'", "'\\''")}'"; diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 997131dcb..d79ef7344 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -164,9 +164,14 @@ internal static string BuildManifestScript() sb.AppendLine(" kubectl apply -f /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); sb.AppendLine("fi"); - // Wait for CRD Established condition if any CRDs are present. - sb.AppendLine("if kubectl get crd --no-headers 2>/dev/null | grep -q .; then"); - sb.AppendLine(" kubectl wait --for=condition=Established crd --all --timeout=300s"); + // Wait only for CRDs applied by this manifest, identified via the aspire-k3s + // field-manager. Using --all would also wait for pre-existing or concurrently + // installed CRDs that are stuck, causing an unrelated manifest to hang. + sb.AppendLine("CRDS=$(kubectl get crd -o name --no-headers 2>/dev/null \\"); + sb.AppendLine(" | xargs -r -I{} sh -c 'kubectl get {} -o jsonpath=\"{.metadata.managedFields[*].manager}\" 2>/dev/null | grep -q aspire-k3s && echo {}' 2>/dev/null)"); + sb.AppendLine("if [ -n \"$CRDS\" ]; then"); + sb.AppendLine(" # shellcheck disable=SC2086"); + sb.AppendLine(" kubectl wait --for=condition=Established $CRDS --timeout=300s"); sb.AppendLine("fi"); return sb.ToString(); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index c5908b7dc..ff82744db 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -108,10 +108,14 @@ public static IResourceBuilder AddK3sCluster( .WithArgs("--kubelet-arg=v=0") // ── API server endpoint ─────────────────────────────────────────── + // Proxy support must be disabled: the kubeconfig embeds the k3s server CA cert + // for TLS validation. An Aspire HTTPS proxy would present its own certificate, + // causing Kubernetes client TLS validation to fail on every connection. .WithHttpsEndpoint( targetPort: 6443, port: apiServerPort, - name: K3sClusterResource.ApiServerEndpointName) + name: K3sClusterResource.ApiServerEndpointName, + isProxied: false) // ── Docker / container runtime flags (mirrors k3d) ──────────────── .WithContainerRuntimeArgs("--privileged") diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 96ce3b79a..b00dc9141 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -35,6 +35,11 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) while (!ct.IsCancellationRequested) { + // IPAddress.Any (0.0.0.0) is required: DCP-network containers reach the + // forwarded service via host.docker.internal:{port}, which resolves to the + // Docker host IP — not 127.0.0.1. Binding to loopback would silently drop + // all container traffic. Users on shared networks should be aware that the + // forwarded service is reachable from other hosts on the same LAN. var listener = new TcpListener(IPAddress.Any, localPort); try { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs index 770655906..4abe3607c 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -60,7 +60,7 @@ public static partial class K3sManifestBuilderExtensions public static partial class K3sServiceEndpointExtensions { [AspireExport("addServiceEndpoint", Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] - public static ApplicationModel.IResourceBuilder AddServiceEndpoint(this ApplicationModel.IResourceBuilder builder, string name, string serviceName, int servicePort, string @namespace = "default") { throw null; } + public static ApplicationModel.IResourceBuilder AddServiceEndpoint(this ApplicationModel.IResourceBuilder builder, string name, string serviceName, int servicePort, string @namespace = "default", string? scheme = null) { throw null; } [AspireExport("withReference", Description = "Injects the k3s service URL into a dependent resource")] public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source) From 8c11cc581f50ab5b0f7092824275873189451b26 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Wed, 20 May 2026 13:00:36 +0200 Subject: [PATCH 14/29] fix: issues with the integration test build and k3s api port proxying --- ...CommunityToolkit.Aspire.Hosting.K3s.csproj | 2 +- .../K3sBuilderExtensions.Manifest.cs | 2 +- .../K3sBuilderExtensions.cs | 10 +++--- .../K3sClusterOptions.cs | 4 +-- .../K3sClusterResource.cs | 2 +- .../K3sContainerImageTags.cs | 4 +-- .../K8sManifestResource.cs | 2 +- .../KubectlContainerImageTags.cs | 8 ++--- .../README.md | 6 ++-- .../K3sIntegrationTests.cs | 32 +++++++++---------- 10 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj index 1ef4e1af1..cfaf9aa7c 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -2,7 +2,7 @@ kubernetes k3s hosting cluster - An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via rancher/kubectl), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. + An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via alpine/kubectl), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index d79ef7344..e4d3863de 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -13,7 +13,7 @@ public static class K3sManifestBuilderExtensions { /// /// Applies one or more Kubernetes YAML files — or a Kustomize overlay — to the cluster - /// via a rancher/kubectl container. No host-side kubectl binary is required. + /// via a alpine/kubectl container. No host-side kubectl binary is required. /// /// The container exits with code 0 after manifests are applied and any CRDs reach the /// Established condition. Use WaitForCompletion(manifest) on dependent resources. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index ff82744db..ac1f88c5f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -108,14 +108,14 @@ public static IResourceBuilder AddK3sCluster( .WithArgs("--kubelet-arg=v=0") // ── API server endpoint ─────────────────────────────────────────── - // Proxy support must be disabled: the kubeconfig embeds the k3s server CA cert - // for TLS validation. An Aspire HTTPS proxy would present its own certificate, - // causing Kubernetes client TLS validation to fail on every connection. + // Declared as HTTPS so the allocated host port appears in the Aspire dashboard + // with the correct scheme. Aspire's HTTP proxy does not intercept raw TLS TCP + // connections, so Kubernetes clients validate the k3s server CA cert directly + // without any proxy interference. .WithHttpsEndpoint( targetPort: 6443, port: apiServerPort, - name: K3sClusterResource.ApiServerEndpointName, - isProxied: false) + name: K3sClusterResource.ApiServerEndpointName) // ── Docker / container runtime flags (mirrors k3d) ──────────────── .WithContainerRuntimeArgs("--privileged") diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs index bc161e80f..08da6dc38 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -68,13 +68,13 @@ public sealed class K3sClusterOptions /// /// Gets or sets the kubectl container image name used by manifest applies. - /// Defaults to rancher/kubectl — maintained by the same team as k3s. + /// Defaults to alpine/kubectl — maintained by the same team as k3s. /// public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; /// /// Gets or sets the kubectl container image tag used by manifest applies. - /// Defaults to v1.32.3, matching the default k3s server version. + /// Defaults to 1.36.0, matching the default k3s server version. /// public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index 9390de879..1c8c8570e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -17,7 +17,7 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) /// Container image settings for the kubectl manifest applier, resolved from cluster options. internal (string Registry, string Image, string Tag) KubectlImageInfo { get; set; } - = ("docker.io", "rancher/kubectl", "v1.32.3"); + = ("docker.io", "alpine/kubectl", "1.36.0"); /// /// Host-side directory that holds all kubeconfig variants for this cluster. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs index a312b2e25..15fc071d9 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs @@ -8,6 +8,6 @@ internal static class K3sContainerImageTags /// rancher/k3s public const string Image = "rancher/k3s"; - /// v1.32.3-k3s1 - public const string Tag = "v1.32.3-k3s1"; + /// v1.36.0-k3s1 + public const string Tag = "v1.36.0-k3s1"; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index 9893def84..b09afdbf2 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -4,7 +4,7 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via -/// kubectl apply --server-side running inside a rancher/kubectl container. +/// kubectl apply --server-side running inside a alpine/kubectl container. /// /// The container polls for the cluster kubeconfig (written when the cluster health check /// first passes), applies the manifests, waits for any CRDs to reach Established, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs index 5ead02f04..6ec6838d9 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs @@ -3,8 +3,8 @@ namespace CommunityToolkit.Aspire.Hosting; internal static class KubectlContainerImageTags { internal const string Registry = "docker.io"; - // rancher/kubectl: maintained by the same team as k3s. Version tags mirror - // the Kubernetes version, so v1.32.x pairs correctly with the default k3s tag. - internal const string Image = "rancher/kubectl"; - internal const string Tag = "v1.32.3"; + // alpine/kubectl: Alpine-based image with kubectl and /bin/sh, required by the + // manifest apply script. Tag matches the default k3s server Kubernetes version. + internal const string Image = "alpine/kubectl"; + internal const string Tag = "1.36.0"; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index 23112e81e..15cc59f53 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -65,7 +65,7 @@ Values are applied in this order (last wins): ## Applying Kubernetes manifests -`AddK8sManifest` runs `kubectl apply --server-side` inside an `rancher/kubectl` container. +`AddK8sManifest` runs `kubectl apply --server-side` inside an `alpine/kubectl` container. No host-side `kubectl` binary is required. Kustomize overlays are auto-detected. ```csharp @@ -152,7 +152,7 @@ The health check waits for all nodes to reach `Ready` before the cluster is mark ## Image overrides -The `alpine/helm` and `rancher/kubectl` images are pinned but configurable: +The `alpine/helm` and `alpine/kubectl` images are pinned but configurable: ```csharp builder.AddK3sCluster("k8s", configure: opts => @@ -160,7 +160,7 @@ builder.AddK3sCluster("k8s", configure: opts => opts.HelmImage = "my-registry/helm"; opts.HelmTag = "3.18.0"; opts.KubectlImage = "my-registry/k8s"; - opts.KubectlTag = "1.33.0"; + opts.KubectlTag = "1.36.0"; }); ``` diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs index 3861c4c16..fb41ac6c4 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -20,7 +20,7 @@ namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; /// macOS / Windows: Docker Desktop (containers run inside WSL2 / Hyper-V VM). /// /// No host-side helm or kubectl is needed — both run as Docker containers -/// (alpine/helm and alpine/k8s). +/// (alpine/helm and alpine/kubectl). /// Tests are gated by [RequiresDocker] and run on ubuntu-latest only — /// GitHub-hosted Windows runners do not support privileged Linux containers reliably. /// @@ -51,8 +51,8 @@ public async ValueTask DisposeAsync() [Fact] public async Task ClusterReachesRunningAndKubeconfigIsValid() { - var cluster = _builder!.AddK3sCluster("k8s"); - _app = _builder.Build(); + _builder!.AddK3sCluster("k8s"); + _app = _builder!.Build(); await _app.StartAsync(); @@ -63,14 +63,14 @@ public async Task ClusterReachesRunningAndKubeconfigIsValid() // local/kubeconfig.yaml must exist on the host. var kubeconfigPath = Path.Combine( - _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); Assert.True(File.Exists(kubeconfigPath), $"Expected local kubeconfig at {kubeconfigPath}"); // container/kubeconfig.yaml must also exist. var containerKubeconfigPath = Path.Combine( - _builder.AppHostDirectory, ".k3s", "k8s", "container", "kubeconfig.yaml"); + _builder!.AppHostDirectory, ".k3s", "k8s", "container", "kubeconfig.yaml"); Assert.True(File.Exists(containerKubeconfigPath), $"Expected container kubeconfig at {containerKubeconfigPath}"); } @@ -87,7 +87,7 @@ public async Task HelmReleaseExitsSuccessfully() version: "18.3.6", @namespace: "nginx"); - _app = _builder.Build(); + _app = _builder!.Build(); await _app.StartAsync(); var rns = _app.Services.GetRequiredService(); @@ -116,7 +116,7 @@ public async Task ServiceEndpointExposesHttpPort() cluster.AddServiceEndpoint("nginx-web", "nginx", servicePort: 80, @namespace: "nginx") .WaitForCompletion(nginx); - _app = _builder.Build(); + _app = _builder!.Build(); await _app.StartAsync(); var rns = _app.Services.GetRequiredService(); @@ -141,9 +141,9 @@ await rns.WaitForResourceAsync("nginx", [Fact] public async Task WithReferenceInjectsKubeconfigForProject() { - var cluster = _builder!.AddK3sCluster("k8s"); + _builder!.AddK3sCluster("k8s"); - _app = _builder.Build(); + _app = _builder!.Build(); await _app.StartAsync(); var rns = _app.Services.GetRequiredService(); @@ -152,7 +152,7 @@ public async Task WithReferenceInjectsKubeconfigForProject() await rns.WaitForResourceHealthyAsync("k8s", cts.Token); var kubeconfigPath = Path.Combine( - _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); var yaml = await File.ReadAllTextAsync(kubeconfigPath, cts.Token); Assert.Contains("localhost", yaml); @@ -197,10 +197,10 @@ public async Task ManifestAppliesCrdAndReachesEstablished() try { - var cluster = _builder.AddK3sCluster("k8s"); - var crd = cluster.AddK8sManifest("widget-crd", crdPath); + var cluster = _builder!.AddK3sCluster("k8s"); + cluster.AddK8sManifest("widget-crd", crdPath); - _app = _builder.Build(); + _app = _builder!.Build(); await _app.StartAsync(); var rns = _app.Services.GetRequiredService(); @@ -214,7 +214,7 @@ await rns.WaitForResourceAsync("widget-crd", // Independently verify the CRD is Established via KubernetesClient. var kubeconfigPath = Path.Combine( - _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); using var k8sClient = new Kubernetes(config); @@ -241,7 +241,7 @@ public async Task WithDataVolumePreservesStateAcrossRestarts() // ── First run ───────────────────────────────────────────────────── _builder!.AddK3sCluster("k8s").WithDataVolume(volumeName); - _app = _builder.Build(); + _app = _builder!.Build(); await _app.StartAsync(); var rns1 = _app.Services.GetRequiredService(); @@ -249,7 +249,7 @@ public async Task WithDataVolumePreservesStateAcrossRestarts() await rns1.WaitForResourceHealthyAsync("k8s", cts1.Token); var kubeconfigPath = Path.Combine( - _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); using (var k8sClient = new Kubernetes( KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath))) From e33ae5f732fe489da38adfcbd83caba62305f7ef Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Wed, 20 May 2026 15:17:18 +0200 Subject: [PATCH 15/29] chore: excluding the integration test since take too long to run --- .github/workflows/tests.yaml | 6 - CommunityToolkit.Aspire.slnx | 1 - ...Aspire.Hosting.K3s.IntegrationTests.csproj | 8 - .../K3sIntegrationTests.cs | 315 ------------------ 4 files changed, 330 deletions(-) delete mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj delete mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a08ff774c..ecca49a96 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -38,7 +38,6 @@ jobs: Hosting.JavaScript.Extensions.Tests, Hosting.Java.Tests, Hosting.K3s.Tests, - Hosting.K3s.IntegrationTests, Hosting.k6.Tests, Hosting.Keycloak.Extensions.Tests, Hosting.KurrentDB.Tests, @@ -83,11 +82,6 @@ jobs: Sftp.Tests, SurrealDb.Tests, ] - exclude: - # k3s integration tests require privileged Linux containers. - # GitHub-hosted Windows runners do not support this reliably. - - os: windows-latest - name: Hosting.K3s.IntegrationTests steps: - name: Checkout code diff --git a/CommunityToolkit.Aspire.slnx b/CommunityToolkit.Aspire.slnx index 2d71685f9..3382bdbb0 100644 --- a/CommunityToolkit.Aspire.slnx +++ b/CommunityToolkit.Aspire.slnx @@ -281,7 +281,6 @@ - diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj deleted file mode 100644 index 95df55b8f..000000000 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs deleted file mode 100644 index fb41ac6c4..000000000 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ /dev/null @@ -1,315 +0,0 @@ -using Aspire.Components.Common.Tests; -using Aspire.Hosting; -using Aspire.Hosting.ApplicationModel; -using Aspire.Hosting.Utils; -using CommunityToolkit.Aspire.Testing; -using k8s; -using k8s.Models; -using Microsoft.Extensions.DependencyInjection; -using Xunit; - -namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; - -/// -/// End-to-end integration tests that spin up a real k3s cluster inside Docker. -/// -/// Requirements: -/// -/// A container runtime that supports privileged Linux containers. -/// Linux: Docker Engine 20.10+ or rootful Podman 4.0+. -/// macOS / Windows: Docker Desktop (containers run inside WSL2 / Hyper-V VM). -/// -/// No host-side helm or kubectl is needed — both run as Docker containers -/// (alpine/helm and alpine/kubectl). -/// Tests are gated by [RequiresDocker] and run on ubuntu-latest only — -/// GitHub-hosted Windows runners do not support privileged Linux containers reliably. -/// -/// -[RequiresDocker] -public class K3sIntegrationTests : IAsyncLifetime -{ - private DistributedApplication? _app; - private IDistributedApplicationTestingBuilder? _builder; - - public async ValueTask InitializeAsync() - { - _builder = TestDistributedApplicationBuilder.Create(); - await ValueTask.CompletedTask; - } - - public async ValueTask DisposeAsync() - { - if (_app is not null) - { - await _app.StopAsync(); - await _app.DisposeAsync(); - } - - _builder?.Dispose(); - } - - [Fact] - public async Task ClusterReachesRunningAndKubeconfigIsValid() - { - _builder!.AddK3sCluster("k8s"); - _app = _builder!.Build(); - - await _app.StartAsync(); - - var rns = _app.Services.GetRequiredService(); - using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - - await rns.WaitForResourceHealthyAsync("k8s", cts.Token); - - // local/kubeconfig.yaml must exist on the host. - var kubeconfigPath = Path.Combine( - _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); - - Assert.True(File.Exists(kubeconfigPath), - $"Expected local kubeconfig at {kubeconfigPath}"); - - // container/kubeconfig.yaml must also exist. - var containerKubeconfigPath = Path.Combine( - _builder!.AppHostDirectory, ".k3s", "k8s", "container", "kubeconfig.yaml"); - Assert.True(File.Exists(containerKubeconfigPath), - $"Expected container kubeconfig at {containerKubeconfigPath}"); - } - - [Fact] - public async Task HelmReleaseExitsSuccessfully() - { - var cluster = _builder!.AddK3sCluster("k8s"); - - cluster.AddHelmRelease( - name: "nginx", - chart: "nginx", - repo: "https://charts.bitnami.com/bitnami", - version: "18.3.6", - @namespace: "nginx"); - - _app = _builder!.Build(); - await _app.StartAsync(); - - var rns = _app.Services.GetRequiredService(); - using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); - - await rns.WaitForResourceHealthyAsync("k8s", cts.Token); - - // HelmReleaseResource is a run-to-completion container — it exits with code 0 - // when helm upgrade --install --wait completes successfully. - await rns.WaitForResourceAsync("nginx", - s => s.Snapshot.State?.Text == "Exited", cts.Token); - } - - [Fact] - public async Task ServiceEndpointExposesHttpPort() - { - var cluster = _builder!.AddK3sCluster("k8s"); - - var nginx = cluster.AddHelmRelease( - name: "nginx", - chart: "nginx", - repo: "https://charts.bitnami.com/bitnami", - version: "18.3.6", - @namespace: "nginx"); - - cluster.AddServiceEndpoint("nginx-web", "nginx", servicePort: 80, @namespace: "nginx") - .WaitForCompletion(nginx); - - _app = _builder!.Build(); - await _app.StartAsync(); - - var rns = _app.Services.GetRequiredService(); - using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); - - await rns.WaitForResourceHealthyAsync("k8s", cts.Token); - await rns.WaitForResourceAsync("nginx", - s => s.Snapshot.State?.Text == "Exited", cts.Token); - await rns.WaitForResourceHealthyAsync("nginx-web", cts.Token); - - var model = _app.Services.GetRequiredService(); - var ep = model.Resources.OfType().Single(); - - Assert.True(ep.HostPort > 0, "HostPort should be allocated"); - - using var http = new HttpClient(); - var response = await http.GetAsync($"http://localhost:{ep.HostPort}", cts.Token); - Assert.True(response.IsSuccessStatusCode, - $"Expected HTTP 200 from nginx at localhost:{ep.HostPort}, got {response.StatusCode}"); - } - - [Fact] - public async Task WithReferenceInjectsKubeconfigForProject() - { - _builder!.AddK3sCluster("k8s"); - - _app = _builder!.Build(); - await _app.StartAsync(); - - var rns = _app.Services.GetRequiredService(); - using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - - await rns.WaitForResourceHealthyAsync("k8s", cts.Token); - - var kubeconfigPath = Path.Combine( - _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); - - var yaml = await File.ReadAllTextAsync(kubeconfigPath, cts.Token); - Assert.Contains("localhost", yaml); - Assert.DoesNotContain("127.0.0.1:6443", yaml); - } - - [Fact] - public async Task ManifestAppliesCrdAndReachesEstablished() - { - // Write a minimal CRD manifest to a temp file so AddK8sManifest can find it. - var crdYaml = """ - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - name: widgets.example.com - spec: - group: example.com - scope: Namespaced - names: - plural: widgets - singular: widget - kind: Widget - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - color: - type: string - """; - - var manifestDir = Path.Combine(_builder!.AppHostDirectory, "k8s-test-crds"); - Directory.CreateDirectory(manifestDir); - var crdPath = Path.Combine(manifestDir, "widget-crd.yaml"); - await File.WriteAllTextAsync(crdPath, crdYaml); - - try - { - var cluster = _builder!.AddK3sCluster("k8s"); - cluster.AddK8sManifest("widget-crd", crdPath); - - _app = _builder!.Build(); - await _app.StartAsync(); - - var rns = _app.Services.GetRequiredService(); - using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - - await rns.WaitForResourceHealthyAsync("k8s", cts.Token); - - // K8sManifestResource is run-to-completion — exits 0 after CRD reaches Established. - await rns.WaitForResourceAsync("widget-crd", - s => s.Snapshot.State?.Text == "Exited", cts.Token); - - // Independently verify the CRD is Established via KubernetesClient. - var kubeconfigPath = Path.Combine( - _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); - using var k8sClient = new Kubernetes(config); - - var widgetCrd = await k8sClient.ApiextensionsV1 - .ReadCustomResourceDefinitionAsync("widgets.example.com", cancellationToken: cts.Token); - - var established = widgetCrd.Status?.Conditions?.Any(c => - c.Type == "Established" && - string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true; - - Assert.True(established, "CRD 'widgets.example.com' should be Established"); - } - finally - { - Directory.Delete(manifestDir, recursive: true); - } - } - - [Fact] - public async Task WithDataVolumePreservesStateAcrossRestarts() - { - // Use an explicit volume name shared between both app instances. - var volumeName = $"aspire-k3s-persist-{Guid.NewGuid():N}"; - - // ── First run ───────────────────────────────────────────────────── - _builder!.AddK3sCluster("k8s").WithDataVolume(volumeName); - _app = _builder!.Build(); - await _app.StartAsync(); - - var rns1 = _app.Services.GetRequiredService(); - using var cts1 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - await rns1.WaitForResourceHealthyAsync("k8s", cts1.Token); - - var kubeconfigPath = Path.Combine( - _builder!.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); - - using (var k8sClient = new Kubernetes( - KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath))) - { - await k8sClient.CoreV1.CreateNamespacedConfigMapAsync( - new V1ConfigMap - { - Metadata = new V1ObjectMeta { Name = "persist-check" }, - Data = new Dictionary { ["run"] = "first" }, - }, - "default", - cancellationToken: cts1.Token); - } - - // Stop the first app. The volume is retained; the container is removed by DCP. - await _app.StopAsync(); - await _app.DisposeAsync(); - _app = null; - - // ── Second run with the same named volume ───────────────────────── - using var builder2 = TestDistributedApplicationBuilder.Create(); - builder2.AddK3sCluster("k8s").WithDataVolume(volumeName); - - await using var app2 = builder2.Build(); - try - { - await app2.StartAsync(); - - var rns2 = app2.Services.GetRequiredService(); - using var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - await rns2.WaitForResourceHealthyAsync("k8s", cts2.Token); - - var kubeconfigPath2 = Path.Combine( - builder2.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); - - using var k8sClient2 = new Kubernetes( - KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath2)); - - var cm = await k8sClient2.CoreV1.ReadNamespacedConfigMapAsync( - "persist-check", "default", cancellationToken: cts2.Token); - - Assert.Equal("first", cm.Data["run"]); - } - finally - { - await app2.StopAsync(); - - // Remove the named volume so it does not accumulate on CI runners. - try - { - using var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo("docker") - { - ArgumentList = { "volume", "rm", "--force", volumeName }, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - }); - if (process is not null) await process.WaitForExitAsync(); - } - catch { /* best effort */ } - } - } -} From 62e9018bb47cfa8caed22b3b5fc343d7f943abed Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Wed, 20 May 2026 17:47:44 +0200 Subject: [PATCH 16/29] =?UTF-8?q?fix:=20address=20code=20review=20findings?= =?UTF-8?q?=20=E2=80=94=20lifecycle,=20idempotency=20and=20mount=20isolati?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Resolve .gitignore merge conflict markers (keep both sides) - Register K3sReadinessHealthCheck as a singleton to prevent per-tick client leaks - Write kubeconfig variants atomically via temp-file-then-rename - Fix stale credential recovery to not delete the raw kubeconfig k3s will not rewrite - Fix AllocatePort to probe on IPAddress.Any and re-allocate a fresh port on retry - Make WithDataVolume and WithReference(cluster) idempotent against duplicate calls - Apply HelmEscape to --set keys as well as values - Mark kubeconfig bind-mounts as read-only - Throw InvalidOperationException on no-selector services instead of silently marking ready - Add whitespace validation on serviceName/namespace in AddServiceEndpoint - Remove WithHttpsDeveloperCertificate from k3s container (no-op for k3s) - Extract K3sFileHelpers constant for 0755 script mode and /tmp/k3s-kubeconfig.yaml path - Switch kubeconfig container mount from directory to file-level to isolate kubectl cache --- .gitignore | 3 - .../K3sBuilderExtensions.Helm.cs | 19 +++-- .../K3sBuilderExtensions.Manifest.cs | 13 ++-- .../K3sBuilderExtensions.ServiceEndpoint.cs | 8 +- .../K3sBuilderExtensions.cs | 69 ++++++++++------- .../K3sFileHelpers.cs | 77 +++++++++++++++++++ .../K3sInProcessPortForwarder.cs | 11 ++- .../K3sReadinessHealthCheck.cs | 16 +++- .../K3sClusterResourceTests.cs | 14 ++-- 9 files changed, 177 insertions(+), 53 deletions(-) create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sFileHelpers.cs diff --git a/.gitignore b/.gitignore index c914256dc..eac85d46a 100644 --- a/.gitignore +++ b/.gitignore @@ -23,10 +23,8 @@ examples/perl/**/local/* **cpanfile.snapshot **/.modules/ **/*.AppHost.TypeScript/nuget.config -<<<<<<< main **/.k3s/ -======= tsconfig.apphost.json .ngrok bun.lock @@ -35,4 +33,3 @@ yarn.lock solr-data *.lscache apphost.js ->>>>>>> main diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index d0d159bfc..6f00e62f8 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -55,8 +55,10 @@ public static IResourceBuilder AddHelmRelease( // the k3s API via DCP DNS (https://{clusterName}:6443). The directory is created // by AddK3sCluster; the kubeconfig file is written by K3sReadinessHealthCheck on // first successful health check. WaitFor(cluster) guarantees the file exists. + // Ensure the host-side container/ directory exists so the health check can write to it. var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); Directory.CreateDirectory(containerKubeconfigDir); + var containerKubeconfigFile = Path.Combine(containerKubeconfigDir, "kubeconfig.yaml"); var (helmRegistry, helmImage, helmTag) = cluster.HelmImageInfo; @@ -76,9 +78,7 @@ public static IResourceBuilder AddHelmRelease( { Name = "helm-install.sh", Contents = script, - Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute - | UnixFileMode.GroupRead | UnixFileMode.GroupExecute - | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + Mode = K3sFileHelpers.ExecutableScriptMode, }]; return Task.FromResult(items); }) @@ -96,8 +96,11 @@ public static IResourceBuilder AddHelmRelease( })]; return Task.FromResult(items); }) - .WithBindMount(containerKubeconfigDir, "/root/.kube") - .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + // File-level mount: only the kubeconfig YAML is visible inside the container. + // Mounting the full container/ directory would expose it to kubectl's cache + // (cache/, http-cache/) and cause concurrent-container cache corruption. + .WithBindMount(containerKubeconfigFile, K3sFileHelpers.ContainerKubeconfigPath, isReadOnly: true) + .WithEnvironment("KUBECONFIG", K3sFileHelpers.ContainerKubeconfigPath) .WithIconName("Rocket") .ExcludeFromManifest() .WithInitialState(new CustomResourceSnapshot @@ -171,7 +174,7 @@ internal static string BuildHelmScript(HelmReleaseResource release) // can appear in the bind-mount before the k8s hostname resolves in the helm // container. Using `helm list` (which calls the k8s API) verifies both the // file and the network path before proceeding. - sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ] && helm list --kubeconfig /root/.kube/kubeconfig.yaml > /dev/null 2>&1; do"); + sb.AppendLine($"until [ -f {K3sFileHelpers.ContainerKubeconfigPath} ] && helm list --kubeconfig {K3sFileHelpers.ContainerKubeconfigPath} > /dev/null 2>&1; do"); sb.AppendLine(" echo 'Waiting for k3s cluster to be ready and reachable...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); @@ -211,8 +214,10 @@ internal static string BuildHelmScript(HelmReleaseResource release) // to Helm without any shell interpretation. // For values containing commas, braces, or backslashes that Helm --set cannot // represent safely (e.g. multi-line strings), use WithHelmValuesFile instead. + // Apply HelmEscape to BOTH key and value: Helm's --set parser splits on commas + // and treats braces/brackets as list/map syntax in both positions. foreach (var (key, value) in release.HelmValues) - sb.Append($" --set {ShellEscape($"{key}={HelmEscape(value)}")}"); + sb.Append($" --set {ShellEscape($"{HelmEscape(key)}={HelmEscape(value)}")}"); return sb.ToString(); } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index e4d3863de..216de6624 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -55,6 +55,7 @@ public static IResourceBuilder AddK8sManifest( var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); Directory.CreateDirectory(containerKubeconfigDir); + var containerKubeconfigFile = Path.Combine(containerKubeconfigDir, "kubeconfig.yaml"); var (kubectlRegistry, kubectlImage, kubectlTag) = cluster.KubectlImageInfo; @@ -71,20 +72,20 @@ public static IResourceBuilder AddK8sManifest( { Name = "kubectl-apply.sh", Contents = BuildManifestScript(), - Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute - | UnixFileMode.GroupRead | UnixFileMode.GroupExecute - | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + Mode = K3sFileHelpers.ExecutableScriptMode, }]; return Task.FromResult(items); }) .WithArgs("/kubectl-apply.sh") - .WithBindMount(containerKubeconfigDir, "/root/.kube"); + .WithBindMount(containerKubeconfigFile, K3sFileHelpers.ContainerKubeconfigPath, isReadOnly: true); if (isKustomize) { // Bind-mount the overlay directory so kubectl kustomize can resolve relative // references to base manifests (e.g. ../../base). WithContainerFiles copies // files, not directory structure, so it would break cross-directory references. + // Read-write is intentional here: kustomize traverses into subdirectories. + // The kubectl/helm scripts never write to /k8s-manifests, so this is latent. resourceBuilder.WithBindMount(absolutePath, "/k8s-manifests"); } else @@ -125,7 +126,7 @@ public static IResourceBuilder AddK8sManifest( } return resourceBuilder - .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + .WithEnvironment("KUBECONFIG", K3sFileHelpers.ContainerKubeconfigPath) .WithIconName("Code") .ExcludeFromManifest() .WithInitialState(new CustomResourceSnapshot @@ -150,7 +151,7 @@ internal static string BuildManifestScript() // DCP sets up container network aliases asynchronously, so the kubeconfig file // can appear in the bind-mount before the k8s hostname resolves in the kubectl // container. Using `kubectl cluster-info` verifies both the file and the network. - sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ] && kubectl cluster-info --kubeconfig /root/.kube/kubeconfig.yaml > /dev/null 2>&1; do"); + sb.AppendLine($"until [ -f {K3sFileHelpers.ContainerKubeconfigPath} ] && kubectl cluster-info --kubeconfig {K3sFileHelpers.ContainerKubeconfigPath} > /dev/null 2>&1; do"); sb.AppendLine(" echo 'Waiting for k3s cluster to be ready and reachable...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs index f84c99e00..baa8e88c3 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -39,7 +39,8 @@ public static IResourceBuilder AddServiceEndpoint( { ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(name); - ArgumentNullException.ThrowIfNull(serviceName); + ArgumentException.ThrowIfNullOrWhiteSpace(serviceName); + ArgumentException.ThrowIfNullOrWhiteSpace(@namespace); if (servicePort is < 1 or > 65535) throw new ArgumentOutOfRangeException(nameof(servicePort), @@ -199,7 +200,10 @@ private static ImmutableArray BuildUrls( private static int AllocatePort() { - using var listener = new TcpListener(IPAddress.Loopback, 0); + // Probe on IPAddress.Any to match the forwarder's actual bind address. + // Probing on Loopback while the forwarder binds on Any can miss conflicts + // on non-loopback interfaces, producing a SocketException at bind time. + using var listener = new TcpListener(IPAddress.Any, 0); listener.Start(); var port = ((IPEndPoint)listener.LocalEndpoint).Port; listener.Stop(); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index ac1f88c5f..5f3927b4f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -68,9 +68,7 @@ public static IResourceBuilder AddK3sCluster( { Name = "aspire-k3s-entrypoint.sh", Contents = K3sInitEntrypointScript, - Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute - | UnixFileMode.GroupRead | UnixFileMode.GroupExecute - | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + Mode = K3sFileHelpers.ExecutableScriptMode, }]) .WithEntrypoint("/bin/sh") .WithArgs("/aspire-k3s-entrypoint.sh") @@ -137,8 +135,7 @@ public static IResourceBuilder AddK3sCluster( .WithEnvironment("K3S_KUBECONFIG_MODE", "644") .WithEnvironment("K3S_KUBECONFIG_OUTPUT", "/tmp/k3s-kubeconfig/kubeconfig.yaml") - .WithIconName("Kubernetes") - .WithHttpsDeveloperCertificate(); + .WithIconName("Kubernetes"); if (options.ClusterCidr is not null) { @@ -208,9 +205,14 @@ public static IResourceBuilder AddK3sCluster( resourceBuilder.WithHealthCheck($"k3s_{name}_ready"); + // Register as a singleton instance so the cached Kubernetes client and kubeconfig + // state survive across health-check ticks. Using a factory (sp => new ...) would + // create a fresh instance on every check, making _cachedClient dead state and + // leaking a Kubernetes/HttpClient on every tick. + var healthCheck = new K3sReadinessHealthCheck(resource, resource.ApiEndpoint); builder.Services.AddHealthChecks().Add(new HealthCheckRegistration( $"k3s_{name}_ready", - sp => new K3sReadinessHealthCheck(resource, resource.ApiEndpoint), + _ => healthCheck, failureStatus: HealthStatus.Unhealthy, tags: null)); @@ -330,6 +332,14 @@ public static IResourceBuilder WithDataVolume( { ArgumentNullException.ThrowIfNull(builder); + // Idempotent: remove any existing data-volume annotation at this target so that + // calling WithDataVolume twice does not produce duplicate mounts that Docker rejects. + var existing = builder.Resource.Annotations + .OfType() + .FirstOrDefault(m => m.Target == "/var/lib/rancher/k3s"); + if (existing is not null) + builder.Resource.Annotations.Remove(existing); + return builder .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s"); } @@ -340,8 +350,8 @@ public static IResourceBuilder WithDataVolume( /// /// /// s receive a physical kubeconfig file copied to - /// /var/k3s/kubeconfig.yaml (container-network variant, - /// server: https://{resourceName}:6443). KUBECONFIG=/var/k3s/kubeconfig.yaml + /// /tmp/k3s-kubeconfig.yaml (container-network variant, + /// server: https://{resourceName}:6443). KUBECONFIG=/tmp/k3s-kubeconfig.yaml /// is set automatically so all standard Kubernetes tooling (kubectl, helm, /// KubernetesClient SDK) works without any custom bootstrap code. /// @@ -367,24 +377,31 @@ public static IResourceBuilder WithReference( if (destination.Resource is ContainerResource) { - // Containers get a bind-mount of the container/ kubeconfig directory at /var/k3s. - // ContainerMountAnnotation is added directly to bypass the T : ContainerResource - // constraint on WithBindMount — the annotation is equivalent. - // - // Bind-mount (not file copy) is used for the same reason as in the helm and - // kubectl installer containers: if the cluster is recreated while a container is - // running, the new kubeconfig appears automatically without restarting the container. - var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); - Directory.CreateDirectory(containerKubeconfigDir); - - destination.Resource.Annotations.Add( - new ContainerMountAnnotation( - containerKubeconfigDir, - "/var/k3s", - ContainerMountType.BindMount, - isReadOnly: true)); - - return destination.WithEnvironment("KUBECONFIG", "/var/k3s/kubeconfig.yaml"); + // Idempotent: skip if the file bind-mount was already added (e.g. by a second + // WithReference call) — Docker rejects duplicate mounts at the same target. + var alreadyMounted = destination.Resource.Annotations + .OfType() + .Any(m => m.Target == K3sFileHelpers.ContainerKubeconfigPath); + + if (!alreadyMounted) + { + // File-level mount: only the kubeconfig YAML is visible inside the container. + // Mounting the entire container/ directory would expose kubectl's cache + // directories (cache/, http-cache/) on the host and cause concurrent-container + // cache corruption when multiple containers share the same kubeconfig directory. + var containerKubeconfigFile = Path.Combine( + cluster.KubeconfigDirectory!, "container", "kubeconfig.yaml"); + Directory.CreateDirectory(Path.GetDirectoryName(containerKubeconfigFile)!); + + destination.Resource.Annotations.Add( + new ContainerMountAnnotation( + containerKubeconfigFile, + K3sFileHelpers.ContainerKubeconfigPath, + ContainerMountType.BindMount, + isReadOnly: true)); + } + + return destination.WithEnvironment("KUBECONFIG", K3sFileHelpers.ContainerKubeconfigPath); } // Projects and executables: KUBECONFIG points to the host-accessible local kubeconfig. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sFileHelpers.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sFileHelpers.cs new file mode 100644 index 000000000..03a8123ab --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sFileHelpers.cs @@ -0,0 +1,77 @@ +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Shared constants for Unix file modes and well-known in-container paths used by +/// k3s-related containers. +/// +internal static class K3sFileHelpers +{ + /// + /// The path inside every container that receives a k3s kubeconfig via a file-level + /// bind-mount from the host's AppHostDirectory/.k3s/{name}/container/kubeconfig.yaml. + /// + /// + /// + /// Placing the file under /tmp/ (which is guaranteed to exist in all POSIX + /// containers) rather than inside ~/.kube/ ensures that kubectl's + /// cache directories (~/.kube/cache/, ~/.kube/http-cache/) are created + /// inside the container's ephemeral filesystem rather than in the host-side + /// container/ directory. This prevents: + /// + /// Host filesystem pollution with kubectl cache directories. + /// Cache corruption when multiple helm or kubectl containers run concurrently + /// and share the same host-side mount directory. + /// + /// + /// + /// Using a file-level bind-mount (source file → target file, not directory → + /// directory) means only the kubeconfig YAML is visible inside the container at this + /// path; any other files the host may later add to container/ are not exposed. + /// + /// + internal const string ContainerKubeconfigPath = "/tmp/k3s-kubeconfig.yaml"; + + /// + /// Unix file mode 0755 (rwxr-xr-x) for shell scripts injected into + /// k3s, alpine/helm, and alpine/kubectl containers. + /// + /// + /// + /// Why 0755 and not a narrower mode like 0700? + /// + /// + /// The scripts are invoked as /bin/sh /script.sh — the shell reads the file, + /// so the execute bit on the script itself is technically not required for the current + /// invocation pattern. We set it anyway for two reasons: + /// + /// + /// Convention: scripts intended to be executed are conventionally marked + /// executable. If the invocation is later changed to direct execution + /// (./script.sh), the permission is already correct without a + /// chmod inside the container. + /// + /// + /// Custom image overrides: K3sClusterOptions lets callers replace + /// the helm/kubectl images. Non-root container users in those custom images need + /// OtherRead | OtherExecute to read and execute the injected scripts + /// when the file is owned by a different UID/GID. The default images + /// (alpine/helm, alpine/kubectl, rancher/k3s) all run as + /// root (UID 0), so UserRead | UserExecute alone would suffice for + /// the defaults — but 0755 is the safe choice for the general case. + /// + /// + /// + /// + /// Why UserWrite? + /// WithContainerFiles calls docker cp which sets the destination owner + /// to root. Without UserWrite, root cannot overwrite the file on a subsequent + /// AppHost restart (when the same container image layer is re-used), leading to a + /// silent stale-script failure. This is the same reason chmod 644 rather than + /// 444 is conventional for data files on Linux. + /// + /// + internal const UnixFileMode ExecutableScriptMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index b00dc9141..95f8a25f4 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -32,6 +32,9 @@ internal sealed class K3sInProcessPortForwarder( public async Task RunAsync(ILogger logger, CancellationToken ct) { var backoff = TimeSpan.FromSeconds(2); + // Track current port separately so a failed bind can re-allocate rather than + // retrying the already-stolen port — avoids the TOCTOU retry loop. + var currentPort = localPort; while (!ct.IsCancellationRequested) { @@ -40,7 +43,7 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) // Docker host IP — not 127.0.0.1. Binding to loopback would silently drop // all container traffic. Users on shared networks should be aware that the // forwarded service is reachable from other hosts on the same LAN. - var listener = new TcpListener(IPAddress.Any, localPort); + var listener = new TcpListener(IPAddress.Any, currentPort); try { listener.Start(); @@ -82,6 +85,12 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) "Port-forward for svc/{Service} failed; retrying in {Delay}s…", serviceName, backoff.TotalSeconds); onReadyChanged(false); + // Allocate a fresh port on retry — the previous one may have been + // stolen between our probe and the forwarder bind (TOCTOU). + using var probe = new TcpListener(IPAddress.Any, 0); + probe.Start(); + currentPort = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); } finally { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index 22851041f..68ae4a501 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -102,12 +102,12 @@ private async Task EnsureClientAsync(string rawPath, CancellationTok var localDir = Path.Combine(dir, "local"); Directory.CreateDirectory(localDir); var localPath = Path.Combine(localDir, "kubeconfig.yaml"); - await File.WriteAllTextAsync(localPath, BuildConfigYaml(parsed, $"https://localhost:{port}"), ct) + await WriteAtomicAsync(localPath, BuildConfigYaml(parsed, $"https://localhost:{port}"), ct) .ConfigureAwait(false); var containerDir = Path.Combine(dir, "container"); Directory.CreateDirectory(containerDir); - await File.WriteAllTextAsync( + await WriteAtomicAsync( Path.Combine(containerDir, "kubeconfig.yaml"), BuildConfigYaml(parsed, $"https://{_resource.Name}:6443"), ct).ConfigureAwait(false); @@ -130,6 +130,18 @@ private static string BuildConfigYaml(K8SConfiguration source, string serverUrl) return KubernetesYaml.Serialize(copy); } + /// + /// Writes to atomically by first + /// writing to a sibling temp file and then renaming. Readers can never observe a + /// partial write; the old file remains readable until the rename commits. + /// + private static async Task WriteAtomicAsync(string path, string content, CancellationToken ct) + { + var tmp = path + ".tmp"; + await File.WriteAllTextAsync(tmp, content, ct).ConfigureAwait(false); + File.Move(tmp, path, overwrite: true); + } + private static bool IsTlsOrAuthFailure(Exception ex) => ex is System.Security.Authentication.AuthenticationException || ex.InnerException is System.Security.Authentication.AuthenticationException diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 2e8b98d5b..5b2d194ef 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -203,7 +203,7 @@ public void WithReferenceSetsKubeconfigEnvForProject() } [Fact] - public void WithReferenceMountsKubeconfigDirForContainer() + public void WithReferenceMountsKubeconfigFileForContainer() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); @@ -217,17 +217,19 @@ public void WithReferenceMountsKubeconfigDirForContainer() .OfType() .Single(r => r.Name == "operator"); - // All containers (user containers, helm, and kubectl installers) receive a bind-mount - // of the container/ kubeconfig directory. Bind-mount is used so the kubeconfig - // updates automatically if the cluster is recreated without restarting the container. + // Containers receive a file-level bind-mount of container/kubeconfig.yaml at + // /tmp/k3s-kubeconfig.yaml (not the directory). Mounting only the file prevents + // kubectl's cache directories (cache/, http-cache/) from appearing on the host + // and avoids concurrent-container cache corruption. var mount = containerResource.Annotations .OfType() - .FirstOrDefault(m => m.Target == "/var/k3s"); + .FirstOrDefault(m => m.Target == "/tmp/k3s-kubeconfig.yaml"); Assert.NotNull(mount); Assert.Equal(ContainerMountType.BindMount, mount.Type); Assert.True(mount.IsReadOnly); - Assert.EndsWith(Path.Combine(".k3s", "k8s", "container"), mount.Source); + // Source is the specific file, not the directory. + Assert.EndsWith(Path.Combine(".k3s", "k8s", "container", "kubeconfig.yaml"), mount.Source); } [Fact] From 5d913d6cb54f183d8b2dc11464e6fa257101cd15 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Wed, 20 May 2026 18:20:39 +0200 Subject: [PATCH 17/29] fix: proper lifecycle management for health check and port forwarder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Health check (K3sReadinessHealthCheck): - Drop _cachedClient — registration was already fixed to singleton so the cache was live, but a fresh Kubernetes client per check is simpler, removes all stale-client invalidation complexity, and the TLS overhead every 5 s is negligible for local dev - Extract EnsureKubeconfigVariantsAsync; stale detection now only deletes the derived local/ and container/ files — rawPath is never touched Port forwarder (K3sInProcessPortForwarder): - Implement IAsyncDisposable; own a CancellationTokenSource and TcpListener as fields so DisposeAsync can cancel and release the socket independently of the outer application-lifetime token - RunAsync links _cts.Token with the outer ct so either side can stop the loop - Per-connection Task.Run now uses the linked token instead of CancellationToken.None — connections can be cancelled on shutdown - On listener failure a fresh port is re-allocated for the retry Service endpoint wiring: - K3sServiceEndpointResource retains the Forwarder reference - RunEndpointAsync wraps forwarder in await using so DisposeAsync runs on any exit path - Subscribe to ResourceStoppedEvent on the cluster to dispose the forwarder immediately when the cluster container stops, releasing the host port without waiting for the application-lifetime token --- .../K3sBuilderExtensions.ServiceEndpoint.cs | 24 +++++- .../K3sInProcessPortForwarder.cs | 67 +++++++++++------ .../K3sReadinessHealthCheck.cs | 73 +++++++++---------- .../K3sServiceEndpointResource.cs | 9 +++ 4 files changed, 110 insertions(+), 63 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs index baa8e88c3..eec7c3302 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -64,6 +64,18 @@ public static IResourceBuilder AddServiceEndpoint( failureStatus: HealthStatus.Unhealthy, tags: null)); + // Dispose the forwarder when the parent cluster stops so the port is released + // promptly rather than waiting for the application-lifetime CT to fire. + // ResourceStoppedEvent fires after the cluster container has stopped — at that + // point port-forwarding is no longer useful regardless. + builder.ApplicationBuilder.Eventing.Subscribe( + cluster, + async (@event, ct) => + { + if (endpoint.Forwarder is { } forwarder) + await forwarder.DisposeAsync().ConfigureAwait(false); + }); + return builder.ApplicationBuilder .AddResource(endpoint) .ExcludeFromManifest() @@ -177,7 +189,17 @@ await notifications.PublishUpdateAsync(endpoint, }); }); - _ = Task.Run(() => forwarder.RunAsync(logger, ct), ct); + // Retain the forwarder so ResourceStoppedEvent on the cluster can dispose it + // independently of the application-lifetime cancellation token. + endpoint.Forwarder = forwarder; + + _ = Task.Run(async () => + { + await using (forwarder.ConfigureAwait(false)) + { + await forwarder.RunAsync(logger, ct).ConfigureAwait(false); + } + }, ct); } catch (Exception ex) when (!ct.IsCancellationRequested) { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 95f8a25f4..e3fd6f5ea 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -27,50 +27,54 @@ internal sealed class K3sInProcessPortForwarder( string serviceName, int localPort, int servicePort, - Action onReadyChanged) + Action onReadyChanged) : IAsyncDisposable { + private readonly CancellationTokenSource _cts = new(); + private TcpListener? _listener; + public async Task RunAsync(ILogger logger, CancellationToken ct) { + // Link the outer application-lifetime token with our own so either can stop the loop. + // _cts.Token lets DisposeAsync cancel independently of the outer token (e.g. when + // the cluster resource stops before the full AppHost shuts down). + using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct, _cts.Token); + var linkedCt = linked.Token; + var backoff = TimeSpan.FromSeconds(2); - // Track current port separately so a failed bind can re-allocate rather than - // retrying the already-stolen port — avoids the TOCTOU retry loop. var currentPort = localPort; - while (!ct.IsCancellationRequested) + while (!linkedCt.IsCancellationRequested) { - // IPAddress.Any (0.0.0.0) is required: DCP-network containers reach the - // forwarded service via host.docker.internal:{port}, which resolves to the - // Docker host IP — not 127.0.0.1. Binding to loopback would silently drop - // all container traffic. Users on shared networks should be aware that the - // forwarded service is reachable from other hosts on the same LAN. var listener = new TcpListener(IPAddress.Any, currentPort); + _listener = listener; try { listener.Start(); - logger.LogInformation( - "Port-forward: 0.0.0.0:{Local} → svc/{Service}.{Ns}:{Port}", - localPort, serviceName, @namespace, servicePort); + if (logger.IsEnabled(LogLevel.Information)) + logger.LogInformation( + "Port-forward: 0.0.0.0:{Local} → svc/{Service}.{Ns}:{Port}", + currentPort, serviceName, @namespace, servicePort); // Probe the service before signalling ready — the Kubernetes service and // a ready pod must exist before any connection can succeed. // This makes the ready signal meaningful for WaitFor(endpoint) consumers. - await WaitForServiceReadyAsync(logger, ct).ConfigureAwait(false); + await WaitForServiceReadyAsync(logger, linkedCt).ConfigureAwait(false); onReadyChanged(true); - while (!ct.IsCancellationRequested) + while (!linkedCt.IsCancellationRequested) { - var tcp = await listener.AcceptTcpClientAsync(ct).ConfigureAwait(false); + var tcp = await listener.AcceptTcpClientAsync(linkedCt).ConfigureAwait(false); _ = Task.Run( - () => ForwardConnectionAsync(tcp, logger, ct), - CancellationToken.None); + () => ForwardConnectionAsync(tcp, logger, linkedCt), + linkedCt); } } - catch (OperationCanceledException) when (ct.IsCancellationRequested) + catch (OperationCanceledException) when (linkedCt.IsCancellationRequested) { break; } - catch (InvalidOperationException ioe) when (!ct.IsCancellationRequested) + catch (InvalidOperationException ioe) when (!linkedCt.IsCancellationRequested) { // Non-retryable configuration error (e.g. service has no pod selector). // Log and stop — retrying will never succeed. @@ -94,14 +98,29 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) } finally { + _listener = null; listener.Stop(); } - if (ct.IsCancellationRequested) break; + if (linkedCt.IsCancellationRequested) break; - try { await Task.Delay(backoff, ct).ConfigureAwait(false); } catch (OperationCanceledException) { break; } + try { await Task.Delay(backoff, linkedCt).ConfigureAwait(false); } + catch (OperationCanceledException) { break; } backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 30)); } + + onReadyChanged(false); + } + + /// + public async ValueTask DisposeAsync() + { + // Cancel the linked CTS so RunAsync exits its loop even if the outer + // application-lifetime token is still live (e.g. cluster stopped but AppHost + // is still running). + await _cts.CancelAsync().ConfigureAwait(false); + _listener?.Stop(); + _cts.Dispose(); } /// @@ -149,7 +168,7 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct if (hasReadyPod) { logger.LogDebug( - "Service {Service}/{Ns} exposes requested port {ServicePort} and has a ready pod — port-forward is ready.", + "Service {Service}/{Ns} exposes port {Port} and has a ready pod — port-forward is ready.", serviceName, @namespace, servicePort); return; } @@ -159,6 +178,10 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct { return; } + catch (InvalidOperationException) + { + throw; // Non-retryable — let RunAsync catch it. + } catch (Exception ex) { logger.LogDebug(ex, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index 68ae4a501..a66ca4c0c 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -15,32 +15,22 @@ namespace CommunityToolkit.Aspire.Hosting; /// local/kubeconfig.yaml — server: https://localhost:{allocatedPort} (host processes) /// container/kubeconfig.yaml — server: https://{name}:6443 (DCP-network containers) /// -/// Then uses a cached client to call ListNodeAsync, -/// confirming that all expected nodes (server + agents) are in Ready state. +/// Then creates a short-lived client (disposed after each check) +/// to call ListNodeAsync, confirming that all expected nodes are Ready. /// No docker exec is involved — works with any container runtime. /// /// -internal sealed class K3sReadinessHealthCheck : IHealthCheck +internal sealed class K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) : IHealthCheck { - private readonly K3sClusterResource _resource; - private readonly EndpointReference _endpoint; - private Kubernetes? _cachedClient; - - internal K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) - { - _resource = resource; - _endpoint = endpoint; - } - /// public async Task CheckHealthAsync( HealthCheckContext context, CancellationToken cancellationToken = default) { - if (!_endpoint.IsAllocated) + if (!endpoint.IsAllocated) return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); - var dir = _resource.KubeconfigDirectory; + var dir = resource.KubeconfigDirectory; if (dir is null) return HealthCheckResult.Unhealthy("Kubeconfig directory not configured on resource"); @@ -50,9 +40,16 @@ public async Task CheckHealthAsync( try { - var client = await EnsureClientAsync(rawPath, cancellationToken).ConfigureAwait(false); + var localPath = await EnsureKubeconfigVariantsAsync(rawPath, dir, endpoint.Port, cancellationToken) + .ConfigureAwait(false); - var nodes = await client.CoreV1 + // Create a fresh client per check — no cached state, no stale connection risk. + // At a 5-second health-check interval the TLS handshake overhead is negligible + // for a local dev integration. + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(localPath); + using var k8sClient = new Kubernetes(config); + + var nodes = await k8sClient.CoreV1 .ListNodeAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -61,7 +58,7 @@ public async Task CheckHealthAsync( c.Type == "Ready" && string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true); - var expected = 1 + _resource.AgentCount; + var expected = 1 + resource.AgentCount; if (readyCount < expected) return HealthCheckResult.Unhealthy($"k3s cluster: {readyCount}/{expected} nodes Ready"); @@ -69,15 +66,12 @@ public async Task CheckHealthAsync( } catch (Exception ex) when (IsTlsOrAuthFailure(ex)) { - // Stale cached client — the cluster was recreated with new certs while the - // health check held an old IKubernetes instance. k3s has already written a fresh - // kubeconfig to rawPath (it writes once at startup, not continuously), so we must - // NOT delete rawPath — that would remove the fresh file and leave the health check - // waiting forever for a file that k3s will never rewrite. - // Instead, discard only the cached client and the derived variants so they are - // regenerated from the fresh raw file on the next check cycle. - _cachedClient?.Dispose(); - _cachedClient = null; + // Stale kubeconfig — cluster was recreated with new certs (e.g. data volume wiped). + // k3s has already written a fresh kubeconfig to rawPath (it writes once at startup, + // not continuously), so we must NOT delete rawPath — that would remove the fresh + // file and leave the health check waiting forever. + // Delete only the derived variants so they are regenerated from the fresh raw file + // on the next check cycle. TryDelete(Path.Combine(dir, "local", "kubeconfig.yaml")); TryDelete(Path.Combine(dir, "container", "kubeconfig.yaml")); return HealthCheckResult.Unhealthy("k3s kubeconfig is stale — retrying with fresh credentials"); @@ -88,14 +82,16 @@ public async Task CheckHealthAsync( } } - private async Task EnsureClientAsync(string rawPath, CancellationToken ct) + /// + /// Reads the raw kubeconfig written by k3s, rewrites it for each consumer variant, + /// and writes both to disk atomically. Returns the path to the local variant. + /// + private async Task EnsureKubeconfigVariantsAsync( + string rawPath, + string dir, + int port, + CancellationToken ct) { - if (_cachedClient is not null) - return _cachedClient; - - var port = _endpoint.Port; - var dir = _resource.KubeconfigDirectory!; - var rawYaml = await File.ReadAllTextAsync(rawPath, ct).ConfigureAwait(false); var parsed = KubernetesYaml.Deserialize(rawYaml); @@ -109,12 +105,10 @@ await WriteAtomicAsync(localPath, BuildConfigYaml(parsed, $"https://localhost:{p Directory.CreateDirectory(containerDir); await WriteAtomicAsync( Path.Combine(containerDir, "kubeconfig.yaml"), - BuildConfigYaml(parsed, $"https://{_resource.Name}:6443"), + BuildConfigYaml(parsed, $"https://{resource.Name}:6443"), ct).ConfigureAwait(false); - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(localPath); - _cachedClient = new Kubernetes(config); - return _cachedClient; + return localPath; } private static string BuildConfigYaml(K8SConfiguration source, string serverUrl) @@ -132,8 +126,7 @@ private static string BuildConfigYaml(K8SConfiguration source, string serverUrl) /// /// Writes to atomically by first - /// writing to a sibling temp file and then renaming. Readers can never observe a - /// partial write; the old file remains readable until the rename commits. + /// writing to a sibling temp file then renaming. Readers never observe a partial write. /// private static async Task WriteAtomicAsync(string path, string content, CancellationToken ct) { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs index d2a039de0..96d53b8f8 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -1,5 +1,7 @@ #pragma warning disable ASPIREATS001 // AspireExport is experimental +using CommunityToolkit.Aspire.Hosting; + namespace Aspire.Hosting.ApplicationModel; /// @@ -54,4 +56,11 @@ public sealed class K3sServiceEndpointResource( /// Set by K3sInProcessPortForwarder; read by the health check. /// internal volatile bool IsReady; + + /// + /// The active port-forwarder, retained so it can be disposed when the parent cluster + /// stops (via ResourceStoppedEvent) or when the AppHost shuts down — whichever + /// comes first. + /// + internal K3sInProcessPortForwarder? Forwarder { get; set; } } From 06a7b0fbf16e9e83d4293c448d39ff5819461be9 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Wed, 20 May 2026 19:14:57 +0200 Subject: [PATCH 18/29] fix: manifest apply exit code when no kustomize present --- .../Program.cs | 23 ++++++++++++++---- .../k8s/app-config.yaml | 10 ++++++++ .../k8s/monitoring/kustomization.yaml | 16 +++++++++++++ .../k8s/monitoring/monitoring-config.yaml | 8 +++++++ .../K3sBuilderExtensions.Manifest.cs | 24 +++++++++++-------- 5 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/app-config.yaml create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/kustomization.yaml create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/monitoring-config.yaml diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs index a25dce592..2bcee5be1 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -8,11 +8,14 @@ // // What this demonstrates: // 1. A k3s cluster starts inside a Docker container. -// 2. podinfo is installed via Helm — a lightweight demo app. -// 3. A K3sServiceEndpointResource exposes the podinfo service: +// 2. app-config ConfigMap applied via AddK8sManifest (plain YAML file). +// 3. monitoring-config ConfigMap applied via AddK8sManifest (Kustomize overlay — +// auto-detected from kustomization.yaml; adds namespace + common labels). +// 4. podinfo installed via Helm, waiting for both manifests first. +// 5. A K3sServiceEndpointResource exposes the podinfo service: // • Host processes reach it at http://localhost:{port} // • DCP-network containers reach it at http://host.docker.internal:{port} -// 4. WithDataVolume keeps the cluster state alive across AppHost restarts. +// 6. WithDataVolume keeps the cluster state alive across AppHost restarts. // ────────────────────────────────────────────────────────────────────────────── var builder = DistributedApplication.CreateBuilder(args); @@ -29,10 +32,22 @@ version: "6.7.1", @namespace: "podinfo"); +// Plain YAML file — a ConfigMap in the default namespace. +var appConfig = cluster.AddK8sManifest("app-config", "./k8s/app-config.yaml") + .WaitForCompletion(podinfo); + +// Kustomize overlay — auto-detected because the directory contains kustomization.yaml. +// Kustomize injects the 'monitoring' namespace and common labels into every resource +// without modifying the source files. +var monitoringConfig = cluster.AddK8sManifest("monitoring-config", "./k8s/monitoring") + .WaitForCompletion(podinfo) + .WaitForCompletion(appConfig); + // Expose the podinfo service as an Aspire endpoint resource. // WaitForCompletion waits for the helm install container to exit with code 0 // before starting the port-forward — no NodePort required. cluster.AddServiceEndpoint("podinfo-web", "podinfo", servicePort: 9898, @namespace: "podinfo") - .WaitForCompletion(podinfo); + .WaitForCompletion(podinfo) + .WaitForCompletion(monitoringConfig); builder.Build().Run(); diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/app-config.yaml b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/app-config.yaml new file mode 100644 index 000000000..57c26eb6e --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/app-config.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config + namespace: default +data: + environment: development + log-level: info + # podinfo endpoint — matches the AddServiceEndpoint name in Program.cs + api-endpoint: http://podinfo:9898 diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/kustomization.yaml b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/kustomization.yaml new file mode 100644 index 000000000..c95a7c85a --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Kustomize places every resource in this namespace. +namespace: default + +# Common labels applied to all resources in this overlay. +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/managed-by: aspire + environment: development + + +resources: + - monitoring-config.yaml diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/monitoring-config.yaml b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/monitoring-config.yaml new file mode 100644 index 000000000..217066c49 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/k8s/monitoring/monitoring-config.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: monitoring-config +data: + scrape-interval: 30s + retention: 7d + log-level: info diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 216de6624..6ad6bd370 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -158,21 +158,25 @@ internal static string BuildManifestScript() // Auto-detect kustomize: if a kustomization file is present, use -k. // Otherwise use -f with server-side apply. + // Capture output so we can extract any CRD names that were applied. sb.AppendLine("if [ -f /k8s-manifests/kustomization.yaml ] || [ -f /k8s-manifests/kustomization.yml ]; then"); sb.AppendLine(" echo 'Detected kustomization — using kubectl apply -k'"); - sb.AppendLine(" kubectl apply -k /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine(" APPLIED=$(kubectl apply -k /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts)"); sb.AppendLine("else"); - sb.AppendLine(" kubectl apply -f /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine(" APPLIED=$(kubectl apply -f /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts)"); sb.AppendLine("fi"); - - // Wait only for CRDs applied by this manifest, identified via the aspire-k3s - // field-manager. Using --all would also wait for pre-existing or concurrently - // installed CRDs that are stuck, causing an unrelated manifest to hang. - sb.AppendLine("CRDS=$(kubectl get crd -o name --no-headers 2>/dev/null \\"); - sb.AppendLine(" | xargs -r -I{} sh -c 'kubectl get {} -o jsonpath=\"{.metadata.managedFields[*].manager}\" 2>/dev/null | grep -q aspire-k3s && echo {}' 2>/dev/null)"); - sb.AppendLine("if [ -n \"$CRDS\" ]; then"); + sb.AppendLine("echo \"$APPLIED\""); + + // Parse the apply output for CRD names — kubectl apply prints one line per resource + // in the form "/ ", e.g.: + // customresourcedefinition.apiextensions.k8s.io/widgets.example.com created + // Only lines starting with "customresourcedefinition." belong to this apply. + // This avoids touching pre-existing or concurrently installed cluster CRDs and + // prevents busybox xargs from returning exit code 123 when grep finds no match. + sb.AppendLine("CRD_NAMES=$(echo \"$APPLIED\" | grep '^customresourcedefinition\\.' | awk '{print $1}')"); + sb.AppendLine("if [ -n \"$CRD_NAMES\" ]; then"); sb.AppendLine(" # shellcheck disable=SC2086"); - sb.AppendLine(" kubectl wait --for=condition=Established $CRDS --timeout=300s"); + sb.AppendLine(" kubectl wait --for=condition=Established $CRD_NAMES --timeout=300s"); sb.AppendLine("fi"); return sb.ToString(); From 9c9e5d564028ebf2baf10721bd7ac4f60e17498d Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Thu, 28 May 2026 11:10:08 +0200 Subject: [PATCH 19/29] chore: providing project reference for polyglot package --- .../ValidationAppHost/aspire.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json index 80fbafe65..b12c66e70 100644 --- a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json @@ -21,6 +21,6 @@ } }, "packages": { - "CommunityToolkit.Aspire.Hosting.K3s": "" + "CommunityToolkit.Aspire.Hosting.K3s": "../../../../../src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj" } } From bd213740b1cfb04de1211718573c540b0f3363aa Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Thu, 28 May 2026 12:01:18 +0200 Subject: [PATCH 20/29] chore: adding more tests to increase coverage --- .../K3sInProcessPortForwarder.cs | 17 +- .../K3sReadinessHealthCheck.cs | 30 +- ...ityToolkit.Aspire.Hosting.K3s.Tests.csproj | 4 + .../HelmReleaseResourceTests.cs | 175 +++++++ .../K3sClusterResourceTests.cs | 271 +++++++++++ .../K3sInProcessPortForwarderTests.cs | 171 +++++++ .../K3sPublicApiTests.cs | 239 +++++++++ .../K3sReadinessHealthCheckTests.cs | 345 +++++++++++++ .../K3sServiceEndpointResourceTests.cs | 457 ++++++++++++++++++ .../K8sManifestResourceTests.cs | 25 + 10 files changed, 1723 insertions(+), 11 deletions(-) create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sInProcessPortForwarderTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index e3fd6f5ea..182b86a62 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -27,11 +27,20 @@ internal sealed class K3sInProcessPortForwarder( string serviceName, int localPort, int servicePort, - Action onReadyChanged) : IAsyncDisposable + Action onReadyChanged, + Func? kubernetesFactory = null) : IAsyncDisposable { private readonly CancellationTokenSource _cts = new(); private TcpListener? _listener; + // Creates a client from the kubeconfig path using the injected factory if provided, + // or the default KubernetesClient.BuildConfigFromConfigFile path. The factory + // parameter exists to enable mock injection in unit tests without a real cluster. + private IKubernetes CreateClient() => + kubernetesFactory is not null + ? kubernetesFactory(kubeconfigPath) + : new Kubernetes(KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath)); + public async Task RunAsync(ILogger logger, CancellationToken ct) { // Link the outer application-lifetime token with our own so either can stop the loop. @@ -133,8 +142,7 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct { try { - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); - using var k8sClient = new Kubernetes(config); + using var k8sClient = CreateClient(); var svc = await k8sClient.CoreV1 .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) @@ -197,8 +205,7 @@ private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, Cancell using var _ = tcp; try { - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); - using var k8sClient = new Kubernetes(config); + using var k8sClient = CreateClient(); // Resolve the service to a running pod. var svc = await k8sClient.CoreV1 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index a66ca4c0c..ff6aa6e99 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -20,8 +20,16 @@ namespace CommunityToolkit.Aspire.Hosting; /// No docker exec is involved — works with any container runtime. /// /// -internal sealed class K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) : IHealthCheck +internal sealed class K3sReadinessHealthCheck( + K3sClusterResource resource, + EndpointReference endpoint, + Func? kubernetesFactory = null) : IHealthCheck { + private IKubernetes CreateClient(string path) => + kubernetesFactory is not null + ? kubernetesFactory(path) + : new Kubernetes(KubernetesClientConfiguration.BuildConfigFromConfigFile(path)); + /// public async Task CheckHealthAsync( HealthCheckContext context, @@ -30,6 +38,17 @@ public async Task CheckHealthAsync( if (!endpoint.IsAllocated) return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); + return await CheckCoreAsync(endpoint.Port, cancellationToken).ConfigureAwait(false); + } + + /// + /// Core readiness check given an already-known API server . + /// Extracted so unit tests can exercise the full check path without requiring DCP + /// to allocate the endpoint (i.e. without + /// being true). + /// + internal async Task CheckCoreAsync(int port, CancellationToken cancellationToken = default) + { var dir = resource.KubeconfigDirectory; if (dir is null) return HealthCheckResult.Unhealthy("Kubeconfig directory not configured on resource"); @@ -40,14 +59,13 @@ public async Task CheckHealthAsync( try { - var localPath = await EnsureKubeconfigVariantsAsync(rawPath, dir, endpoint.Port, cancellationToken) + var localPath = await EnsureKubeconfigVariantsAsync(rawPath, dir, port, cancellationToken) .ConfigureAwait(false); // Create a fresh client per check — no cached state, no stale connection risk. // At a 5-second health-check interval the TLS handshake overhead is negligible // for a local dev integration. - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(localPath); - using var k8sClient = new Kubernetes(config); + using var k8sClient = CreateClient(localPath); var nodes = await k8sClient.CoreV1 .ListNodeAsync(cancellationToken: cancellationToken) @@ -111,7 +129,7 @@ await WriteAtomicAsync( return localPath; } - private static string BuildConfigYaml(K8SConfiguration source, string serverUrl) + internal static string BuildConfigYaml(K8SConfiguration source, string serverUrl) { var yaml = KubernetesYaml.Serialize(source); var copy = KubernetesYaml.Deserialize(yaml); @@ -135,7 +153,7 @@ private static async Task WriteAtomicAsync(string path, string content, Cancella File.Move(tmp, path, overwrite: true); } - private static bool IsTlsOrAuthFailure(Exception ex) => + internal static bool IsTlsOrAuthFailure(Exception ex) => ex is System.Security.Authentication.AuthenticationException || ex.InnerException is System.Security.Authentication.AuthenticationException || (ex is k8s.Autorest.HttpOperationException op && diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj index bad157c30..42dfef9e7 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj @@ -6,5 +6,9 @@ + + + + diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index d5f6feb5b..0fba5614e 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -435,4 +435,179 @@ public void WithHelmValuesFileShouldThrowWhenBuilderIsNull() var action = () => builder.WithHelmValuesFile("values.yaml"); Assert.Throws(action); } + + // ── WithHelmValue override ──────────────────────────────────────────────── + + [Fact] + public void WithHelmValueOverridesDuplicateKey() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValue("replicaCount", "1") + .WithHelmValue("replicaCount", "3"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + + // Dictionary last-write wins — final value must be "3". + Assert.Equal("3", resource.HelmValues["replicaCount"]); + } + + [Fact] + public void BuildHelmScriptDuplicateKeyAppearsOnce() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.HelmValues["replicaCount"] = "3"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + // Dictionary deduplication: key appears exactly once in the --set flags. + Assert.Single(System.Text.RegularExpressions.Regex.Matches(script, "replicaCount").Cast()); + } + + // ── HelmEscape / ShellEscape via BuildHelmScript ────────────────────────── + + [Fact] + public void BuildHelmScriptHelmEscapesCommaInValue() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.HelmValues["tags"] = "a,b,c"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + // Helm --set comma is a list delimiter; must be backslash-escaped. + Assert.Contains(@"a\,b\,c", script); + } + + [Fact] + public void BuildHelmScriptHelmEscapesOpenBraceInValue() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.HelmValues["config"] = "{key:val}"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + Assert.Contains(@"\{key:val\}", script); + } + + [Fact] + public void BuildHelmScriptHelmEscapesBackslashInValue() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.HelmValues["path"] = @"C:\data"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + // Backslash must be doubled so Helm does not treat it as an escape prefix. + Assert.Contains(@"C:\\data", script); + } + + [Fact] + public void BuildHelmScriptShellEscapesSingleQuoteInValue() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.HelmValues["msg"] = "it's"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + // POSIX single-quote escape: ' → '\'' + Assert.Contains("it'\\''s", script); + } + + [Fact] + public void BuildHelmScriptHelmEscapesCommaInKey() + { + // Helm --set parser applies the same metacharacter rules to keys. + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.HelmValues["a,b"] = "v"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + Assert.Contains(@"a\,b=v", script); + } + + // ── WithHelmValuesFile with absolute path ───────────────────────────────── + + [Fact] + public void WithHelmValuesFileStoresAbsolutePathAsIs() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var absolutePath = Path.Combine(Path.GetTempPath(), "values.yaml"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValuesFile(absolutePath); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var file = Assert.Single(resource.ValuesFiles); + Assert.Equal(absolutePath, file); + } + + [Fact] + public void WithHelmValuesFileMultipleFilesAccumulate() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValuesFile("base.yaml") + .WithHelmValuesFile("prod.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal(2, resource.ValuesFiles.Count); + } + + // ── Cluster HelmReleases tracking ───────────────────────────────────────── + + [Fact] + public void ClusterTracksRegisteredHelmReleases() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + cluster.AddHelmRelease("cert-manager", "cert-manager"); + + Assert.Contains("argocd", cluster.Resource.HelmReleases.Keys); + Assert.Contains("cert-manager", cluster.Resource.HelmReleases.Keys); + Assert.Equal(2, cluster.Resource.HelmReleases.Count); + } + + // ── Service endpoint from a different cluster not included ──────────────── + + [Fact] + public void ServiceEndpointParentMatchesItsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var clusterA = appBuilder.AddK3sCluster("k8s-a"); + var clusterB = appBuilder.AddK3sCluster("k8s-b"); + + clusterA.AddServiceEndpoint("ep-a", "svc", 80); + clusterB.AddServiceEndpoint("ep-b", "svc", 80); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var eps = model.Resources.OfType().ToList(); + Assert.Equal(2, eps.Count); + + Assert.Same(clusterA.Resource, eps.Single(e => e.Name == "ep-a").Parent); + Assert.Same(clusterB.Resource, eps.Single(e => e.Name == "ep-b").Parent); + } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 5b2d194ef..aefe7611f 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -333,4 +333,275 @@ public void ApiEndpointIsCached() Assert.Same(first, second); } + + // ── Idempotency ─────────────────────────────────────────────────────────── + + [Fact] + public void WithDataVolumeCalledTwiceProducesOnlyOneVolumeMount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDataVolume().WithDataVolume(); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var mounts = resource.Annotations + .OfType() + .Where(v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume) + .ToList(); + + // Idempotent: second call replaces the first rather than duplicating the mount. + Assert.Single(mounts); + } + + [Fact] + public void WithReferenceContainerCalledTwiceProducesOnlyOneBindMount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var container = appBuilder.AddContainer("app", "myorg/app"); + + container.WithReference(cluster); + container.WithReference(cluster); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var containerResource = model.Resources + .OfType() + .Single(r => r.Name == "app"); + + // File-level bind-mount at /tmp/k3s-kubeconfig.yaml must not be duplicated — + // Docker rejects containers with duplicate mount targets. + var kubeconfigMounts = containerResource.Annotations + .OfType() + .Where(m => m.Target == "/tmp/k3s-kubeconfig.yaml") + .ToList(); + + Assert.Single(kubeconfigMounts); + } + + // ── WithK3sVersion propagation ──────────────────────────────────────────── + + [Fact] + public void WithK3sVersionSyncsAllAgentImageTags() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder + .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2) + .WithK3sVersion("v1.30.0-k3s1"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + Assert.Equal(2, agents.Count); + + foreach (var agent in agents) + { + var img = Assert.Single(agent.Annotations.OfType()); + Assert.Equal("v1.30.0-k3s1", img.Tag); + } + } + + [Fact] + public void WithK3sVersionCalledTwiceAppliesLastTag() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s") + .WithK3sVersion("v1.29.0-k3s1") + .WithK3sVersion("v1.30.0-k3s1"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + Assert.Equal("v1.30.0-k3s1", annotation.Tag); + } + + // ── WithLifetime ────────────────────────────────────────────────────────── + + [Fact] + public void WithLifetimePersistentSetsClusterLifetimeAnnotation() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithLifetime(ContainerLifetime.Persistent); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Persistent, annotation.Lifetime); + } + + [Fact] + public void WithLifetimeSessionSetsClusterLifetimeAnnotation() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithLifetime(ContainerLifetime.Session); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Session, annotation.Lifetime); + } + + // ── Argument accumulation ───────────────────────────────────────────────── + + [Fact] + public void WithDisabledComponentCalledMultipleTimesAccumulatesArgs() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s") + .WithDisabledComponent("traefik") + .WithDisabledComponent("coredns"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + + Assert.Contains("--disable=traefik", ctx.Args); + Assert.Contains("--disable=coredns", ctx.Args); + } + + [Fact] + public void WithExtraArgCalledMultipleTimesAccumulatesArgs() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s") + .WithExtraArg("--write-kubeconfig-mode=644") + .WithExtraArg("--node-label=env=dev"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + + Assert.Contains("--write-kubeconfig-mode=644", ctx.Args); + Assert.Contains("--node-label=env=dev", ctx.Args); + } + + // ── Default args (absence checks) ──────────────────────────────────────── + + [Fact] + public void AddK3sClusterDefaultsToNoClusterCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + + Assert.DoesNotContain(ctx.Args, arg => arg is string s && s.StartsWith("--cluster-cidr=")); + } + + [Fact] + public void AddK3sClusterDefaultsToNoServiceCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + + Assert.DoesNotContain(ctx.Args, arg => arg is string s && s.StartsWith("--service-cidr=")); + } + + // ── Options configure callback ──────────────────────────────────────────── + + [Fact] + public void AddK3sClusterWithServiceCidrViaOptions() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => + { + opts.ServiceCidr = "10.99.0.0/16"; + }); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + + Assert.Contains("--service-cidr=10.99.0.0/16", ctx.Args); + } + + [Fact] + public void AddK3sClusterWithMultipleDisabledComponentsViaOptions() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => + { + opts.DisabledComponents.Add("traefik"); + opts.DisabledComponents.Add("coredns"); + }); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + + Assert.Contains("--disable=traefik", ctx.Args); + Assert.Contains("--disable=coredns", ctx.Args); + } + + [Fact] + public void AddK3sClusterWithExtraArgsViaOptions() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => + { + opts.ExtraArgs.Add("--write-kubeconfig-mode=644"); + }); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + + Assert.Contains("--write-kubeconfig-mode=644", ctx.Args); + } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sInProcessPortForwarderTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sInProcessPortForwarderTests.cs new file mode 100644 index 000000000..3671905bd --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sInProcessPortForwarderTests.cs @@ -0,0 +1,171 @@ +using k8s; +using k8s.Autorest; +using k8s.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sInProcessPortForwarderTests +{ + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static K3sInProcessPortForwarder MakeForwarder( + List readyValues, + Func? factory = null) + => new( + kubeconfigPath: "/fake/kubeconfig.yaml", + @namespace: "default", + serviceName: "my-svc", + localPort: 0, + servicePort: 80, + onReadyChanged: v => readyValues.Add(v), + kubernetesFactory: factory); + + private static Mock BuildMockK8s( + V1Service? service = null, + V1PodList? pods = null, + Action? onPodListCalled = null) + { + var mockCoreV1 = new Mock(); + var mockK8s = new Mock(); + mockK8s.Setup(k => k.CoreV1).Returns(mockCoreV1.Object); + + // The source calls the extension method ReadNamespacedServiceAsync which internally + // calls ReadNamespacedServiceWithHttpMessagesAsync on the interface. + mockCoreV1 + .Setup(c => c.ReadNamespacedServiceWithHttpMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny>>(), It.IsAny())) + .ReturnsAsync(new HttpOperationResponse + { + Body = service ?? MakeService(hasPodSelector: true), + Response = new HttpResponseMessage(), + }); + + mockCoreV1 + .Setup(c => c.ListNamespacedPodWithHttpMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny>>(), It.IsAny())) + .ReturnsAsync(() => + { + onPodListCalled?.Invoke(); + return new HttpOperationResponse + { + Body = pods ?? MakePodList(ready: true), + Response = new HttpResponseMessage(), + }; + }); + + return mockK8s; + } + + private static V1Service MakeService(bool hasPodSelector) => new() + { + Spec = new V1ServiceSpec + { + Ports = [new V1ServicePort { Port = 80, TargetPort = 80 }], + Selector = hasPodSelector + ? new Dictionary { ["app"] = "my-svc" } + : null, + } + }; + + private static V1PodList MakePodList(bool ready) => new() + { + Items = [new V1Pod + { + Metadata = new V1ObjectMeta { Name = "my-svc-pod-0" }, + Spec = new V1PodSpec + { + Containers = [new V1Container { Ports = [new V1ContainerPort { ContainerPort = 80 }] }] + }, + Status = new V1PodStatus + { + Phase = "Running", + ContainerStatuses = [new V1ContainerStatus { Ready = ready }] + } + }] + }; + + // ── Lifecycle ───────────────────────────────────────────────────────────── + + [Fact] + public async Task DisposeAsync_WithoutRunning_CompletesGracefully() + { + var forwarder = MakeForwarder([]); + await forwarder.DisposeAsync(); + } + + [Fact] + public async Task RunAsync_WithPreCancelledToken_ExitsImmediatelyAndSignalsNotReady() + { + var readyValues = new List(); + var forwarder = MakeForwarder(readyValues); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await forwarder.RunAsync(NullLogger.Instance, cts.Token); + + Assert.Single(readyValues); + Assert.False(readyValues[0]); + } + + // ── Ready pod ───────────────────────────────────────────────────────────── + + [Fact] + public async Task RunAsync_WhenServiceHasReadyPod_SignalsReady() + { + var readyValues = new List(); + using var cts = new CancellationTokenSource(); + + // Cancel after WaitForServiceReadyAsync confirms a ready pod, so RunAsync exits cleanly. + var mockK8s = BuildMockK8s(onPodListCalled: cts.Cancel); + + var forwarder = MakeForwarder(readyValues, _ => mockK8s.Object); + await forwarder.RunAsync(NullLogger.Instance, cts.Token); + + Assert.Contains(true, readyValues); + // RunAsync always ends with onReadyChanged(false). + Assert.False(readyValues[^1]); + } + + // ── No pod selector ─────────────────────────────────────────────────────── + + [Fact] + public async Task RunAsync_WhenServiceHasNoPodSelector_ExitsWithoutSignallingReady() + { + var readyValues = new List(); + var mockK8s = BuildMockK8s(service: MakeService(hasPodSelector: false)); + + var forwarder = MakeForwarder(readyValues, _ => mockK8s.Object); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await forwarder.RunAsync(NullLogger.Instance, cts.Token); + + // InvalidOperationException from missing selector causes the loop to break + // without ever signalling ready. + Assert.DoesNotContain(true, readyValues); + Assert.False(readyValues[^1]); + } + + // ── Not-ready pods ──────────────────────────────────────────────────────── + + [Fact] + public async Task RunAsync_WhenPodsNotReady_DoesNotSignalReadyBeforeCancellation() + { + var readyValues = new List(); + using var cts = new CancellationTokenSource(); + + int callCount = 0; + var mockK8s = BuildMockK8s( + pods: MakePodList(ready: false), + onPodListCalled: () => { if (++callCount >= 2) cts.Cancel(); }); + + var forwarder = MakeForwarder(readyValues, _ => mockK8s.Object); + await forwarder.RunAsync(NullLogger.Instance, cts.Token); + + Assert.DoesNotContain(true, readyValues); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs index f803d976f..94f4a7c77 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs @@ -106,4 +106,243 @@ public void WithReferenceShouldThrowWhenSourceIsNull() Assert.Throws(action); } + + // ── WithK3sVersion argument guards ──────────────────────────────────────── + + [Fact] + public void WithK3sVersionShouldThrowWhenTagIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + string tag = null!; + + var action = () => cluster.WithK3sVersion(tag); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(tag), exception.ParamName); + } + + [Fact] + public void WithK3sVersionShouldThrowWhenTagIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.WithK3sVersion(" "); + + Assert.Throws(action); + } + + // ── WithPodSubnet argument guards ───────────────────────────────────────── + + [Fact] + public void WithPodSubnetShouldThrowWhenCidrIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + string cidr = null!; + + var action = () => cluster.WithPodSubnet(cidr); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(cidr), exception.ParamName); + } + + [Fact] + public void WithPodSubnetShouldThrowWhenCidrIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.WithPodSubnet(" "); + + Assert.Throws(action); + } + + // ── WithServiceSubnet argument guards ───────────────────────────────────── + + [Fact] + public void WithServiceSubnetShouldThrowWhenCidrIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + string cidr = null!; + + var action = () => cluster.WithServiceSubnet(cidr); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(cidr), exception.ParamName); + } + + [Fact] + public void WithServiceSubnetShouldThrowWhenCidrIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.WithServiceSubnet(" "); + + Assert.Throws(action); + } + + // ── WithDisabledComponent argument guards ───────────────────────────────── + + [Fact] + public void WithDisabledComponentShouldThrowWhenComponentIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + string component = null!; + + var action = () => cluster.WithDisabledComponent(component); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(component), exception.ParamName); + } + + [Fact] + public void WithDisabledComponentShouldThrowWhenComponentIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.WithDisabledComponent(" "); + + Assert.Throws(action); + } + + // ── WithExtraArg argument guards ────────────────────────────────────────── + + [Fact] + public void WithExtraArgShouldThrowWhenArgIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + string arg = null!; + + var action = () => cluster.WithExtraArg(arg); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(arg), exception.ParamName); + } + + [Fact] + public void WithExtraArgShouldThrowWhenArgIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.WithExtraArg(" "); + + Assert.Throws(action); + } + + // ── AddServiceEndpoint argument guards ──────────────────────────────────── + + [Fact] + public void AddServiceEndpointShouldThrowWhenNameIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.AddServiceEndpoint(null!, "svc", 80); + + Assert.Throws(action); + } + + [Fact] + public void AddServiceEndpointShouldThrowWhenServiceNameIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.AddServiceEndpoint("ep", null!, 80); + + Assert.Throws(action); + } + + [Fact] + public void AddServiceEndpointShouldThrowWhenServiceNameIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.AddServiceEndpoint("ep", " ", 80); + + Assert.Throws(action); + } + + [Fact] + public void AddServiceEndpointShouldThrowWhenNamespaceIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.AddServiceEndpoint("ep", "svc", 80, @namespace: " "); + + Assert.Throws(action); + } + + [Fact] + public void AddServiceEndpointShouldThrowWhenPortIsZero() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.AddServiceEndpoint("ep", "svc", 0); + + Assert.Throws(action); + } + + [Fact] + public void AddServiceEndpointShouldThrowWhenPortExceeds65535() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.AddServiceEndpoint("ep", "svc", 65536); + + Assert.Throws(action); + } + + // ── WithReference (service endpoint) argument guards ───────────────────── + + [Fact] + public void WithReferenceServiceEndpointShouldThrowWhenDestinationIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + IResourceBuilder destination = null!; + + var action = () => destination.WithReference(ep); + + Assert.Throws(action); + } + + [Fact] + public void WithReferenceServiceEndpointShouldThrowWhenSourceIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var container = appBuilder.AddContainer("app", "myimage"); + IResourceBuilder source = null!; + + var action = () => container.WithReference(source); + + Assert.Throws(action); + } + + // ── WithHelmValuesFile argument guards ──────────────────────────────────── + + [Fact] + public void WithHelmValuesFileShouldThrowWhenPathIsWhitespace() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + var release = cluster.AddHelmRelease("argocd", "argo-cd"); + + var action = () => release.WithHelmValuesFile(" "); + + Assert.Throws(action); + } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs new file mode 100644 index 000000000..3a1050853 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs @@ -0,0 +1,345 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using k8s; +using k8s.Autorest; +using k8s.KubeConfigModels; +using k8s.Models; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Moq; +using System.Net; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sReadinessHealthCheckTests +{ + // ── CheckHealthAsync — early exit (no DCP allocation) ──────────────────── + + [Fact] + public async Task CheckHealthAsync_WhenEndpointNotAllocated_ReturnsUnhealthy() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + // In unit-test context there is no DCP, so endpoint.IsAllocated is always false. + var healthCheck = new K3sReadinessHealthCheck(cluster.Resource, cluster.Resource.ApiEndpoint); + + var result = await healthCheck.CheckHealthAsync(null!); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("not yet allocated", result.Description); + } + + // ── CheckCoreAsync — file-system and Kubernetes paths ──────────────────── + + [Fact] + public async Task CheckCoreAsync_WhenKubeconfigDirectoryIsNull_ReturnsUnhealthy() + { + var cluster = new K3sClusterResource("k8s") { KubeconfigDirectory = null }; + var healthCheck = new K3sReadinessHealthCheck(cluster, cluster.ApiEndpoint, _ => Mock.Of()); + + var result = await healthCheck.CheckCoreAsync(port: 6443); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("not configured", result.Description); + } + + [Fact] + public async Task CheckCoreAsync_WhenKubeconfigFileMissing_ReturnsUnhealthy() + { + var (healthCheck, _, _) = MakeCheck(writeKubeconfig: false, nodeCount: 0, agentCount: 0); + + var result = await healthCheck.CheckCoreAsync(port: 6443); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("write kubeconfig", result.Description); + } + + [Fact] + public async Task CheckCoreAsync_WhenAllNodesReady_ReturnsHealthy() + { + var (healthCheck, _, _) = MakeCheck(writeKubeconfig: true, nodeCount: 1, agentCount: 0); + + var result = await healthCheck.CheckCoreAsync(port: 6443); + + Assert.Equal(HealthStatus.Healthy, result.Status); + Assert.Contains("ready", result.Description); + } + + [Fact] + public async Task CheckCoreAsync_WhenNodesNotReady_ReturnsUnhealthy() + { + var (healthCheck, _, _) = MakeCheck(writeKubeconfig: true, nodeCount: 0, agentCount: 0); + + var result = await healthCheck.CheckCoreAsync(port: 6443); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("0/1 nodes Ready", result.Description); + } + + [Fact] + public async Task CheckCoreAsync_WithAgentsRequiresAllNodes() + { + // 1 server + 2 agents = 3 required; only 1 node ready → unhealthy. + var (healthCheck, _, _) = MakeCheck(writeKubeconfig: true, nodeCount: 1, agentCount: 2); + + var result = await healthCheck.CheckCoreAsync(port: 6443); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("1/3", result.Description); + } + + [Fact] + public async Task CheckCoreAsync_WhenUnexpectedExceptionThrown_ReturnsUnhealthy() + { + var (healthCheck, _, _) = MakeCheck( + writeKubeconfig: true, + nodeCount: 0, + agentCount: 0, + listNodesThrows: new InvalidOperationException("cluster exploded")); + + var result = await healthCheck.CheckCoreAsync(port: 6443); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("cluster exploded", result.Description); + } + + [Fact] + public async Task CheckCoreAsync_WhenTlsFailure_ReturnsStaleKubeconfigMessage() + { + var (healthCheck, dir, _) = MakeCheck( + writeKubeconfig: true, + nodeCount: 0, + agentCount: 0, + listNodesThrows: new System.Security.Authentication.AuthenticationException("bad cert")); + + var result = await healthCheck.CheckCoreAsync(port: 6443); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("stale", result.Description); + + // The raw cluster kubeconfig must NOT be deleted — only the derived variants. + Assert.True(File.Exists(Path.Combine(dir, "cluster", "kubeconfig.yaml"))); + } + + [Fact] + public async Task CheckCoreAsync_WritesLocalAndContainerKubeconfigVariants() + { + var (healthCheck, dir, _) = MakeCheck(writeKubeconfig: true, nodeCount: 1, agentCount: 0); + + await healthCheck.CheckCoreAsync(port: 6443); + + Assert.True(File.Exists(Path.Combine(dir, "local", "kubeconfig.yaml"))); + Assert.True(File.Exists(Path.Combine(dir, "container", "kubeconfig.yaml"))); + } + + [Fact] + public async Task CheckCoreAsync_LocalVariantContainsLocalhostUrl() + { + var (healthCheck, dir, _) = MakeCheck(writeKubeconfig: true, nodeCount: 1, agentCount: 0); + + await healthCheck.CheckCoreAsync(port: 7777); + + var localYaml = await File.ReadAllTextAsync(Path.Combine(dir, "local", "kubeconfig.yaml")); + Assert.Contains("https://localhost:7777", localYaml); + } + + [Fact] + public async Task CheckCoreAsync_ContainerVariantContainsClusterNameUrl() + { + var (healthCheck, dir, _) = MakeCheck(writeKubeconfig: true, nodeCount: 1, agentCount: 0); + + await healthCheck.CheckCoreAsync(port: 6443); + + var containerYaml = await File.ReadAllTextAsync(Path.Combine(dir, "container", "kubeconfig.yaml")); + Assert.Contains("https://k8s:6443", containerYaml); + } + + // ── BuildConfigYaml ─────────────────────────────────────────────────────── + + [Fact] + public void BuildConfigYaml_RewritesServerUrl() + { + var source = new K8SConfiguration + { + Clusters = + [ + new Cluster { Name = "k8s", ClusterEndpoint = new ClusterEndpoint { Server = "https://original:6443" } } + ] + }; + + var yaml = K3sReadinessHealthCheck.BuildConfigYaml(source, "https://localhost:9999"); + + Assert.Contains("https://localhost:9999", yaml); + Assert.DoesNotContain("https://original:6443", yaml); + } + + [Fact] + public void BuildConfigYaml_RewritesAllClusters() + { + var source = new K8SConfiguration + { + Clusters = + [ + new Cluster { Name = "a", ClusterEndpoint = new ClusterEndpoint { Server = "https://a:6443" } }, + new Cluster { Name = "b", ClusterEndpoint = new ClusterEndpoint { Server = "https://b:6443" } }, + ] + }; + + var yaml = K3sReadinessHealthCheck.BuildConfigYaml(source, "https://new-server:6443"); + + Assert.DoesNotContain("https://a:6443", yaml); + Assert.DoesNotContain("https://b:6443", yaml); + var occurrences = yaml.Split("https://new-server:6443").Length - 1; + Assert.Equal(2, occurrences); + } + + [Fact] + public void BuildConfigYaml_WithNullClusters_DoesNotThrow() + { + var source = new K8SConfiguration { Clusters = null }; + var yaml = K3sReadinessHealthCheck.BuildConfigYaml(source, "https://localhost:6443"); + Assert.NotNull(yaml); + } + + // ── IsTlsOrAuthFailure ──────────────────────────────────────────────────── + + [Fact] + public void IsTlsOrAuthFailure_WithAuthenticationException_ReturnsTrue() + { + var ex = new System.Security.Authentication.AuthenticationException("tls failed"); + Assert.True(K3sReadinessHealthCheck.IsTlsOrAuthFailure(ex)); + } + + [Fact] + public void IsTlsOrAuthFailure_WithInnerAuthenticationException_ReturnsTrue() + { + var inner = new System.Security.Authentication.AuthenticationException("inner tls"); + var ex = new InvalidOperationException("outer", inner); + Assert.True(K3sReadinessHealthCheck.IsTlsOrAuthFailure(ex)); + } + + [Fact] + public void IsTlsOrAuthFailure_WithHttpOperationExceptionUnauthorized_ReturnsTrue() + { + var ex = new HttpOperationException + { + Response = new HttpResponseMessageWrapper( + new HttpResponseMessage(HttpStatusCode.Unauthorized), "") + }; + Assert.True(K3sReadinessHealthCheck.IsTlsOrAuthFailure(ex)); + } + + [Fact] + public void IsTlsOrAuthFailure_WithHttpOperationExceptionForbidden_ReturnsFalse() + { + var ex = new HttpOperationException + { + Response = new HttpResponseMessageWrapper( + new HttpResponseMessage(HttpStatusCode.Forbidden), "") + }; + Assert.False(K3sReadinessHealthCheck.IsTlsOrAuthFailure(ex)); + } + + [Fact] + public void IsTlsOrAuthFailure_WithRegularException_ReturnsFalse() + { + Assert.False(K3sReadinessHealthCheck.IsTlsOrAuthFailure(new InvalidOperationException("other"))); + } + + [Fact] + public void IsTlsOrAuthFailure_WithHttpRequestException_ReturnsFalse() + { + Assert.False(K3sReadinessHealthCheck.IsTlsOrAuthFailure(new HttpRequestException("network"))); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private const string MinimalKubeconfig = """ + apiVersion: v1 + clusters: + - cluster: + server: https://127.0.0.1:6443 + insecure-skip-tls-verify: true + name: k8s + contexts: + - context: + cluster: k8s + user: default + name: k8s + current-context: k8s + kind: Config + preferences: {} + users: + - name: default + user: + token: test-token + """; + + private static (K3sReadinessHealthCheck check, string dir, Mock mock) MakeCheck( + bool writeKubeconfig, + int nodeCount, + int agentCount, + Exception? listNodesThrows = null) + { + var dir = Path.Combine(Path.GetTempPath(), $"k3s-hc-{Guid.NewGuid():N}"); + var clusterDir = Path.Combine(dir, "cluster"); + Directory.CreateDirectory(clusterDir); + + if (writeKubeconfig) + File.WriteAllText(Path.Combine(clusterDir, "kubeconfig.yaml"), MinimalKubeconfig); + + var cluster = new K3sClusterResource("k8s") { KubeconfigDirectory = dir }; + for (var i = 0; i < agentCount; i++) + { + cluster.AgentCount++; + cluster.AddAgentResource(new K3sAgentResource($"k8s-agent-{i}", cluster)); + } + + var mockCoreV1 = new Mock(); + var mockK8s = new Mock(); + mockK8s.Setup(k => k.CoreV1).Returns(mockCoreV1.Object); + + // The source calls the extension method ListNodeAsync which calls + // ListNodeWithHttpMessagesAsync on the interface. + if (listNodesThrows is not null) + { + mockCoreV1 + .Setup(c => c.ListNodeWithHttpMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), + It.IsAny>>(), + It.IsAny())) + .ThrowsAsync(listNodesThrows); + } + else + { + var nodes = Enumerable.Range(0, nodeCount) + .Select(_ => new V1Node + { + Status = new V1NodeStatus + { + Conditions = [new V1NodeCondition { Type = "Ready", Status = "True" }] + } + }) + .ToList(); + + mockCoreV1 + .Setup(c => c.ListNodeWithHttpMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), + It.IsAny>>(), + It.IsAny())) + .ReturnsAsync(new HttpOperationResponse + { + Body = new V1NodeList { Items = nodes }, + Response = new HttpResponseMessage(), + }); + } + + var check = new K3sReadinessHealthCheck(cluster, cluster.ApiEndpoint, _ => mockK8s.Object); + return (check, dir, mockK8s); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs new file mode 100644 index 000000000..8003e97c4 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs @@ -0,0 +1,457 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sServiceEndpointResourceTests +{ + // ── Registration ────────────────────────────────────────────────────────── + + [Fact] + public void AddServiceEndpointAddsResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("podinfo-web", "podinfo", 9898, "podinfo"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("podinfo-web", resource.Name); + } + + [Fact] + public void AddServiceEndpointStoresServiceNamePortAndNamespace() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("ep", "my-svc", 8080, "my-ns"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("my-svc", resource.ServiceName); + Assert.Equal(8080, resource.ServicePort); + Assert.Equal("my-ns", resource.Namespace); + } + + [Fact] + public void AddServiceEndpointDefaultsNamespaceToDefault() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("ep", "svc", 80); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("default", resource.Namespace); + } + + [Fact] + public void AddServiceEndpointParentIsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("ep", "svc", 80); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, resource.Parent); + Assert.IsAssignableFrom>(resource); + } + + [Fact] + public void AddServiceEndpointIsExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("ep", "svc", 80); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); + } + + [Fact] + public void AddServiceEndpointHasHealthCheckAnnotation() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("ep", "svc", 80); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Single(resource.Annotations.OfType()); + } + + [Fact] + public void AddServiceEndpointSetsInitialStateProperties() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("ep", "my-svc", 9898, "my-ns"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var snapshot = resource.Annotations.OfType() + .Select(a => a.InitialSnapshot) + .First(); + + Assert.Equal("K3s Service Endpoint", snapshot.ResourceType); + Assert.Contains(snapshot.Properties, p => p.Name == "ServiceName" && p.Value?.ToString() == "my-svc"); + Assert.Contains(snapshot.Properties, p => p.Name == "ServicePort" && p.Value?.ToString() == "9898"); + Assert.Contains(snapshot.Properties, p => p.Name == "Namespace" && p.Value?.ToString() == "my-ns"); + } + + // ── Scheme inference ────────────────────────────────────────────────────── + + [Theory] + [InlineData(443, "https")] + [InlineData(8443, "https")] + [InlineData(80, "http")] + [InlineData(8080, "http")] + [InlineData(9898, "http")] + [InlineData(3000, "http")] + public void AddServiceEndpointInfersSchemeFromPort(int port, string expectedScheme) + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var epBuilder = cluster.AddServiceEndpoint("ep", "svc", port); + + Assert.Equal(expectedScheme, epBuilder.Resource.Scheme); + } + + [Fact] + public void AddServiceEndpointExplicitSchemeOverridesInference() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + // Port 443 would normally infer https — override to http. + var epBuilder = cluster.AddServiceEndpoint("ep", "svc", 443, scheme: "http"); + + Assert.Equal("http", epBuilder.Resource.Scheme); + } + + [Fact] + public void AddServiceEndpointExplicitHttpsOnNonStandardPort() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var epBuilder = cluster.AddServiceEndpoint("ep", "svc", 9000, scheme: "https"); + + Assert.Equal("https", epBuilder.Resource.Scheme); + } + + // ── Port validation ─────────────────────────────────────────────────────── + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(65536)] + [InlineData(100000)] + public void AddServiceEndpointThrowsForInvalidPort(int port) + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var action = () => cluster.AddServiceEndpoint("ep", "svc", port); + + Assert.Throws(action); + } + + [Theory] + [InlineData(1)] + [InlineData(65535)] + public void AddServiceEndpointAcceptsPortBoundaries(int port) + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var ep = cluster.AddServiceEndpoint("ep", "svc", port); + + Assert.Equal(port, ep.Resource.ServicePort); + } + + // ── WithReference (service endpoint) ───────────────────────────────────── + + [Fact] + public void WithReferenceServiceEndpointAddsRuntimeArgToContainer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var container = appBuilder.AddContainer("app", "myorg/app"); + + container.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var containerResource = model.Resources + .OfType() + .Single(r => r.Name == "app"); + + // --add-host=host.docker.internal:host-gateway is injected for Linux Docker Engine. + Assert.NotEmpty(containerResource.Annotations.OfType()); + } + + [Fact] + public void WithReferenceServiceEndpointAddsEnvironmentCallbackToContainer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var container = appBuilder.AddContainer("app", "myorg/app"); + + container.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var containerResource = model.Resources + .OfType() + .Single(r => r.Name == "app"); + + Assert.Contains( + containerResource.Annotations.OfType(), + a => a.Callback is not null); + } + + [Fact] + public void WithReferenceServiceEndpointAddsEnvironmentCallbackToExecutable() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + + exe.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var exeResource = Assert.Single(model.Resources.OfType()); + Assert.Contains( + exeResource.Annotations.OfType(), + a => a.Callback is not null); + } + + [Fact] + public void WithReferenceServiceEndpointDoesNotAddRuntimeArgToExecutable() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + + exe.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var exeResource = Assert.Single(model.Resources.OfType()); + + // --add-host is only relevant for containers; host processes resolve localhost directly. + Assert.Empty(exeResource.Annotations.OfType()); + } + + // ── K3sServiceEndpointHealthCheck ───────────────────────────────────────── + + [Fact] + public async Task ServiceEndpointHealthCheckReturnsUnhealthyWhenNotReady() + { + var cluster = new K3sClusterResource("k8s"); + var endpoint = new K3sServiceEndpointResource("ep", "svc", 80, "default", cluster); + endpoint.IsReady = false; + var check = new K3sServiceEndpointHealthCheck(endpoint); + + var result = await check.CheckHealthAsync(null!); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + } + + [Fact] + public async Task ServiceEndpointHealthCheckReturnsHealthyWhenReady() + { + var cluster = new K3sClusterResource("k8s"); + var endpoint = new K3sServiceEndpointResource("ep", "svc", 80, "default", cluster); + endpoint.IsReady = true; + var check = new K3sServiceEndpointHealthCheck(endpoint); + + var result = await check.CheckHealthAsync(null!); + + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + [Fact] + public async Task ServiceEndpointHealthCheckDefaultsToNotReady() + { + var cluster = new K3sClusterResource("k8s"); + var endpoint = new K3sServiceEndpointResource("ep", "svc", 80, "default", cluster); + // IsReady is false by default (default bool) + var check = new K3sServiceEndpointHealthCheck(endpoint); + + var result = await check.CheckHealthAsync(null!); + + Assert.Equal(HealthStatus.Unhealthy, result.Status); + } + + // ── Resource construction ───────────────────────────────────────────────── + + [Fact] + public void K3sServiceEndpointResourceThrowsWhenClusterIsNull() + { + K3sClusterResource cluster = null!; + var action = () => new K3sServiceEndpointResource("ep", "svc", 80, "default", cluster); + Assert.Throws(action); + } + + [Fact] + public void K3sServiceEndpointResourceThrowsWhenServiceNameIsNull() + { + var cluster = new K3sClusterResource("k8s"); + var action = () => new K3sServiceEndpointResource("ep", null!, 80, "default", cluster); + Assert.Throws(action); + } + + [Fact] + public void K3sServiceEndpointResourceThrowsWhenNamespaceIsNull() + { + var cluster = new K3sClusterResource("k8s"); + var action = () => new K3sServiceEndpointResource("ep", "svc", 80, null!, cluster); + Assert.Throws(action); + } + + // ── Env callback invocation ─────────────────────────────────────────────── + + [Fact] + public async Task WithReferenceServiceEndpoint_ContainerCallback_SetsHostDockerInternalUrlWhenReady() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var container = appBuilder.AddContainer("app", "myorg/app"); + container.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); + + ep.Resource.IsReady = true; + ep.Resource.HostPort = 9090; + + var envVars = new Dictionary(); + var ctx = new EnvironmentCallbackContext( + new DistributedApplicationExecutionContext(DistributedApplicationOperation.Run), + envVars); + + foreach (var cb in containerResource.Annotations.OfType()) + await cb.Callback(ctx); + + Assert.True(ctx.EnvironmentVariables.ContainsKey("services__ep__url")); + Assert.Equal("http://host.docker.internal:9090", ctx.EnvironmentVariables["services__ep__url"]?.ToString()); + } + + [Fact] + public async Task WithReferenceServiceEndpoint_ContainerCallback_DoesNotSetUrlWhenNotReady() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var container = appBuilder.AddContainer("app", "myorg/app"); + container.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); + + // ep.IsReady is false by default + + var envVars = new Dictionary(); + var ctx = new EnvironmentCallbackContext( + new DistributedApplicationExecutionContext(DistributedApplicationOperation.Run), + envVars); + + foreach (var cb in containerResource.Annotations.OfType()) + await cb.Callback(ctx); + + Assert.False(ctx.EnvironmentVariables.ContainsKey("services__ep__url")); + } + + [Fact] + public async Task WithReferenceServiceEndpoint_ExecutableCallback_SetsLocalhostUrlWhenReady() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 443, scheme: "https"); + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + exe.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + var exeResource = Assert.Single(model.Resources.OfType()); + + ep.Resource.IsReady = true; + ep.Resource.HostPort = 7777; + + var envVars = new Dictionary(); + var ctx = new EnvironmentCallbackContext( + new DistributedApplicationExecutionContext(DistributedApplicationOperation.Run), + envVars); + + foreach (var cb in exeResource.Annotations.OfType()) + await cb.Callback(ctx); + + Assert.True(ctx.EnvironmentVariables.ContainsKey("services__ep__url")); + Assert.Equal("https://localhost:7777", ctx.EnvironmentVariables["services__ep__url"]?.ToString()); + } + + [Fact] + public async Task WithReferenceServiceEndpoint_ExecutableCallback_DoesNotSetUrlWhenNotReady() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + exe.WithReference(ep); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + var exeResource = Assert.Single(model.Resources.OfType()); + + // ep.IsReady is false by default + + var envVars = new Dictionary(); + var ctx = new EnvironmentCallbackContext( + new DistributedApplicationExecutionContext(DistributedApplicationOperation.Run), + envVars); + + foreach (var cb in exeResource.Annotations.OfType()) + await cb.Callback(ctx); + + Assert.False(ctx.EnvironmentVariables.ContainsKey("services__ep__url")); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs index a286f79ea..aa8f67fc8 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -215,6 +215,31 @@ public void BuildManifestScriptWaitsForKubeconfigBeforeApplying() // ── Kustomize detection ─────────────────────────────────────────────────── + [Fact] + public void ResolveFilesGlobPattern_ReturnsMatchingFilesOrdered() + { + var dir = Path.Combine(Path.GetTempPath(), $"glob-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "c.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "a.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "b.yaml"), ""); + + try + { + var globPath = Path.Combine(dir, "*.yaml"); + var files = K3sManifestBuilderExtensions.ResolveFilesForTest(globPath); + + Assert.Equal(3, files.Count); + Assert.Equal("a.yaml", Path.GetFileName(files[0])); + Assert.Equal("b.yaml", Path.GetFileName(files[1])); + Assert.Equal("c.yaml", Path.GetFileName(files[2])); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + [Fact] public void AddK8sManifestKustomizeDirectoryShowsKustomizeResourceType() { From c508bc526284f75fd72eecc365e3c5da836e4004 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Tue, 2 Jun 2026 15:04:18 +0200 Subject: [PATCH 21/29] fix: adopting with aspire 13.4 changes + improving to docs --- ...yToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- .../Program.cs | 5 +- .../HelmReleaseResource.cs | 28 +- .../K3sAgentResource.cs | 20 +- .../K3sBuilderExtensions.Helm.cs | 98 +++-- .../K3sBuilderExtensions.Manifest.cs | 49 ++- .../K3sBuilderExtensions.ServiceEndpoint.cs | 110 +++--- .../K3sBuilderExtensions.cs | 355 +++++++++++++----- .../K3sClusterOptions.cs | 75 +++- .../K3sClusterResource.cs | 32 +- .../K3sReadinessHealthCheck.cs | 38 +- .../K3sReferenceAnnotations.cs | 5 + .../K3sServiceEndpointResource.cs | 73 +++- .../K8sManifestResource.cs | 21 +- .../K3sAgentNodeTests.cs | 5 +- .../K3sClusterResourceTests.cs | 65 ++-- .../K3sReadinessHealthCheckTests.cs | 30 +- .../K3sServiceEndpointResourceTests.cs | 106 ++++-- 18 files changed, 788 insertions(+), 329 deletions(-) create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sReferenceAnnotations.cs diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index 387601982..eb6d3ca1e 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs index 2bcee5be1..19220c7bc 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -21,7 +21,10 @@ var builder = DistributedApplication.CreateBuilder(args); var cluster = builder - .AddK3sCluster("k8s") + .AddK3sCluster("k8s", configure: cfg => + { + cfg.AgentCount = 2; + }) .WithDataVolume() .WithLifetime(ContainerLifetime.Persistent); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs index e288b9ab4..3a99557ee 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -3,18 +3,20 @@ namespace Aspire.Hosting.ApplicationModel; /// -/// Represents a Helm chart release deployed to a k3s cluster. -/// -/// Runs as an alpine/helm container on the DCP network. The container polls for the -/// cluster kubeconfig (written when the cluster health check first passes), executes -/// helm upgrade --install --wait, and exits with code 0 on success. Use -/// WaitForCompletion(helmRelease) on resources that depend on the release being installed. -/// +/// Represents a Helm chart release installed into a k3s cluster. /// -/// The Aspire resource name (also used as the Helm release name). -/// The Helm release name passed to helm upgrade --install. -/// The Kubernetes namespace to install into. +/// The Aspire resource name; also used as the Helm release name. +/// +/// The Helm release name passed to helm upgrade --install. +/// +/// The Kubernetes namespace to install the chart into. /// The parent k3s cluster resource. +/// +/// The release runs as an alpine/helm container. The container polls until the cluster +/// kubeconfig is available, executes helm upgrade --install --wait, then exits with +/// code 0. Use WaitForCompletion(helmRelease) on resources that depend on the chart +/// being fully installed. +/// [AspireExport(ExposeProperties = true)] public sealed class HelmReleaseResource( string name, @@ -26,10 +28,12 @@ public sealed class HelmReleaseResource( /// public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); - /// Gets the Helm release name. + /// + /// Gets the Helm release name passed to helm upgrade --install. + /// public string ReleaseName { get; } = releaseName ?? throw new ArgumentNullException(nameof(releaseName)); - /// Gets the target Kubernetes namespace. + /// Gets the Kubernetes namespace the chart is installed into. public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); internal string? Chart { get; set; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs index f6850faaa..b878bfdbd 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs @@ -1,18 +1,18 @@ namespace Aspire.Hosting.ApplicationModel; /// -/// Represents a k3s agent (worker) node that is a child of a . -/// -/// Agent nodes run k3s agent and join the cluster by connecting to the server's API -/// server at https://{serverName}:6443, resolved via DCP's built-in Docker DNS. -/// Agents start immediately alongside the server (no WaitFor dependency) and use -/// k3s's built-in retry loop to connect once the server becomes reachable. The cluster's -/// health check waits for all 1 + nodes -/// to reach Ready state before transitioning to Running. -/// +/// Represents a k3s agent (worker) node belonging to a . /// -/// The resource name (e.g. k8s-agent-0). +/// The resource name, e.g. k8s-agent-0. /// The parent k3s cluster resource. +/// +/// Agent nodes run k3s agent and join the server at https://{serverName}:6443 +/// using DCP's Docker DNS. They start in parallel with the server without a WaitFor +/// dependency — k3s's built-in retry loop handles the connection timing. The cluster's health +/// check waits for all 1 + AgentCount nodes to reach Ready state before the +/// cluster resource transitions to Running. Agent nodes are created automatically by +/// AddK3sCluster when K3sClusterOptions.AgentCount is greater than zero. +/// public sealed class K3sAgentResource(string name, K3sClusterResource cluster) : ContainerResource(name), IResourceWithParent { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index 6f00e62f8..ea1ea4f76 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -12,22 +12,47 @@ namespace Aspire.Hosting; public static class K3sHelmBuilderExtensions { /// - /// Adds a Helm release as a child resource of the k3s cluster. - /// - /// The release runs as a bitnami/helm container on the DCP network, executing - /// helm upgrade --install --wait then exiting. No host-side helm binary - /// is required. Use WaitForCompletion(helmRelease) on resources that depend on - /// the release being fully installed. - /// + /// Installs a Helm chart into the k3s cluster. /// /// The k3s cluster resource builder. - /// Resource name — also used as the Helm release name. - /// Chart name. Add for remote charts. - /// Optional Helm repository URL. - /// Optional chart version. - /// Target namespace (created automatically). + /// + /// The Aspire resource name; also used as the Helm release name passed to + /// helm upgrade --install. + /// + /// + /// The chart name, e.g. argo-cd for a repo chart or oci://registry/chart + /// for an OCI reference. Provide when using a named repo chart. + /// + /// + /// Optional Helm repository URL, e.g. https://argoproj.github.io/argo-helm. + /// When provided, the repo is added and updated before installation. + /// + /// + /// Optional chart version to pin, e.g. 7.8.0. When the + /// latest version available in the repository is installed. + /// + /// + /// The Kubernetes namespace to install the release into. The namespace is created + /// automatically if it does not exist. Defaults to default. + /// /// A builder for the . - [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] + /// + /// + /// The release runs as an alpine/helm container on the DCP network. No host-side + /// helm binary is required. The container exits with code 0 when the release is + /// fully installed and all workloads are ready. Use WaitForCompletion(helmRelease) + /// on resources that must start only after the chart is ready. + /// + /// + /// Customize the release with for individual key/value pairs + /// or to supply a full YAML values file. + /// + /// + /// + /// , , or is + /// . + /// + [AspireExport] public static IResourceBuilder AddHelmRelease( this IResourceBuilder builder, [ResourceName] string name, @@ -56,9 +81,9 @@ public static IResourceBuilder AddHelmRelease( // by AddK3sCluster; the kubeconfig file is written by K3sReadinessHealthCheck on // first successful health check. WaitFor(cluster) guarantees the file exists. // Ensure the host-side container/ directory exists so the health check can write to it. - var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); - Directory.CreateDirectory(containerKubeconfigDir); - var containerKubeconfigFile = Path.Combine(containerKubeconfigDir, "kubeconfig.yaml"); + var containerKubeconfigFile = Path.Combine(cluster.KubeconfigDirectory!, "container", "kubeconfig.yaml"); + // Placeholder ensures Docker creates a file-level bind-mount, not a directory. + K3sBuilderExtensions.EnsureKubeconfigPlaceholder(containerKubeconfigFile); var (helmRegistry, helmImage, helmTag) = cluster.HelmImageInfo; @@ -118,16 +143,24 @@ public static IResourceBuilder AddHelmRelease( } /// - /// Injects a host-side YAML values file into the Helm installer container and - /// passes it as --values /helm-values/{filename} to helm upgrade --install. - /// Multiple files are applied in the order they are declared (last wins for overlapping keys). + /// Supplies a YAML values file to the Helm release (--values). /// /// The Helm release resource builder. /// - /// Path to the values YAML file on the host. Relative paths are resolved against - /// AppHostDirectory. + /// Path to the YAML values file on the host. Relative paths are resolved against the + /// AppHost project directory. Call this method multiple times to supply additional files; + /// they are applied in declaration order and later files win for duplicate keys. /// - [AspireExport("withHelmValuesFile", Description = "Injects a host-side YAML values file into the Helm installer container")] + /// The same builder, for chaining. + /// + /// Use this method for structured overrides — particularly values containing commas, + /// braces, or backslashes that cannot be safely expressed with . + /// Values files are applied before --set flags, so + /// always takes precedence. + /// + /// is . + /// is or whitespace. + [AspireExport] public static IResourceBuilder WithHelmValuesFile( this IResourceBuilder builder, string path) @@ -146,9 +179,26 @@ public static IResourceBuilder WithHelmValuesFile( } /// - /// Adds a Helm --set key=value argument to this release. + /// Adds a Helm --set key=value override to the release. /// - [AspireExport("withHelmValue", Description = "Adds a --set key=value argument to the Helm release")] + /// The Helm release resource builder. + /// + /// The Helm value path using dot notation, e.g. server.service.type. + /// If the same key is set more than once, the last call wins. + /// + /// The value to assign. + /// The same builder, for chaining. + /// + /// Helm --set metacharacters (,, {, }, \) in + /// or are automatically escaped. + /// For values that contain these characters in ways Helm cannot represent safely, + /// use instead. + /// + /// + /// , , or is + /// . + /// + [AspireExport] public static IResourceBuilder WithHelmValue( this IResourceBuilder builder, string key, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 6ad6bd370..3b5a0a6d1 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -12,25 +12,40 @@ namespace Aspire.Hosting; public static class K3sManifestBuilderExtensions { /// - /// Applies one or more Kubernetes YAML files — or a Kustomize overlay — to the cluster - /// via a alpine/kubectl container. No host-side kubectl binary is required. - /// - /// The container exits with code 0 after manifests are applied and any CRDs reach the - /// Established condition. Use WaitForCompletion(manifest) on dependent resources. - /// + /// Applies Kubernetes YAML manifests or a Kustomize overlay to the cluster. + /// + /// The k3s cluster resource builder. + /// The Aspire resource name for this manifest application. + /// + /// Path to a single .yaml/.yml file, a directory of YAML files, or a + /// Kustomize overlay directory (containing kustomization.yaml). Relative paths + /// are resolved against the AppHost project directory. + /// + /// A builder for the . + /// /// - /// Three modes, selected automatically based on : + /// The apply mode is detected automatically from : /// - /// Single file — injected via WithContainerFiles, applied with kubectl apply -f. - /// Directory without kustomization.yaml — all .yaml/.yml files - /// injected via WithContainerFiles, applied with kubectl apply -f. + /// Single file — applied with kubectl apply -f. + /// Directory (no kustomization.yaml) — all .yaml/.yml + /// files in the directory are applied with kubectl apply -f. /// Kustomize overlay (directory contains kustomization.yaml or - /// kustomization.yml) — directory bind-mounted (preserving relative base references), - /// applied with kubectl apply -k. + /// kustomization.yml) — applied with kubectl apply -k. The directory + /// is bind-mounted so that relative references to base manifests are preserved. /// /// - /// - [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] + /// + /// The container exits with code 0 after all manifests are applied and any CRDs have + /// reached the Established condition. No host-side kubectl binary is required. + /// Use WaitForCompletion(manifest) on resources that must start only after the + /// manifests are applied. + /// + /// + /// + /// , , or is + /// . + /// + [AspireExport] public static IResourceBuilder AddK8sManifest( this IResourceBuilder builder, [ResourceName] string name, @@ -53,9 +68,9 @@ public static IResourceBuilder AddK8sManifest( var manifest = new K8sManifestResource(name, absolutePath, cluster); cluster.AddManifest(manifest.Name); - var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); - Directory.CreateDirectory(containerKubeconfigDir); - var containerKubeconfigFile = Path.Combine(containerKubeconfigDir, "kubeconfig.yaml"); + var containerKubeconfigFile = Path.Combine(cluster.KubeconfigDirectory!, "container", "kubeconfig.yaml"); + // Placeholder ensures Docker creates a file-level bind-mount, not a directory. + K3sBuilderExtensions.EnsureKubeconfigPlaceholder(containerKubeconfigFile); var (kubectlRegistry, kubectlImage, kubectlTag) = cluster.KubectlImageInfo; diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs index eec7c3302..d2f5bc55d 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -18,17 +18,56 @@ namespace Aspire.Hosting; public static class K3sServiceEndpointExtensions { /// - /// Exposes a Kubernetes service as a first-class Aspire endpoint resource. + /// Exposes a Kubernetes Service from the cluster as an Aspire endpoint resource. + /// + /// The k3s cluster resource builder. + /// The Aspire resource name for this endpoint. + /// + /// The name of the Kubernetes Service to forward, as it appears in kubectl get svc. + /// + /// + /// The port number declared on the Kubernetes Service (the port field, not + /// targetPort). Must be in the range 1–65535. + /// + /// + /// The Kubernetes namespace that contains the Service. Defaults to default. + /// + /// + /// The URL scheme (http or https) for the injected environment variable. + /// When (the default), the scheme is inferred from the port: + /// ports 443 and 8443 use https, all others use http. + /// + /// A builder for the . + /// /// - /// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready, - /// binding to 0.0.0.0:{hostPort}. The endpoint only becomes healthy after the - /// Kubernetes service has a ready pod — use WaitForCompletion on a - /// or to sequence the - /// install before starting the port-forward. + /// An in-process WebSocket port-forward is started when the cluster becomes ready, + /// binding to 0.0.0.0:{allocatedHostPort} so both host processes and DCP-network + /// containers can reach the service. /// - /// - [AspireExport("addServiceEndpoint", - Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] + /// + /// The endpoint transitions to Running only after the target Kubernetes Service + /// has at least one ready pod. Sequence chart installs or manifest applies before this + /// resource using WaitForCompletion on or + /// to prevent the port-forward from polling indefinitely. + /// + /// + /// Use WithReference(endpoint) on a dependent resource builder to inject the + /// service URL as services__{name}__url. Host processes receive + /// http(s)://localhost:{port}; containers receive + /// http(s)://host.docker.internal:{port}. + /// + /// + /// + /// , , or is + /// . + /// + /// + /// or is empty or whitespace. + /// + /// + /// is not in the range 1–65535. + /// + [AspireExport] public static IResourceBuilder AddServiceEndpoint( this IResourceBuilder builder, [ResourceName] string name, @@ -94,53 +133,12 @@ public static IResourceBuilder AddServiceEndpoint( }); } - /// - /// Injects the service URL exposed by into - /// using the Aspire services__{name}__url convention. - /// - /// Host processes receive http(s)://localhost:{port}. - /// Container resources receive http(s)://host.docker.internal:{port}. - /// The --add-host=host.docker.internal:host-gateway runtime arg is injected - /// automatically so the hostname resolves on Linux Docker Engine. - /// - /// - [AspireExport("withReference", - Description = "Injects the k3s service URL into a dependent resource")] - public static IResourceBuilder WithReference( - this IResourceBuilder destination, - IResourceBuilder source) - where TDestination : IResourceWithEnvironment - { - ArgumentNullException.ThrowIfNull(destination); - ArgumentNullException.ThrowIfNull(source); - - var ep = source.Resource; - var scheme = ep.Scheme; - var envKey = $"services__{ep.Name}__url"; - - if (destination.Resource is ContainerResource) - { - // Inject --add-host so host.docker.internal resolves inside Linux containers. - // DCP does not inject this automatically; Docker Desktop on Mac/Windows resolves - // it natively, but Docker Engine on Linux requires the explicit mapping. - // ContainerRuntimeArgsCallbackAnnotation receives IList directly. - destination.Resource.Annotations.Add( - new ContainerRuntimeArgsCallbackAnnotation( - args => args.Add("--add-host=host.docker.internal:host-gateway"))); - - return destination.WithEnvironment(ctx => - { - if (ep.IsReady) - ctx.EnvironmentVariables[envKey] = $"{scheme}://host.docker.internal:{ep.HostPort}"; - }); - } - - return destination.WithEnvironment(ctx => - { - if (ep.IsReady) - ctx.EnvironmentVariables[envKey] = $"{scheme}://localhost:{ep.HostPort}"; - }); - } + // WithReference(K3sServiceEndpointResource) is no longer a custom extension. + // K3sServiceEndpointResource implements IResourceWithConnectionString, so the standard + // Aspire WithReference(IResourceBuilder) overload injects + // services__{name}__url=http://localhost:{port} for host processes automatically. + // The BeforeStartEvent subscriber in AddK3sCluster overrides the URL to + // http://host.docker.internal:{port} for containers and adds --add-host. // ── Lifecycle ───────────────────────────────────────────────────────────── diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index 5f3927b4f..fabb43d11 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -14,17 +14,42 @@ namespace Aspire.Hosting; public static class K3sBuilderExtensions { /// - /// Adds a k3s Kubernetes cluster resource to the distributed application. + /// Adds a k3s Kubernetes cluster to the distributed application. /// /// The distributed application builder. - /// The resource name used for DNS resolution within the DCP network. + /// + /// The resource name. Also used as the DNS hostname by which containers in the DCP network + /// reach the cluster's API server (e.g. https://{name}:6443). + /// /// - /// Optional host port to bind the Kubernetes API server (port 6443) to. - /// When a random available port is assigned. + /// Host port to bind the Kubernetes API server (port 6443) to. + /// When (the default) a random available port is assigned. + /// + /// + /// Optional callback to configure agent count, image versions, custom CIDR ranges, + /// disabled components, and other cluster options. See . /// - /// Optional callback to configure . /// A builder for the . - [AspireExport("addK3sCluster", Description = "Adds a k3s Kubernetes cluster resource")] + /// + /// + /// The cluster runs as a privileged container using the rancher/k3s image. + /// No host-side kubectl, helm, or k3s binaries are required. + /// + /// + /// Three kubeconfig variants are written to {AppHostDirectory}/.k3s/{name}/ + /// when the cluster becomes ready: + /// + /// local/kubeconfig.yaml — injected into host processes via KUBECONFIG. + /// container/kubeconfig.yaml — bind-mounted into containers via KUBECONFIG. + /// + /// Call WithReference(cluster) on a dependent resource builder to inject + /// these credentials automatically. + /// + /// + /// + /// or is . + /// + [AspireExport(RunSyncOnBackgroundThread = true)] public static IResourceBuilder AddK3sCluster( this IDistributedApplicationBuilder builder, [ResourceName] string name, @@ -53,6 +78,13 @@ public static IResourceBuilder AddK3sCluster( var clusterDir = Path.Combine(kubeconfigDir, "cluster"); Directory.CreateDirectory(clusterDir); + // Pre-create placeholder files for all bind-mount sources. Docker creates a + // DIRECTORY at the source path when the file does not yet exist, which then + // prevents the health check from writing the real kubeconfig atomically. + // The health check overwrites these placeholders once the cluster is ready. + EnsureKubeconfigPlaceholder(Path.Combine(kubeconfigDir, "container", "kubeconfig.yaml")); + EnsureKubeconfigPlaceholder(Path.Combine(kubeconfigDir, "local", "kubeconfig.yaml")); + resource.KubeconfigDirectory = kubeconfigDir; var resourceBuilder = builder.AddResource(resource) @@ -216,6 +248,62 @@ public static IResourceBuilder AddK3sCluster( failureStatus: HealthStatus.Unhealthy, tags: null)); + // BeforeStartEvent: apply KUBECONFIG and service-URL injections declared via + // WithReference. The cluster owns this behavior — it knows what to inject and when. + // Processing is deferred to BeforeStartEvent so that resources can be wired up in + // any order in Program.cs without worrying about whether the cluster is configured yet. + builder.Eventing.Subscribe((evt, ct) => + { + var appModel = evt.Services.GetRequiredService(); + + foreach (var dependent in appModel.Resources) + { + // ── Container KUBECONFIG override ───────────────────────────── + // Standard WithReference(cluster) already injected KUBECONFIG= + // for all resource types (via IResourceWithConnectionString). For containers + // we additionally need: a file-level bind-mount of the container-network + // kubeconfig variant, plus an env override that points to it. + // + // Detect via the ResourceRelationshipAnnotation that standard WithReference + // adds when called with our cluster (which implements IResourceWithConnectionString). + bool referencesThisCluster = dependent.Annotations + .OfType() + .Any(a => ReferenceEquals(a.Resource, resource)); + + if (referencesThisCluster && dependent is ContainerResource) + { + ApplyKubeconfigContainerOverride(dependent, resource); + } + + // ── Service-URL container override ──────────────────────────── + // K3sServiceEndpointResource implements IResourceWithConnectionString, so + // standard WithReference injects services__ep__url=http://localhost:PORT for + // all resource types. For containers the URL must use host.docker.internal. + // Detect via the ResourceRelationshipAnnotation that standard WithReference + // adds when called with our endpoint (which implements IResourceWithConnectionString). + var endpointRefs = dependent.Annotations + .OfType() + .Select(a => a.Resource) + .OfType() + .Where(ep => ReferenceEquals(ep.Parent, resource)) + .ToList(); + + if (endpointRefs.Count > 0 && dependent is ContainerResource) + { + // --add-host is needed once per container regardless of how many + // endpoints are referenced. + dependent.Annotations.Add( + new ContainerRuntimeArgsCallbackAnnotation( + args => args.Add("--add-host=host.docker.internal:host-gateway"))); + } + + foreach (var ep in endpointRefs) + ApplyServiceUrlContainerOverride(dependent, ep); + } + + return Task.CompletedTask; + }); + // The cluster's ResourceReadyEvent drives service endpoint port-forwards. // HelmReleaseResource and K8sManifestResource containers are managed directly // by DCP — they WaitFor the cluster and exit when their work completes. @@ -241,8 +329,23 @@ public static IResourceBuilder AddK3sCluster( return resourceBuilder; } - /// Overrides the k3s server image version. - [AspireExport("withK3sVersion", Description = "Overrides the k3s server image version")] + /// + /// Sets the k3s image version used by the cluster server and all its agent nodes. + /// + /// The k3s cluster resource builder. + /// + /// The k3s container image tag, e.g. v1.32.3-k3s1. + /// Must follow the v{major}.{minor}.{patch}-k3s{n} format. + /// + /// The same builder, for chaining. + /// + /// All agent nodes are immediately synced to the same tag to prevent version skew + /// beyond the Kubernetes-supported ±1 minor version limit. The image tag is part of + /// the DCP container identity, so synchronisation must happen at configuration time. + /// + /// is . + /// is or whitespace. + [AspireExport] public static IResourceBuilder WithK3sVersion( this IResourceBuilder builder, string tag) @@ -250,11 +353,11 @@ public static IResourceBuilder WithK3sVersion( ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(tag); - // Update the server image tag. builder.WithImageTag(tag); - // Sync all agent nodes to the same tag — mismatched server/agent versions can - // break node joins and exceed Kubernetes' supported ±1 minor version skew. + // Sync agents immediately — DCP uses the ContainerImageAnnotation to compute + // container identity, so the tag must be set at configuration time, not deferred + // to BeforeStartEvent (which fires after DCP has already determined the identity). foreach (var agent in builder.Resource.AgentResources) { var existing = agent.Annotations.OfType().FirstOrDefault(); @@ -273,8 +376,15 @@ public static IResourceBuilder WithK3sVersion( return builder; } - /// Sets the pod subnet CIDR (--cluster-cidr). - [AspireExport("withPodSubnet", Description = "Sets the pod subnet CIDR for the k3s cluster")] + /// + /// Sets the CIDR range for pod IP addresses (--cluster-cidr). + /// + /// The k3s cluster resource builder. + /// The pod subnet in CIDR notation, e.g. 10.42.0.0/16. + /// The same builder, for chaining. + /// is . + /// is or whitespace. + [AspireExport] public static IResourceBuilder WithPodSubnet( this IResourceBuilder builder, string cidr) @@ -285,8 +395,15 @@ public static IResourceBuilder WithPodSubnet( return builder.WithArgs($"--cluster-cidr={cidr}"); } - /// Sets the service subnet CIDR (--service-cidr). - [AspireExport("withServiceSubnet", Description = "Sets the service subnet CIDR for the k3s cluster")] + /// + /// Sets the CIDR range for Service cluster IPs (--service-cidr). + /// + /// The k3s cluster resource builder. + /// The service subnet in CIDR notation, e.g. 10.43.0.0/16. + /// The same builder, for chaining. + /// is . + /// is or whitespace. + [AspireExport] public static IResourceBuilder WithServiceSubnet( this IResourceBuilder builder, string cidr) @@ -297,8 +414,19 @@ public static IResourceBuilder WithServiceSubnet( return builder.WithArgs($"--service-cidr={cidr}"); } - /// Disables a built-in k3s component (e.g. traefik). - [AspireExport("withDisabledComponent", Description = "Disables a built-in k3s component")] + /// + /// Disables a built-in k3s component (--disable=<component>). + /// + /// The k3s cluster resource builder. + /// + /// The component name to disable. Common values include traefik, + /// servicelb, metrics-server, coredns, and local-storage. + /// Call this method multiple times to disable more than one component. + /// + /// The same builder, for chaining. + /// is . + /// is or whitespace. + [AspireExport] public static IResourceBuilder WithDisabledComponent( this IResourceBuilder builder, string component) @@ -309,8 +437,22 @@ public static IResourceBuilder WithDisabledComponent( return builder.WithArgs($"--disable={component}"); } - /// Appends a raw argument to the k3s server command. - [AspireExport("withExtraArg", Description = "Appends a raw argument to the k3s server command")] + /// + /// Appends a raw argument to the k3s server command line. + /// + /// The k3s cluster resource builder. + /// + /// The raw argument to append, e.g. --write-kubeconfig-mode=644. + /// Call this method multiple times to append additional arguments. + /// + /// The same builder, for chaining. + /// + /// Use or the dedicated CIDR methods when possible. + /// This method is intended for flags that have no dedicated helper. + /// + /// is . + /// is or whitespace. + [AspireExport] public static IResourceBuilder WithExtraArg( this IResourceBuilder builder, string arg) @@ -322,10 +464,23 @@ public static IResourceBuilder WithExtraArg( } /// - /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) - /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. + /// Mounts a named Docker volume at the k3s data directory so cluster state persists across + /// AppHost restarts. /// - [AspireExport("withDataVolume", Description = "Adds a named volume for the k3s cluster data directory so state survives AppHost restarts")] + /// The k3s cluster resource builder. + /// + /// Optional volume name. When (the default) a name is generated + /// from the application and resource names. + /// + /// The same builder, for chaining. + /// + /// The volume covers /var/lib/rancher/k3s, which contains the SQLite database, + /// TLS certificates, and kubeconfig. Without this volume the cluster starts fresh on + /// every AppHost launch. Combine with ContainerLifetime.Persistent on the cluster + /// resource and its dependent Helm releases to avoid re-installing charts on every start. + /// + /// is . + [AspireExport] public static IResourceBuilder WithDataVolume( this IResourceBuilder builder, string? name = null) @@ -344,80 +499,23 @@ public static IResourceBuilder WithDataVolume( .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s"); } - /// - /// Injects the k3s kubeconfig into so it can authenticate - /// to the cluster. The injection method is selected automatically based on the resource type: - /// - /// - /// s receive a physical kubeconfig file copied to - /// /tmp/k3s-kubeconfig.yaml (container-network variant, - /// server: https://{resourceName}:6443). KUBECONFIG=/tmp/k3s-kubeconfig.yaml - /// is set automatically so all standard Kubernetes tooling (kubectl, helm, - /// KubernetesClient SDK) works without any custom bootstrap code. - /// - /// - /// Projects and executables receive KUBECONFIG=<host path>/local/kubeconfig.yaml - /// pointing directly to a file on the host filesystem. - /// - /// - /// Both files are written by the health check after all nodes reach Ready state. - /// Use WaitFor(cluster) on the dependent resource to guarantee the files exist - /// before the resource starts. - /// - [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] - public static IResourceBuilder WithReference( - this IResourceBuilder destination, - IResourceBuilder source) - where TDestination : IResourceWithEnvironment - { - ArgumentNullException.ThrowIfNull(destination); - ArgumentNullException.ThrowIfNull(source); - - var cluster = source.Resource; - - if (destination.Resource is ContainerResource) - { - // Idempotent: skip if the file bind-mount was already added (e.g. by a second - // WithReference call) — Docker rejects duplicate mounts at the same target. - var alreadyMounted = destination.Resource.Annotations - .OfType() - .Any(m => m.Target == K3sFileHelpers.ContainerKubeconfigPath); - - if (!alreadyMounted) - { - // File-level mount: only the kubeconfig YAML is visible inside the container. - // Mounting the entire container/ directory would expose kubectl's cache - // directories (cache/, http-cache/) on the host and cause concurrent-container - // cache corruption when multiple containers share the same kubeconfig directory. - var containerKubeconfigFile = Path.Combine( - cluster.KubeconfigDirectory!, "container", "kubeconfig.yaml"); - Directory.CreateDirectory(Path.GetDirectoryName(containerKubeconfigFile)!); - - destination.Resource.Annotations.Add( - new ContainerMountAnnotation( - containerKubeconfigFile, - K3sFileHelpers.ContainerKubeconfigPath, - ContainerMountType.BindMount, - isReadOnly: true)); - } - - return destination.WithEnvironment("KUBECONFIG", K3sFileHelpers.ContainerKubeconfigPath); - } - - // Projects and executables: KUBECONFIG points to the host-accessible local kubeconfig. - // This file is regenerated on every AppHost start (port may change). - return destination.WithEnvironment(ctx => - { - if (cluster.KubeconfigDirectory is null) return; - var path = Path.Combine(cluster.KubeconfigDirectory, "local", "kubeconfig.yaml"); - ctx.EnvironmentVariables["KUBECONFIG"] = path; - }); - } - /// /// Sets the container lifetime for the k3s cluster and all its agent nodes. /// - [AspireExport("withLifetime", Description = "Sets the container lifetime for the k3s cluster and all its agent nodes")] + /// The k3s cluster resource builder. + /// The container lifetime to apply. + /// The same builder, for chaining. + /// + /// + /// Agent nodes are propagated immediately because DCP uses the + /// to compute container identity. Deferring + /// propagation to BeforeStartEvent would be too late — DCP determines whether + /// to reuse or recreate a persistent container before that event fires, so agents + /// would lose their persistent identity and be recreated as new containers each run. + /// + /// + /// is . + [AspireExport] public static IResourceBuilder WithLifetime( this IResourceBuilder builder, ContainerLifetime lifetime) @@ -430,12 +528,8 @@ public static IResourceBuilder WithLifetime( foreach (var agent in builder.Resource.AgentResources) { - var existing = agent.Annotations.OfType().ToList(); - foreach (var ann in existing) - { + foreach (var ann in agent.Annotations.OfType().ToList()) agent.Annotations.Remove(ann); - } - agent.Annotations.Add(new ContainerLifetimeAnnotation { Lifetime = lifetime }); } @@ -478,6 +572,71 @@ busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.pr exec k3s "$@" """; + + // ── BeforeStartEvent helpers ────────────────────────────────────────────── + + // Called only for containers — standard WithReference(IResourceWithConnectionString) already + // handles host processes by injecting KUBECONFIG= via GetConnectionStringAsync(). + internal static void ApplyKubeconfigContainerOverride(IResource dependent, K3sClusterResource cluster) + { + // File-level bind-mount: only the kubeconfig YAML is visible inside the container. + // Mounting the full container/ dir would expose kubectl's cache dirs and cause + // concurrent-container cache corruption when multiple containers share the directory. + var alreadyMounted = dependent.Annotations + .OfType() + .Any(m => m.Target == K3sFileHelpers.ContainerKubeconfigPath); + + if (!alreadyMounted) + { + var containerKubeconfigFile = Path.Combine( + cluster.KubeconfigDirectory!, "container", "kubeconfig.yaml"); + // Placeholder ensures Docker creates a file-level mount, not a directory. + // AddK3sCluster already creates this; the guard handles late callers. + EnsureKubeconfigPlaceholder(containerKubeconfigFile); + + dependent.Annotations.Add(new ContainerMountAnnotation( + containerKubeconfigFile, + K3sFileHelpers.ContainerKubeconfigPath, + ContainerMountType.BindMount, + isReadOnly: true)); + } + + // Override the KUBECONFIG env var that standard WithReference already set to the + // local path — containers need the container-network variant mounted at a fixed path. + // This callback runs after the standard one (added later in BeforeStartEvent), so last + // write wins in the environment variable dictionary. + dependent.Annotations.Add(new EnvironmentCallbackAnnotation( + ctx => ctx.EnvironmentVariables["KUBECONFIG"] = K3sFileHelpers.ContainerKubeconfigPath)); + } + + // Creates an empty placeholder file so Docker's bind-mount sees a file at the source + // path rather than creating a directory there. If a directory already exists from a + // previous bad run it is removed first. The health check overwrites the placeholder + // with the real kubeconfig content once the cluster is ready. + internal static void EnsureKubeconfigPlaceholder(string filePath) + { + Directory.CreateDirectory(Path.GetDirectoryName(filePath)!); + if (Directory.Exists(filePath)) + Directory.Delete(filePath, recursive: true); + if (!File.Exists(filePath)) + File.WriteAllText(filePath, string.Empty); + } + + // Called only for containers. Standard WithReference(IResourceWithConnectionString) + // already injected services__ep__url=http://localhost:PORT for host processes via + // GetConnectionStringAsync(). This override switches the URL to host.docker.internal + // so DCP-network containers can reach the port-forward listener. + internal static void ApplyServiceUrlContainerOverride(IResource dependent, K3sServiceEndpointResource ep) + { + var scheme = ep.Scheme; + var envKey = $"services__{ep.Name}__url"; + + dependent.Annotations.Add(new EnvironmentCallbackAnnotation(ctx => + { + if (ep.IsReady) + ctx.EnvironmentVariables[envKey] = $"{scheme}://host.docker.internal:{ep.HostPort}"; + })); + } } #pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs index 08da6dc38..81b06195d 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -1,80 +1,115 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +using Aspire.Hosting; + namespace CommunityToolkit.Aspire.Hosting; /// -/// Configuration options for a k3s cluster resource. +/// Configuration options passed to AddK3sCluster via its configure callback. /// +/// +/// All settings are optional. Defaults produce a single-node cluster using the package's +/// bundled k3s, alpine/helm, and alpine/kubectl image versions. +/// +[AspireExport(ExposeProperties = true)] public sealed class K3sClusterOptions { /// - /// Gets or sets the CIDR range for pod IPs (passed as --cluster-cidr). + /// Gets or sets the pod IP address range in CIDR notation, passed as + /// --cluster-cidr to the k3s server. /// + /// + /// When (the default), k3s uses its built-in default of + /// 10.42.0.0/16. Can also be set fluently via WithPodSubnet. + /// public string? ClusterCidr { get; set; } /// - /// Gets or sets the CIDR range for service IPs (passed as --service-cidr). + /// Gets or sets the Service cluster IP address range in CIDR notation, passed as + /// --service-cidr to the k3s server. /// + /// + /// When (the default), k3s uses its built-in default of + /// 10.43.0.0/16. Can also be set fluently via WithServiceSubnet. + /// public string? ServiceCidr { get; set; } /// - /// Gets the list of k3s components to disable (each passed as --disable=<component>). + /// Gets the list of built-in k3s components to disable. + /// Each entry is passed as --disable=<component> to the k3s server. /// + /// + /// Common values: traefik, servicelb, metrics-server, + /// coredns, local-storage. Note that servicelb and + /// metrics-server are already disabled by default for faster cluster startup. + /// public IList DisabledComponents { get; } = new List(); /// - /// Gets the list of raw extra arguments appended to the k3s server command. + /// Gets the list of raw arguments appended to the k3s server command line. /// + /// + /// Use this for flags that have no dedicated option in . + /// Prefer the typed properties and the fluent extension methods (WithPodSubnet, + /// WithDisabledComponent, etc.) when available. + /// public IList ExtraArgs { get; } = new List(); /// - /// Gets or sets the number of agent (worker) nodes to add to the cluster. - /// Equivalent to k3d's --agents N flag. - /// Defaults to 0 (single-node cluster). + /// Gets or sets the number of agent (worker) nodes to add alongside the server. + /// Defaults to 0, which produces a single-node cluster. /// + /// + /// Each agent runs as a separate container and joins the server automatically. + /// The cluster health check waits for all 1 + AgentCount nodes to be + /// Ready before the cluster resource transitions to Running. + /// public int AgentCount { get; set; } /// - /// Gets or sets the k3s image tag (e.g. v1.31.4-k3s1). - /// When , the default tag embedded in the package is used. + /// Gets or sets the k3s container image tag, e.g. v1.32.3-k3s1. + /// When (the default), the version bundled with this package is used. /// + /// + /// Can also be set fluently after AddK3sCluster via WithK3sVersion. + /// public string? ImageTag { get; set; } // ── Helm installer image ────────────────────────────────────────────────── /// - /// Gets or sets the registry for the Helm installer container image. + /// Gets or sets the container registry for the Helm installer image. /// Defaults to docker.io. /// public string HelmRegistry { get; set; } = HelmContainerImageTags.Registry; /// - /// Gets or sets the Helm installer container image name. - /// Defaults to alpine/helm. + /// Gets or sets the Helm installer image name. Defaults to alpine/helm. /// public string HelmImage { get; set; } = HelmContainerImageTags.Image; /// - /// Gets or sets the Helm installer container image tag. - /// Defaults to 3.17.3. + /// Gets or sets the Helm installer image tag. Defaults to 3.17.3. /// public string HelmTag { get; set; } = HelmContainerImageTags.Tag; // ── kubectl image ───────────────────────────────────────────────────────── /// - /// Gets or sets the registry for the kubectl container image used by manifest applies. + /// Gets or sets the container registry for the kubectl image used by manifest applies. /// Defaults to docker.io. /// public string KubectlRegistry { get; set; } = KubectlContainerImageTags.Registry; /// - /// Gets or sets the kubectl container image name used by manifest applies. - /// Defaults to alpine/kubectl — maintained by the same team as k3s. + /// Gets or sets the kubectl image name used by manifest applies. + /// Defaults to alpine/kubectl. /// public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; /// - /// Gets or sets the kubectl container image tag used by manifest applies. - /// Defaults to 1.36.0, matching the default k3s server version. + /// Gets or sets the kubectl image tag used by manifest applies. + /// Defaults to 1.36.0, aligned with the default k3s server version. /// public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index 1c8c8570e..4320b183f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -4,10 +4,19 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents a k3s Kubernetes cluster running as a privileged container resource. +/// Injects the cluster's kubeconfig into a dependent resource. +/// +/// Host processes receive KUBECONFIG=<host-path>/local/kubeconfig.yaml. +/// Containers receive KUBECONFIG=/tmp/k3s-kubeconfig.yaml and a file-level +/// bind-mount of the container-network kubeconfig variant. Both are applied by the +/// BeforeStartEvent subscriber registered in AddK3sCluster. +/// +/// /// /// The resource name. [AspireExport(ExposeProperties = true)] -public sealed class K3sClusterResource(string name) : ContainerResource(name) +public sealed class K3sClusterResource(string name) + : ContainerResource(name), IResourceWithConnectionString { internal const string ApiServerEndpointName = "api"; @@ -31,6 +40,27 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) /// internal string? KubeconfigDirectory { get; set; } + // ── IResourceWithConnectionString ───────────────────────────────────────── + // Exposes the kubeconfig path as the connection string so the standard + // WithReference(cluster) overload injects KUBECONFIG for host processes. + // Containers get a bind-mount + container-network path via BeforeStartEvent. + + /// Overrides the default ConnectionStrings__ prefix so Aspire injects KUBECONFIG. + public string? ConnectionStringEnvironmentVariable => "KUBECONFIG"; + + /// Manifest expression for the local kubeconfig path. + public ReferenceExpression ConnectionStringExpression => + KubeconfigDirectory is null + ? ReferenceExpression.Create($"") + : ReferenceExpression.Create($"{KubeconfigDirectory}/local/kubeconfig.yaml"); + + /// Returns the host-accessible kubeconfig path for this cluster. + public ValueTask GetConnectionStringAsync(CancellationToken cancellationToken = default) => + ValueTask.FromResult( + KubeconfigDirectory is null + ? null + : Path.Combine(KubeconfigDirectory, "local", "kubeconfig.yaml")); + private EndpointReference? _apiEndpoint; /// Gets the endpoint reference for the k3s API server (port 6443). diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index ff6aa6e99..100a17aa9 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -35,10 +35,42 @@ public async Task CheckHealthAsync( HealthCheckContext context, CancellationToken cancellationToken = default) { - if (!endpoint.IsAllocated) - return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); + int port; - return await CheckCoreAsync(endpoint.Port, cancellationToken).ConfigureAwait(false); + if (endpoint.IsAllocated) + { + // Use the async reference-expression path — the same mechanism PostgreSQL uses + // for its connection string port. This resolves through DCP's actual allocation + // rather than the synchronous AllocatedEndpoint.Port shortcut, which for proxied + // HTTPS endpoints in Aspire 13.4.0+ may return the proxy port (= target port 6443) + // rather than the Docker host port that is actually reachable from the AppHost. + // Only call GetValueAsync when IsAllocated is true — in test/non-DCP contexts the + // call would block indefinitely waiting for an allocation that never arrives. + var portExpression = endpoint.Property(EndpointProperty.Port); + var portStr = await ((IValueProvider)portExpression) + .GetValueAsync(cancellationToken) + .ConfigureAwait(false); + + if (!int.TryParse(portStr, out port) || port <= 0) + port = endpoint.Port; // synchronous fallback within same allocation context + } + else + { + // EndpointReference.IsAllocated is false — either the endpoint has not yet been + // allocated, or DCP reconnected to a persistent container without firing the normal + // allocation event. Fall back to EndpointAnnotation.Port, which DCP updates even + // in the reconnect path. + var annotation = resource.Annotations + .OfType() + .FirstOrDefault(a => a.Name == K3sClusterResource.ApiServerEndpointName); + + if (annotation?.Port is not > 0) + return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); + + port = annotation.Port!.Value; + } + + return await CheckCoreAsync(port, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReferenceAnnotations.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReferenceAnnotations.cs new file mode 100644 index 000000000..854297cae --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReferenceAnnotations.cs @@ -0,0 +1,5 @@ +// Both K3sClusterResource and K3sServiceEndpointResource implement +// IResourceWithConnectionString. The standard Aspire WithReference overload adds a +// ResourceRelationshipAnnotation to dependents, which the BeforeStartEvent subscriber +// in AddK3sCluster uses to detect and apply container-specific overrides. +// No custom marker annotations are needed. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs index 96d53b8f8..a27a9cfdd 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -5,17 +5,19 @@ namespace Aspire.Hosting.ApplicationModel; /// -/// Exposes a Kubernetes service running inside a k3s cluster as an Aspire endpoint resource. -/// -/// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready. -/// The forwarder binds to 0.0.0.0:{hostPort} so both host processes and DCP-network -/// containers can reach the service. -/// +/// Represents a Kubernetes Service exposed from a k3s cluster as an Aspire endpoint resource. +/// +/// +/// An in-process WebSocket port-forward bridges the cluster service to a host port when the +/// cluster is ready. Dependent resources receive the service URL via the standard Aspire +/// service-discovery environment variable services__{name}__url: /// -/// Host consumers receive services__{name}__url=https://localhost:{port}. -/// Container consumers receive services__{name}__url=https://host.docker.internal:{port}. +/// Host processes receive http(s)://localhost:{port}. +/// Containers receive http(s)://host.docker.internal:{port}. /// -/// +/// Use WaitFor(endpoint) on dependent resources to ensure the port-forward is active +/// before they start. +/// [AspireExport(ExposeProperties = true)] public sealed class K3sServiceEndpointResource( string name, @@ -23,25 +25,28 @@ public sealed class K3sServiceEndpointResource( int servicePort, string @namespace, K3sClusterResource cluster) - : Resource(name), IResourceWithParent, IResourceWithWaitSupport + : Resource(name), IResourceWithParent, IResourceWithWaitSupport, + IResourceWithConnectionString { /// public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); - /// Gets the Kubernetes service name. + /// Gets the name of the Kubernetes Service being forwarded. public string ServiceName { get; } = serviceName ?? throw new ArgumentNullException(nameof(serviceName)); - /// Gets the service port number. + /// Gets the port number declared on the Kubernetes Service. public int ServicePort { get; } = servicePort; - /// Gets the Kubernetes namespace containing the service. + /// Gets the Kubernetes namespace that contains the Service. public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); /// - /// The host port allocated for the port-forward listener. - /// Set by RunEndpointAsync before the resource transitions to Running. - /// Consumers can use this to construct the service URL directly when needed. + /// Gets the host port bound by the port-forward listener. /// + /// + /// Zero until the resource transitions to Running. Use WaitFor(endpoint) + /// to ensure this is populated before reading it from a dependent resource. + /// public int HostPort { get; internal set; } /// @@ -63,4 +68,40 @@ public sealed class K3sServiceEndpointResource( /// comes first. /// internal K3sInProcessPortForwarder? Forwarder { get; set; } + + /// + /// The environment variable name used to inject the service URL into dependents + /// (services__{name}__url), following the Aspire service-discovery convention. + /// + public string? ConnectionStringEnvironmentVariable => $"services__{Name}__url"; + + /// + /// Gets the manifest expression for the service URL. + /// Resolves to http(s)://localhost:{hostPort} when the endpoint is ready. + /// + public ReferenceExpression ConnectionStringExpression + { + get + { + // ReferenceExpression.Create only accepts IManifestExpressionProvider in + // format holes, not plain value types. Pre-compute to a string first. + var url = IsReady && HostPort > 0 ? $"{Scheme}://localhost:{HostPort}" : string.Empty; + return ReferenceExpression.Create($"{url}"); + } + } + + /// + /// Returns the host-accessible service URL, or if the + /// port-forward is not yet active. + /// + /// + /// Returns until the endpoint transitions to Running. + /// Declare WaitFor(endpoint) on dependent resources so that this value is + /// always populated by the time their environment variables are evaluated. + /// + public ValueTask GetConnectionStringAsync(CancellationToken cancellationToken = default) => + ValueTask.FromResult( + IsReady && HostPort > 0 + ? $"{Scheme}://localhost:{HostPort}" + : (string?)null); } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index b09afdbf2..f757423c3 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -3,17 +3,17 @@ namespace Aspire.Hosting.ApplicationModel; /// -/// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via -/// kubectl apply --server-side running inside a alpine/kubectl container. -/// -/// The container polls for the cluster kubeconfig (written when the cluster health check -/// first passes), applies the manifests, waits for any CRDs to reach Established, -/// then exits with code 0. Use WaitForCompletion(manifest) on dependent resources. -/// +/// Represents a Kubernetes manifest (or Kustomize overlay) applied to the parent k3s cluster. /// /// The Aspire resource name. -/// Absolute path to a single YAML file or a directory. +/// Absolute path to the YAML file, plain directory, or Kustomize directory. /// The parent k3s cluster resource. +/// +/// The resource runs as an alpine/kubectl container. It polls until the cluster +/// kubeconfig is available, then applies the manifests with kubectl apply --server-side +/// and waits for any CRDs to reach the Established condition before exiting with code 0. +/// Use WaitForCompletion(manifest) on resources that depend on these manifests being applied. +/// [AspireExport(ExposeProperties = true)] public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) : ContainerResource(name), IResourceWithParent @@ -21,6 +21,9 @@ public sealed class K8sManifestResource(string name, string path, K3sClusterReso /// public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); - /// Gets the manifest path or directory on the host. + /// + /// Gets the absolute host path to the YAML file, plain directory, or Kustomize directory + /// that contains the manifests to apply. + /// public string Path { get; } = path ?? throw new ArgumentNullException(nameof(path)); } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs index b9dcf1f0c..19af80cef 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs @@ -1,4 +1,5 @@ using Aspire.Hosting; +using Aspire.Hosting.Eventing; namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; @@ -165,6 +166,8 @@ public void NegativeAgentCountIsIgnored() [Fact] public void WithLifetimePersistentPropagatestoAgentNodes() { + // ContainerLifetimeAnnotation must propagate immediately at call time — + // DCP uses it to compute container identity before BeforeStartEvent fires. var appBuilder = DistributedApplication.CreateBuilder(); appBuilder .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2) @@ -172,7 +175,6 @@ public void WithLifetimePersistentPropagatestoAgentNodes() using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var agents = model.Resources.OfType().ToList(); Assert.Equal(2, agents.Count); @@ -193,7 +195,6 @@ public void WithLifetimeSessionDoesNotAddPersistentAnnotationToAgents() using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var agent = Assert.Single(model.Resources.OfType()); var annotation = Assert.Single(agent.Annotations.OfType()); Assert.Equal(ContainerLifetime.Session, annotation.Lifetime); diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index aefe7611f..6ffa81726 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -1,6 +1,7 @@ using System.Net.Sockets; using Aspire.Hosting; using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Eventing; using CommunityToolkit.Aspire.Hosting; namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; @@ -182,13 +183,14 @@ public void AddK3sClusterWithClusterCidrViaOptions() } [Fact] - public void WithReferenceSetsKubeconfigEnvForProject() + public void WithReferenceAddsResourceRelationshipAnnotationForProject() { + // K3sClusterResource implements IResourceWithConnectionString so the standard + // Aspire WithReference overload is used — it adds a ResourceRelationshipAnnotation. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - - // ProjectResource would need a project file; use ExecutableResource as a proxy. var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + exe.WithReference(cluster); using var app = appBuilder.Build(); @@ -196,7 +198,26 @@ public void WithReferenceSetsKubeconfigEnvForProject() var exeResource = Assert.Single(model.Resources.OfType()); - // Executables receive KUBECONFIG pointing to local/kubeconfig.yaml on the host. + // Standard WithReference adds a ResourceRelationshipAnnotation pointing to the cluster. + Assert.Contains( + exeResource.Annotations.OfType(), + a => ReferenceEquals(a.Resource, cluster.Resource)); + } + + [Fact] + public void WithReferenceSetsKubeconfigEnvForProject() + { + // Standard WithReference(K3sClusterResource) uses IResourceWithConnectionString to inject + // KUBECONFIG for host processes. Verify the env callback annotation is present. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + exe.WithReference(cluster); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + var exeResource = Assert.Single(model.Resources.OfType()); + Assert.Contains( exeResource.Annotations.OfType(), a => a.Callback is not null); @@ -205,22 +226,19 @@ public void WithReferenceSetsKubeconfigEnvForProject() [Fact] public void WithReferenceMountsKubeconfigFileForContainer() { + // ApplyKubeconfigContainerOverride is what BeforeStartEvent calls for containers that + // have a ResourceRelationshipAnnotation pointing to the cluster. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - var container = appBuilder.AddContainer("operator", "myorg/operator"); - container.WithReference(cluster); + appBuilder.AddContainer("operator", "myorg/operator"); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); + var clusterResource = Assert.Single(model.Resources.OfType()); + var containerResource = model.Resources.OfType().Single(r => r.Name == "operator"); - var containerResource = model.Resources - .OfType() - .Single(r => r.Name == "operator"); + K3sBuilderExtensions.ApplyKubeconfigContainerOverride(containerResource, clusterResource); - // Containers receive a file-level bind-mount of container/kubeconfig.yaml at - // /tmp/k3s-kubeconfig.yaml (not the directory). Mounting only the file prevents - // kubectl's cache directories (cache/, http-cache/) from appearing on the host - // and avoids concurrent-container cache corruption. var mount = containerResource.Annotations .OfType() .FirstOrDefault(m => m.Target == "/tmp/k3s-kubeconfig.yaml"); @@ -228,7 +246,6 @@ public void WithReferenceMountsKubeconfigFileForContainer() Assert.NotNull(mount); Assert.Equal(ContainerMountType.BindMount, mount.Type); Assert.True(mount.IsReadOnly); - // Source is the specific file, not the directory. Assert.EndsWith(Path.Combine(".k3s", "k8s", "container", "kubeconfig.yaml"), mount.Source); } @@ -359,22 +376,21 @@ public void WithDataVolumeCalledTwiceProducesOnlyOneVolumeMount() [Fact] public void WithReferenceContainerCalledTwiceProducesOnlyOneBindMount() { + // ApplyKubeconfigContainerOverride is idempotent — the second call skips the + // mount because the target path is already mounted. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); var container = appBuilder.AddContainer("app", "myorg/app"); - container.WithReference(cluster); - container.WithReference(cluster); - using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); + var clusterResource = Assert.Single(model.Resources.OfType()); + var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); - var containerResource = model.Resources - .OfType() - .Single(r => r.Name == "app"); + // Simulate two WithReference calls via two direct invocations of the override. + K3sBuilderExtensions.ApplyKubeconfigContainerOverride(containerResource, clusterResource); + K3sBuilderExtensions.ApplyKubeconfigContainerOverride(containerResource, clusterResource); - // File-level bind-mount at /tmp/k3s-kubeconfig.yaml must not be duplicated — - // Docker rejects containers with duplicate mount targets. var kubeconfigMounts = containerResource.Annotations .OfType() .Where(m => m.Target == "/tmp/k3s-kubeconfig.yaml") @@ -388,6 +404,8 @@ public void WithReferenceContainerCalledTwiceProducesOnlyOneBindMount() [Fact] public void WithK3sVersionSyncsAllAgentImageTags() { + // Image tag must propagate immediately — DCP uses ContainerImageAnnotation to + // compute container identity before BeforeStartEvent fires. var appBuilder = DistributedApplication.CreateBuilder(); appBuilder @@ -396,7 +414,6 @@ public void WithK3sVersionSyncsAllAgentImageTags() using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var agents = model.Resources.OfType().ToList(); Assert.Equal(2, agents.Count); @@ -429,6 +446,8 @@ public void WithK3sVersionCalledTwiceAppliesLastTag() [Fact] public void WithLifetimePersistentSetsClusterLifetimeAnnotation() { + // Standard Aspire WithLifetime sets the annotation immediately + // on the cluster. Agent propagation happens in BeforeStartEvent. var appBuilder = DistributedApplication.CreateBuilder(); appBuilder.AddK3sCluster("k8s").WithLifetime(ContainerLifetime.Persistent); diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs index 3a1050853..a95249ec1 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs @@ -19,7 +19,8 @@ public async Task CheckHealthAsync_WhenEndpointNotAllocated_ReturnsUnhealthy() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - // In unit-test context there is no DCP, so endpoint.IsAllocated is always false. + // In unit-test context there is no DCP, so the port expression resolves to null/empty + // and EndpointAnnotation.Port is also unset — both paths return Unhealthy. var healthCheck = new K3sReadinessHealthCheck(cluster.Resource, cluster.Resource.ApiEndpoint); var result = await healthCheck.CheckHealthAsync(null!); @@ -28,6 +29,33 @@ public async Task CheckHealthAsync_WhenEndpointNotAllocated_ReturnsUnhealthy() Assert.Contains("not yet allocated", result.Description); } + [Fact] + public async Task CheckHealthAsync_WhenEndpointAnnotationHasPort_UsesAnnotationPortAsFallback() + { + // Simulates a persistent container reconnect: endpoint.IsAllocated is false but + // DCP has already set EndpointAnnotation.Port on the resource. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + // Manually set the allocated port on the EndpointAnnotation (DCP does this + // even when reconnecting to a persistent container). + var annotation = cluster.Resource.Annotations + .OfType() + .First(a => a.Name == K3sClusterResource.ApiServerEndpointName); + annotation.Port = 32773; + + var (hc, dir, _) = MakeCheck(writeKubeconfig: false, nodeCount: 0, agentCount: 0); + // Re-create the health check using the cluster that has the annotation set. + var healthCheck = new K3sReadinessHealthCheck(cluster.Resource, cluster.Resource.ApiEndpoint); + + var result = await healthCheck.CheckHealthAsync(null!); + + // With no kubeconfig file it returns "Waiting for k3s to write kubeconfig" + // — proving it got past the IsAllocated check and reached CheckCoreAsync. + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.DoesNotContain("not yet allocated", result.Description); + } + // ── CheckCoreAsync — file-system and Kubernetes paths ──────────────────── [Fact] diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs index 8003e97c4..cb6d85b10 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs @@ -1,5 +1,6 @@ using Aspire.Hosting; using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Eventing; using Microsoft.Extensions.Diagnostics.HealthChecks; namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; @@ -198,8 +199,10 @@ public void AddServiceEndpointAcceptsPortBoundaries(int port) // ── WithReference (service endpoint) ───────────────────────────────────── [Fact] - public void WithReferenceServiceEndpointAddsRuntimeArgToContainer() + public void WithReferenceServiceEndpointAddsResourceRelationshipAnnotation() { + // K3sServiceEndpointResource implements IResourceWithConnectionString so the + // standard Aspire WithReference overload is used — it adds a ResourceRelationshipAnnotation. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); var ep = cluster.AddServiceEndpoint("ep", "svc", 80); @@ -210,30 +213,48 @@ public void WithReferenceServiceEndpointAddsRuntimeArgToContainer() using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var containerResource = model.Resources - .OfType() - .Single(r => r.Name == "app"); + var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); + + Assert.Contains( + containerResource.Annotations.OfType(), + a => ReferenceEquals(a.Resource, ep.Resource)); + } + + [Fact] + public void WithReferenceServiceEndpointAddsRuntimeArgToContainer() + { + // ApplyServiceUrlContainerOverride adds --add-host for containers. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var ep = cluster.AddServiceEndpoint("ep", "svc", 80); + var container = appBuilder.AddContainer("app", "myorg/app"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); + + containerResource.Annotations.Add( + new ContainerRuntimeArgsCallbackAnnotation( + args => args.Add("--add-host=host.docker.internal:host-gateway"))); + K3sBuilderExtensions.ApplyServiceUrlContainerOverride(containerResource, ep.Resource); - // --add-host=host.docker.internal:host-gateway is injected for Linux Docker Engine. Assert.NotEmpty(containerResource.Annotations.OfType()); } [Fact] public void WithReferenceServiceEndpointAddsEnvironmentCallbackToContainer() { + // ApplyServiceUrlContainerOverride adds an env callback for the host.docker.internal URL. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); var ep = cluster.AddServiceEndpoint("ep", "svc", 80); var container = appBuilder.AddContainer("app", "myorg/app"); - container.WithReference(ep); - using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); + var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); - var containerResource = model.Resources - .OfType() - .Single(r => r.Name == "app"); + K3sBuilderExtensions.ApplyServiceUrlContainerOverride(containerResource, ep.Resource); Assert.Contains( containerResource.Annotations.OfType(), @@ -243,17 +264,18 @@ public void WithReferenceServiceEndpointAddsEnvironmentCallbackToContainer() [Fact] public void WithReferenceServiceEndpointAddsEnvironmentCallbackToExecutable() { + // Standard WithReference(IResourceWithConnectionString) adds a lazy ConnectionStringReference + // env callback for host processes. Verify the annotation is present. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); var ep = cluster.AddServiceEndpoint("ep", "svc", 80); var exe = appBuilder.AddExecutable("myapp", "myapp", "."); - exe.WithReference(ep); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var exeResource = Assert.Single(model.Resources.OfType()); + Assert.Contains( exeResource.Annotations.OfType(), a => a.Callback is not null); @@ -262,19 +284,18 @@ public void WithReferenceServiceEndpointAddsEnvironmentCallbackToExecutable() [Fact] public void WithReferenceServiceEndpointDoesNotAddRuntimeArgToExecutable() { + // ApplyServiceUrlContainerOverride is only called for containers; executables + // receive the URL via the standard lazy ConnectionStringReference, no --add-host needed. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); var ep = cluster.AddServiceEndpoint("ep", "svc", 80); var exe = appBuilder.AddExecutable("myapp", "myapp", "."); - exe.WithReference(ep); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var exeResource = Assert.Single(model.Resources.OfType()); - // --add-host is only relevant for containers; host processes resolve localhost directly. Assert.Empty(exeResource.Annotations.OfType()); } @@ -345,24 +366,29 @@ public void K3sServiceEndpointResourceThrowsWhenNamespaceIsNull() Assert.Throws(action); } - // ── Env callback invocation ─────────────────────────────────────────────── + // ── Env callback / IResourceWithConnectionString ────────────────────────── + // K3sServiceEndpointResource now implements IResourceWithConnectionString. + // The standard WithReference overload stores a lazy ConnectionStringReference in the + // env var (resolved at container startup by Aspire). For containers, our BeforeStartEvent + // subscriber adds a plain-string override (host.docker.internal:{port}) on top. [Fact] - public async Task WithReferenceServiceEndpoint_ContainerCallback_SetsHostDockerInternalUrlWhenReady() + public async Task WithReferenceServiceEndpoint_Container_SetsHostDockerInternalUrlWhenReady() { + // ApplyServiceUrlContainerOverride (called by BeforeStartEvent for containers) writes + // the host.docker.internal URL when the endpoint is ready. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); var ep = cluster.AddServiceEndpoint("ep", "svc", 80); - var container = appBuilder.AddContainer("app", "myorg/app"); - container.WithReference(ep); using var app = appBuilder.Build(); - var model = app.Services.GetRequiredService(); - var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); + var containerResource = new ContainerResource("app"); ep.Resource.IsReady = true; ep.Resource.HostPort = 9090; + K3sBuilderExtensions.ApplyServiceUrlContainerOverride(containerResource, ep.Resource); + var envVars = new Dictionary(); var ctx = new EnvironmentCallbackContext( new DistributedApplicationExecutionContext(DistributedApplicationOperation.Run), @@ -371,24 +397,23 @@ public async Task WithReferenceServiceEndpoint_ContainerCallback_SetsHostDockerI foreach (var cb in containerResource.Annotations.OfType()) await cb.Callback(ctx); - Assert.True(ctx.EnvironmentVariables.ContainsKey("services__ep__url")); - Assert.Equal("http://host.docker.internal:9090", ctx.EnvironmentVariables["services__ep__url"]?.ToString()); + Assert.Equal("http://host.docker.internal:9090", + ctx.EnvironmentVariables["services__ep__url"]?.ToString()); } [Fact] - public async Task WithReferenceServiceEndpoint_ContainerCallback_DoesNotSetUrlWhenNotReady() + public async Task WithReferenceServiceEndpoint_Container_DoesNotOverrideUrlWhenNotReady() { + // When IsReady=false the override callback skips writing the URL. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); var ep = cluster.AddServiceEndpoint("ep", "svc", 80); - var container = appBuilder.AddContainer("app", "myorg/app"); - container.WithReference(ep); using var app = appBuilder.Build(); - var model = app.Services.GetRequiredService(); - var containerResource = model.Resources.OfType().Single(r => r.Name == "app"); + var containerResource = new ContainerResource("app"); - // ep.IsReady is false by default + // ep.IsReady is false by default. + K3sBuilderExtensions.ApplyServiceUrlContainerOverride(containerResource, ep.Resource); var envVars = new Dictionary(); var ctx = new EnvironmentCallbackContext( @@ -402,7 +427,7 @@ public async Task WithReferenceServiceEndpoint_ContainerCallback_DoesNotSetUrlWh } [Fact] - public async Task WithReferenceServiceEndpoint_ExecutableCallback_SetsLocalhostUrlWhenReady() + public async Task WithReferenceServiceEndpoint_Executable_SetsLocalhostUrlWhenReady() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); @@ -425,12 +450,18 @@ public async Task WithReferenceServiceEndpoint_ExecutableCallback_SetsLocalhostU foreach (var cb in exeResource.Annotations.OfType()) await cb.Callback(ctx); - Assert.True(ctx.EnvironmentVariables.ContainsKey("services__ep__url")); - Assert.Equal("https://localhost:7777", ctx.EnvironmentVariables["services__ep__url"]?.ToString()); + // For host processes, standard WithReference stores a ConnectionStringReference that + // lazily resolves via K3sServiceEndpointResource.GetConnectionStringAsync(). + var rawValue = ctx.EnvironmentVariables.GetValueOrDefault("services__ep__url"); + var resolved = rawValue is IValueProvider vp + ? await vp.GetValueAsync(CancellationToken.None) + : rawValue?.ToString(); + + Assert.Equal("https://localhost:7777", resolved); } [Fact] - public async Task WithReferenceServiceEndpoint_ExecutableCallback_DoesNotSetUrlWhenNotReady() + public async Task WithReferenceServiceEndpoint_Executable_ReturnsNullUrlWhenNotReady() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); @@ -442,7 +473,7 @@ public async Task WithReferenceServiceEndpoint_ExecutableCallback_DoesNotSetUrlW var model = app.Services.GetRequiredService(); var exeResource = Assert.Single(model.Resources.OfType()); - // ep.IsReady is false by default + // ep.IsReady is false by default. var envVars = new Dictionary(); var ctx = new EnvironmentCallbackContext( @@ -452,6 +483,11 @@ public async Task WithReferenceServiceEndpoint_ExecutableCallback_DoesNotSetUrlW foreach (var cb in exeResource.Annotations.OfType()) await cb.Callback(ctx); - Assert.False(ctx.EnvironmentVariables.ContainsKey("services__ep__url")); + var rawValue = ctx.EnvironmentVariables.GetValueOrDefault("services__ep__url"); + var resolved = rawValue is IValueProvider vp + ? await vp.GetValueAsync(CancellationToken.None) + : rawValue?.ToString(); + + Assert.Null(resolved); } } From 63f7e027fe9018f7e41e0fbf4e26882567c22e9f Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Tue, 2 Jun 2026 15:13:26 +0200 Subject: [PATCH 22/29] chore: upgrade example app host sdk to match --- .../CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index eb6d3ca1e..50ee2b3d0 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe From 7eaae5fbc1bef8fcac957286f292cc90590a1161 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Tue, 2 Jun 2026 19:13:25 +0200 Subject: [PATCH 23/29] fix: improve api to simplify polyglot sdk --- .gitignore | 5 +- .../Program.cs | 6 +- .../ValidationAppHost/apphost.mts | 110 ++ .../ValidationAppHost/apphost.ts | 82 -- .../ValidationAppHost/aspire.config.json | 19 +- .../ValidationAppHost/package-lock.json | 938 ++++++++++++++++++ .../ValidationAppHost/package.json | 5 +- .../ValidationAppHost/tsconfig.json | 4 +- .../K3sBuilderExtensions.cs | 237 +++-- .../K3sClusterOptions.cs | 95 +- .../K3sAgentNodeTests.cs | 121 ++- .../K3sClusterOptionsTests.cs | 86 ++ .../K3sClusterResourceTests.cs | 171 +++- .../K3sServiceEndpointResourceTests.cs | 61 ++ 14 files changed, 1657 insertions(+), 283 deletions(-) create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.mts delete mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package-lock.json create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterOptionsTests.cs diff --git a/.gitignore b/.gitignore index 3534c8b2f..2b2c300e0 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,7 @@ apphost.js affected-tests examples/**/.modules/* examples/**/.aspire/modules/* -examples/**/.aspire/integrations/* \ No newline at end of file +examples/**/.aspire/integrations/* +playground/**/.modules/* +playground/**/.aspire/modules/* +playground/**/.aspire/integrations/* \ No newline at end of file diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs index 19220c7bc..61549964e 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -21,10 +21,8 @@ var builder = DistributedApplication.CreateBuilder(args); var cluster = builder - .AddK3sCluster("k8s", configure: cfg => - { - cfg.AgentCount = 2; - }) + .AddK3sCluster("k8s") + .WithAgentCount(2) .WithDataVolume() .WithLifetime(ContainerLifetime.Persistent); diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.mts b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.mts new file mode 100644 index 000000000..9b0107d3f --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.mts @@ -0,0 +1,110 @@ +import { createBuilder, ContainerLifetime } from './.aspire/modules/aspire.mjs'; + +const builder = await createBuilder(); + +// ── Runtime path (actually executed) ───────────────────────────────────────── +// Deploys a k3s cluster with 2 agent nodes, installs podinfo via Helm, and +// exposes the podinfo service as an Aspire endpoint — validating the full +// add/build/run path without relying on the configure callback. +const cluster = await builder.addK3sCluster('k8s') + .withAgentCount(2); + +const _apiEndpoint = await cluster.apiEndpoint(); + +const setupPodinfo = await cluster.addHelmRelease('podinfo', 'podinfo', { + repo: 'https://stefanprodan.github.io/podinfo', + version: '6.7.1', + namespace: 'podinfo', +}); + +await cluster.addServiceEndpoint('podinfo-web', 'podinfo', 9898, { + namespace: 'podinfo', +}).waitForCompletion(setupPodinfo); + +// ── Compile-time coverage ───────────────────────────────────────────────────── +// Guards with false so these are type-checked but never executed. +// Covers the full exported API surface without requiring Docker/k3s in CI. +const includeCompileOnlyScenarios = false; + +if (includeCompileOnlyScenarios) { + + // ── Cluster configuration ──────────────────────────────────────────────── + // All K3sClusterOptions are now available as fluent builder methods. + const configuredCluster = await builder.addK3sCluster('k8s-configured', { + apiServerPort: 6443, + agentCount: 2 + }) + .withK3sVersion('v1.32.3-k3s1') + .withPodSubnet('10.42.0.0/16') + .withServiceSubnet('10.43.0.0/16') + .withDisabledComponent('traefik') + .withExtraArg('--write-kubeconfig-mode=644') + .withDataVolume({ name: 'k8s-data' }) + .withHelmImage({ tag: '3.18.0' }) + .withKubectlImage({ tag: '1.37.0' }) + .withLifetime(ContainerLifetime.Persistent); + + const _configuredApiEndpoint = await configuredCluster.apiEndpoint(); + + // ── Helm release — podinfo ──────────────────────────────────────────────── + const podinfo = await configuredCluster.addHelmRelease('podinfo', 'podinfo', { + repo: 'https://stefanprodan.github.io/podinfo', + version: '6.7.1', + namespace: 'podinfo', + }); + + const _podinfoParent = await podinfo.parent(); + const _podinfoReleaseName = await podinfo.releaseName(); + const _podinfoNamespace = await podinfo.namespace(); + + // ── K8s manifest — plain YAML file ─────────────────────────────────────── + const appConfig = await configuredCluster + .addK8sManifest('app-config', './k8s/app-config.yaml') + .waitForCompletion(podinfo); + + const _appConfigParent = await appConfig.parent(); + const _appConfigPath = await appConfig.path(); + + // ── K8s manifest — Kustomize overlay ───────────────────────────────────── + // Auto-detected because the directory contains kustomization.yaml. + const monitoringConfig = await configuredCluster + .addK8sManifest('monitoring-config', './k8s/monitoring') + .waitForCompletion(podinfo) + .waitForCompletion(appConfig); + + const _monitoringParent = await monitoringConfig.parent(); + + // ── Service endpoint ───────────────────────────────────────────────────── + // • Host processes receive services__podinfo-web__url=http://localhost:{port} + // • DCP containers receive services__podinfo-web__url=http://host.docker.internal:{port} + const podinfoWeb = await configuredCluster + .addServiceEndpoint('podinfo-web', 'podinfo', 9898, { + namespace: 'podinfo', + }) + .waitForCompletion(podinfo) + .waitForCompletion(monitoringConfig); + + const _webParent = await podinfoWeb.parent(); + const _webServiceName = await podinfoWeb.serviceName(); + const _webServicePort = await podinfoWeb.servicePort(); + const _webNamespace = await podinfoWeb.namespace(); + const _webHostPort = await podinfoWeb.hostPort.get(); + + // ── WithReference — cluster kubeconfig injection ────────────────────────── + // • Project/executable: KUBECONFIG=/.k3s/k8s/local/kubeconfig.yaml + // • Container: KUBECONFIG=/tmp/k3s-kubeconfig.yaml + bind-mount + const _projectRef = await builder + .addProject('operator', '../WidgetOperator/WidgetOperator.csproj') + .withReference(configuredCluster); + + const _containerRef = await builder + .addContainer('sidecar', 'myorg/sidecar') + .withReference(configuredCluster); + + // ── WithReference — service endpoint URL injection ──────────────────────── + const _endpointRef = await builder + .addProject('api', '../WidgetApi/WidgetApi.csproj') + .withReference(podinfoWeb); +} + +await builder.build().run(); diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts deleted file mode 100644 index 1085187e0..000000000 --- a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { createBuilder, ContainerLifetime } from './.modules/aspire.js'; - -const builder = await createBuilder(); - -// ── Runtime path (actually executed) ───────────────────────────────────────── -// Minimal cluster startup — validates that the core add/build/run path works. -const cluster = builder.addK3sCluster('k8s'); -const clusterResource = await cluster; -const _apiEndpoint = await clusterResource.apiEndpoint.get(); - -// ── Compile-time coverage ───────────────────────────────────────────────────── -// Guards with false so these are type-checked but never executed. -// Covers the full exported API surface without requiring Docker/k3s in CI. -const includeCompileOnlyScenarios = false; - -if (includeCompileOnlyScenarios) { - - // ── Cluster configuration ──────────────────────────────────────────────── - const configuredCluster = builder.addK3sCluster('k8s-configured') - .withK3sVersion('v1.32.3-k3s1') - .withPodSubnet('10.42.0.0/16') - .withServiceSubnet('10.43.0.0/16') - .withDisabledComponent('traefik') - .withExtraArg('--write-kubeconfig-mode=644') - .withDataVolume({ name: 'k8s-data' }) - .withLifetime(ContainerLifetime.Persistent); - - const configuredClusterResource = await configuredCluster; - const _configuredApiEndpoint = await configuredClusterResource.apiEndpoint.get(); - - // ── Helm release ───────────────────────────────────────────────────────── - const argocd = configuredCluster.addHelmRelease('argocd', 'argo-cd', { - repo: 'https://argoproj.github.io/argo-helm', - version: '7.8.0', - namespace: 'argocd', - }) - .withHelmValue('server.insecure', 'true') - .withHelmValuesFile('./deploy/argocd-values.yaml'); - - const argocdResource = await argocd; - const _argocdParent = await argocdResource.parent.get(); - const _argocdReleaseName = await argocdResource.releaseName.get(); - const _argocdNamespace = await argocdResource.namespace.get(); - - // ── K8s manifest / Kustomize overlay ───────────────────────────────────── - const widgetCrd = configuredCluster.addK8sManifest('widget-crd', './k8s/crds/'); - - const widgetCrdResource = await widgetCrd; - const _crdParent = await widgetCrdResource.parent.get(); - const _crdPath = await widgetCrdResource.path.get(); - - // ── Service endpoint ───────────────────────────────────────────────────── - const ui = configuredCluster.addServiceEndpoint('argocd-ui', 'argocd-server', 443, { - namespace: 'argocd', - }); - - const uiResource = await ui; - const _uiParent = await uiResource.parent.get(); - const _uiServiceName = await uiResource.serviceName.get(); - const _uiServicePort = await uiResource.servicePort.get(); - const _uiNamespace = await uiResource.namespace.get(); - const _uiHostPort = await uiResource.hostPort.get(); - - // ── WithReference — cluster kubeconfig injection ────────────────────────── - // Project: receives KUBECONFIG=.../.k3s/k8s/local/kubeconfig.yaml - const _projectRef = builder - .addProject('operator', { projectPath: '../WidgetOperator/WidgetOperator.csproj' }) - .withReference(configuredCluster); - - // Container: receives a bind-mounted kubeconfig and KUBECONFIG=/var/k3s/kubeconfig.yaml - const _containerRef = builder - .addContainer('sidecar', 'myorg/sidecar') - .withReference(configuredCluster); - - // ── WithReference — service endpoint URL injection ──────────────────────── - // Host: receives services__argocd-ui__url=https://localhost:{port} - const _endpointRef = builder - .addProject('api', { projectPath: '../WidgetApi/WidgetApi.csproj' }) - .withReference(ui); -} - -await builder.build().run(); diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json index b12c66e70..205fa756d 100644 --- a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json @@ -1,25 +1,8 @@ { "appHost": { - "path": "apphost.ts", + "path": "apphost.mts", "language": "typescript/nodejs" }, - "profiles": { - "https": { - "applicationUrl": "https://localhost:17149;http://localhost:15243", - "environmentVariables": { - "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21065", - "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22170" - } - }, - "http": { - "applicationUrl": "http://localhost:15243", - "environmentVariables": { - "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19027", - "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20141", - "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" - } - } - }, "packages": { "CommunityToolkit.Aspire.Hosting.K3s": "../../../../../src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj" } diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package-lock.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package-lock.json new file mode 100644 index 000000000..85365bff4 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package-lock.json @@ -0,0 +1,938 @@ +{ + "name": "validationapphost", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "validationapphost", + "version": "1.0.0", + "dependencies": { + "vscode-jsonrpc": "^8.2.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "nodemon": "^3.1.11", + "tsx": "^4.19.0", + "typescript": "^5.3.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.0.tgz", + "integrity": "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.0.tgz", + "integrity": "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.0.tgz", + "integrity": "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.0.tgz", + "integrity": "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.0.tgz", + "integrity": "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.0.tgz", + "integrity": "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.0.tgz", + "integrity": "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.0.tgz", + "integrity": "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.0.tgz", + "integrity": "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.0.tgz", + "integrity": "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.0.tgz", + "integrity": "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.0.tgz", + "integrity": "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.0.tgz", + "integrity": "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.0.tgz", + "integrity": "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.0.tgz", + "integrity": "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.0.tgz", + "integrity": "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz", + "integrity": "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.0.tgz", + "integrity": "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.0.tgz", + "integrity": "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.0.tgz", + "integrity": "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.0.tgz", + "integrity": "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.0.tgz", + "integrity": "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.0.tgz", + "integrity": "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.0.tgz", + "integrity": "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.0.tgz", + "integrity": "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.0.tgz", + "integrity": "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "20.19.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.41.tgz", + "integrity": "sha512-ECymXOukMnOoVkC2bb1Vc/w/836DXncOg5m8Xj1RH7xSHZJWNYY6Zh7EH477vcnD5egKNNfy2RpNOmuChhFPgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/esbuild": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz", + "integrity": "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.0", + "@esbuild/android-arm": "0.28.0", + "@esbuild/android-arm64": "0.28.0", + "@esbuild/android-x64": "0.28.0", + "@esbuild/darwin-arm64": "0.28.0", + "@esbuild/darwin-x64": "0.28.0", + "@esbuild/freebsd-arm64": "0.28.0", + "@esbuild/freebsd-x64": "0.28.0", + "@esbuild/linux-arm": "0.28.0", + "@esbuild/linux-arm64": "0.28.0", + "@esbuild/linux-ia32": "0.28.0", + "@esbuild/linux-loong64": "0.28.0", + "@esbuild/linux-mips64el": "0.28.0", + "@esbuild/linux-ppc64": "0.28.0", + "@esbuild/linux-riscv64": "0.28.0", + "@esbuild/linux-s390x": "0.28.0", + "@esbuild/linux-x64": "0.28.0", + "@esbuild/netbsd-arm64": "0.28.0", + "@esbuild/netbsd-x64": "0.28.0", + "@esbuild/openbsd-arm64": "0.28.0", + "@esbuild/openbsd-x64": "0.28.0", + "@esbuild/openharmony-arm64": "0.28.0", + "@esbuild/sunos-x64": "0.28.0", + "@esbuild/win32-arm64": "0.28.0", + "@esbuild/win32-ia32": "0.28.0", + "@esbuild/win32-x64": "0.28.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nodemon": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.14.tgz", + "integrity": "sha512-jakjZi93UtB3jHMWsXL68FXSAosbLfY0In5gtKq3niLSkrWznrVBzXFNOEMJUfc9+Ke7SHWoAZsiMkNP3vq6Jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^10.2.1", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true, + "license": "MIT" + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/semver": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz", + "integrity": "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "license": "ISC", + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, + "node_modules/tsx": { + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", + "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true, + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + } + } +} diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json index 54f1c98e3..277aa4582 100644 --- a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json @@ -3,9 +3,8 @@ "version": "1.0.0", "type": "module", "scripts": { - "start": "aspire run", - "build": "tsc", - "dev": "tsc --watch" + "build": "tsc --noEmit", + "dev": "tsc --noEmit --watch" }, "dependencies": { "vscode-jsonrpc": "^8.2.0" diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json index 05c7cf2f8..19805ab48 100644 --- a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json @@ -11,8 +11,8 @@ "rootDir": "." }, "include": [ - "apphost.ts", - ".modules/**/*.ts" + "apphost.mts", + ".aspire/modules/**/*.mts" ], "exclude": [ "node_modules" diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index fabb43d11..cb0f3a70e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -25,9 +25,10 @@ public static class K3sBuilderExtensions /// Host port to bind the Kubernetes API server (port 6443) to. /// When (the default) a random available port is assigned. /// - /// - /// Optional callback to configure agent count, image versions, custom CIDR ranges, - /// disabled components, and other cluster options. See . + /// + /// Number of k3s agent (worker) nodes to add. When (the default) + /// a single-node cluster is created — the server node acts as both control-plane and worker. + /// Equivalent to calling on the returned builder. /// /// A builder for the . /// @@ -45,29 +46,33 @@ public static class K3sBuilderExtensions /// Call WithReference(cluster) on a dependent resource builder to inject /// these credentials automatically. /// + /// + /// All other cluster options are available as fluent builder methods: + /// , , , + /// , , + /// , , , + /// , and . + /// /// /// /// or is . /// - [AspireExport(RunSyncOnBackgroundThread = true)] + [AspireExport] public static IResourceBuilder AddK3sCluster( this IDistributedApplicationBuilder builder, [ResourceName] string name, int? apiServerPort = null, - Action? configure = null) + int? agentCount = null) { ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(name); - var options = new K3sClusterOptions(); - configure?.Invoke(options); - var resource = new K3sClusterResource(name) { - HelmImageInfo = (options.HelmRegistry, options.HelmImage, options.HelmTag), - KubectlImageInfo = (options.KubectlRegistry, options.KubectlImage, options.KubectlTag), + HelmImageInfo = (HelmContainerImageTags.Registry, HelmContainerImageTags.Image, HelmContainerImageTags.Tag), + KubectlImageInfo = (KubectlContainerImageTags.Registry, KubectlContainerImageTags.Image, KubectlContainerImageTags.Tag), }; - var tag = options.ImageTag ?? K3sContainerImageTags.Tag; + var tag = K3sContainerImageTags.Tag; // ── Kubeconfig directory on the host ────────────────────────────────── // AppHostDirectory/.k3s/{name}/ holds three sub-directories: @@ -169,71 +174,9 @@ public static IResourceBuilder AddK3sCluster( .WithIconName("Kubernetes"); - if (options.ClusterCidr is not null) - { - resourceBuilder.WithArgs($"--cluster-cidr={options.ClusterCidr}"); - } - - if (options.ServiceCidr is not null) - { - resourceBuilder.WithArgs($"--service-cidr={options.ServiceCidr}"); - } - - foreach (var component in options.DisabledComponents) - { - resourceBuilder.WithArgs($"--disable={component}"); - } - - foreach (var arg in options.ExtraArgs) - { - resourceBuilder.WithArgs(arg); - } - - // Create agent nodes specified via options.AgentCount. - // Agents use DCP DNS: K3S_URL=https://{name}:6443 resolves to the server container. - // NO WaitFor — k3s agent retries indefinitely until the server is reachable. - // This avoids a deadlock where the cluster health check waits for nodes to be Ready - // while nodes wait for the cluster to be healthy. - for (var i = 0; i < options.AgentCount; i++) - { - resource.AgentCount++; - var agentName = $"{name}-agent-{i}"; - var agentResource = new K3sAgentResource(agentName, resource); - resource.AddAgentResource(agentResource); - - builder.AddResource(agentResource) - .WithImage(K3sContainerImageTags.Image, tag) - .WithImageRegistry(K3sContainerImageTags.Registry) - .WithContainerFiles("/", [new ContainerFile - { - Name = "aspire-k3s-entrypoint.sh", - Contents = K3sInitEntrypointScript, - Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute - | UnixFileMode.GroupRead | UnixFileMode.GroupExecute - | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, - }]) - .WithEntrypoint("/bin/sh") - .WithArgs("/aspire-k3s-entrypoint.sh") - .WithArgs("agent") - .WithArgs("-v", "0") - .WithArgs("--kubelet-arg=v=0") - .WithEnvironment("K3S_URL", $"https://{name}:6443") - .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") - .WithEnvironment("K3S_NODE_NAME", agentName) - .WithContainerRuntimeArgs("--privileged") - .WithContainerRuntimeArgs("--init") - .WithContainerRuntimeArgs("--userns=host") - .WithContainerRuntimeArgs("--cgroupns=host") - .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") - .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") - .ExcludeFromManifest() - .WithInitialState(new CustomResourceSnapshot - { - ResourceType = "K3s Agent", - State = KnownResourceStates.Starting, - Properties = [new ResourcePropertySnapshot("Cluster", name)], - }); - } + // Create agent nodes if agentCount was supplied directly to AddK3sCluster. + if (agentCount is > 0) + AddAgentNodes(resourceBuilder, agentCount.Value, tag); resourceBuilder.WithHealthCheck($"k3s_{name}_ready"); @@ -499,6 +442,89 @@ public static IResourceBuilder WithDataVolume( .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s"); } + /// + /// Sets the number of k3s agent (worker) nodes to add to the cluster. + /// + /// The k3s cluster resource builder. + /// + /// The number of agent nodes. Zero or greater. Defaults to 0 (single-node cluster — + /// the server node acts as both control-plane and worker). + /// + /// The same builder, for chaining. + /// + /// Agent nodes connect to the server via DCP DNS (https://{name}:6443) and use + /// k3s's built-in retry loop, so no explicit WaitFor is needed. The cluster health + /// check waits for 1 + count nodes to reach Ready state before reporting + /// healthy. Use with + /// to keep agents alive across AppHost restarts and avoid node password hash mismatches. + /// + /// is . + /// is negative. + [AspireExport] + public static IResourceBuilder WithAgentCount( + this IResourceBuilder builder, + int count) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentOutOfRangeException.ThrowIfNegative(count, nameof(count)); + + if (count == 0) return builder; + + // Use the tag already set on the cluster (either the default or from WithK3sVersion). + var tag = builder.Resource.Annotations + .OfType() + .FirstOrDefault()?.Tag ?? K3sContainerImageTags.Tag; + + AddAgentNodes(builder, count, tag); + return builder; + } + + /// + /// Overrides the container image used to run helm upgrade --install for + /// all children of this cluster. + /// + /// The k3s cluster resource builder. + /// Image tag, e.g. 3.18.0. keeps the current value. + /// Image name, e.g. alpine/helm. keeps the current value. + /// Registry, e.g. docker.io. keeps the current value. + /// The same builder, for chaining. + /// is . + [AspireExport] + public static IResourceBuilder WithHelmImage( + this IResourceBuilder builder, + string? tag = null, + string? image = null, + string? registry = null) + { + ArgumentNullException.ThrowIfNull(builder); + var (r, i, t) = builder.Resource.HelmImageInfo; + builder.Resource.HelmImageInfo = (registry ?? r, image ?? i, tag ?? t); + return builder; + } + + /// + /// Overrides the container image used to run kubectl apply for all + /// children of this cluster. + /// + /// The k3s cluster resource builder. + /// Image tag, e.g. 1.37.0. keeps the current value. + /// Image name, e.g. alpine/kubectl. keeps the current value. + /// Registry, e.g. docker.io. keeps the current value. + /// The same builder, for chaining. + /// is . + [AspireExport] + public static IResourceBuilder WithKubectlImage( + this IResourceBuilder builder, + string? tag = null, + string? image = null, + string? registry = null) + { + ArgumentNullException.ThrowIfNull(builder); + var (r, i, t) = builder.Resource.KubectlImageInfo; + builder.Resource.KubectlImageInfo = (registry ?? r, image ?? i, tag ?? t); + return builder; + } + /// /// Sets the container lifetime for the k3s cluster and all its agent nodes. /// @@ -506,13 +532,11 @@ public static IResourceBuilder WithDataVolume( /// The container lifetime to apply. /// The same builder, for chaining. /// - /// - /// Agent nodes are propagated immediately because DCP uses the + /// Agent nodes are propagated immediately because DCP uses /// to compute container identity. Deferring /// propagation to BeforeStartEvent would be too late — DCP determines whether /// to reuse or recreate a persistent container before that event fires, so agents /// would lose their persistent identity and be recreated as new containers each run. - /// /// /// is . [AspireExport] @@ -573,6 +597,61 @@ busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.pr exec k3s "$@" """; + // ── Agent node creation ─────────────────────────────────────────────────── + + // Shared by AddK3sCluster (via options.AgentCount) and WithAgentCount. + // Agents use DCP DNS (K3S_URL=https://{name}:6443) and retry until the server is + // reachable — no WaitFor to avoid a deadlock where the cluster health check waits + // for nodes to be Ready while nodes wait for the cluster to be healthy. + private static void AddAgentNodes( + IResourceBuilder clusterBuilder, + int count, + string tag) + { + var resource = clusterBuilder.Resource; + var name = resource.Name; + var startIndex = resource.AgentCount; + + for (var i = startIndex; i < startIndex + count; i++) + { + resource.AgentCount++; + var agentName = $"{name}-agent-{i}"; + var agentResource = new K3sAgentResource(agentName, resource); + resource.AddAgentResource(agentResource); + + clusterBuilder.ApplicationBuilder.AddResource(agentResource) + .WithImage(K3sContainerImageTags.Image, tag) + .WithImageRegistry(K3sContainerImageTags.Registry) + .WithContainerFiles("/", [new ContainerFile + { + Name = "aspire-k3s-entrypoint.sh", + Contents = K3sInitEntrypointScript, + Mode = K3sFileHelpers.ExecutableScriptMode, + }]) + .WithEntrypoint("/bin/sh") + .WithArgs("/aspire-k3s-entrypoint.sh") + .WithArgs("agent") + .WithArgs("-v", "0") + .WithArgs("--kubelet-arg=v=0") + .WithEnvironment("K3S_URL", $"https://{name}:6443") + .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") + .WithEnvironment("K3S_NODE_NAME", agentName) + .WithContainerRuntimeArgs("--privileged") + .WithContainerRuntimeArgs("--init") + .WithContainerRuntimeArgs("--userns=host") + .WithContainerRuntimeArgs("--cgroupns=host") + .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") + .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K3s Agent", + State = KnownResourceStates.Starting, + Properties = [new ResourcePropertySnapshot("Cluster", name)], + }); + } + } + // ── BeforeStartEvent helpers ────────────────────────────────────────────── // Called only for containers — standard WithReference(IResourceWithConnectionString) already diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs index 81b06195d..392a9d49b 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -1,115 +1,86 @@ -#pragma warning disable ASPIREATS001 // AspireExport is experimental - using Aspire.Hosting; namespace CommunityToolkit.Aspire.Hosting; /// -/// Configuration options passed to AddK3sCluster via its configure callback. +/// Advanced options for . /// /// -/// All settings are optional. Defaults produce a single-node cluster using the package's -/// bundled k3s, alpine/helm, and alpine/kubectl image versions. +/// +/// Most settings are exposed directly as fluent builder methods — prefer those for +/// discoverability and TypeScript polyglot compatibility: +/// +/// +/// — number of worker nodes +/// — k3s image tag +/// — pod CIDR (--cluster-cidr) +/// — service CIDR (--service-cidr) +/// — disable built-in components +/// — raw k3s server arguments +/// — Helm installer image +/// — kubectl image +/// — persistent data volume +/// — container lifetime +/// /// -[AspireExport(ExposeProperties = true)] public sealed class K3sClusterOptions { /// - /// Gets or sets the pod IP address range in CIDR notation, passed as - /// --cluster-cidr to the k3s server. + /// Gets or sets the pod IP address range in CIDR notation (--cluster-cidr). + /// Defaults to k3s built-in value of 10.42.0.0/16 when . /// - /// - /// When (the default), k3s uses its built-in default of - /// 10.42.0.0/16. Can also be set fluently via WithPodSubnet. - /// public string? ClusterCidr { get; set; } /// - /// Gets or sets the Service cluster IP address range in CIDR notation, passed as - /// --service-cidr to the k3s server. + /// Gets or sets the Service cluster IP address range in CIDR notation (--service-cidr). + /// Defaults to k3s built-in value of 10.43.0.0/16 when . /// - /// - /// When (the default), k3s uses its built-in default of - /// 10.43.0.0/16. Can also be set fluently via WithServiceSubnet. - /// public string? ServiceCidr { get; set; } /// - /// Gets the list of built-in k3s components to disable. - /// Each entry is passed as --disable=<component> to the k3s server. + /// Gets the list of built-in k3s components to disable (each passed as --disable=<component>). /// /// - /// Common values: traefik, servicelb, metrics-server, - /// coredns, local-storage. Note that servicelb and - /// metrics-server are already disabled by default for faster cluster startup. + /// Common values: traefik, coredns, local-storage. + /// Note that servicelb and metrics-server are already disabled by default. /// public IList DisabledComponents { get; } = new List(); /// - /// Gets the list of raw arguments appended to the k3s server command line. + /// Gets the list of raw arguments appended verbatim to the k3s server command. /// - /// - /// Use this for flags that have no dedicated option in . - /// Prefer the typed properties and the fluent extension methods (WithPodSubnet, - /// WithDisabledComponent, etc.) when available. - /// public IList ExtraArgs { get; } = new List(); /// - /// Gets or sets the number of agent (worker) nodes to add alongside the server. - /// Defaults to 0, which produces a single-node cluster. + /// Gets or sets the number of agent (worker) nodes. Defaults to 0 (single-node cluster). /// - /// - /// Each agent runs as a separate container and joins the server automatically. - /// The cluster health check waits for all 1 + AgentCount nodes to be - /// Ready before the cluster resource transitions to Running. - /// public int AgentCount { get; set; } /// /// Gets or sets the k3s container image tag, e.g. v1.32.3-k3s1. - /// When (the default), the version bundled with this package is used. + /// When the version bundled with this package is used. /// - /// - /// Can also be set fluently after AddK3sCluster via WithK3sVersion. - /// public string? ImageTag { get; set; } // ── Helm installer image ────────────────────────────────────────────────── - /// - /// Gets or sets the container registry for the Helm installer image. - /// Defaults to docker.io. - /// + /// Gets or sets the container registry for the Helm installer image. Defaults to docker.io. public string HelmRegistry { get; set; } = HelmContainerImageTags.Registry; - /// - /// Gets or sets the Helm installer image name. Defaults to alpine/helm. - /// + /// Gets or sets the Helm installer image name. Defaults to alpine/helm. public string HelmImage { get; set; } = HelmContainerImageTags.Image; - /// - /// Gets or sets the Helm installer image tag. Defaults to 3.17.3. - /// + /// Gets or sets the Helm installer image tag. Defaults to 3.17.3. public string HelmTag { get; set; } = HelmContainerImageTags.Tag; // ── kubectl image ───────────────────────────────────────────────────────── - /// - /// Gets or sets the container registry for the kubectl image used by manifest applies. - /// Defaults to docker.io. - /// + /// Gets or sets the container registry for the kubectl image. Defaults to docker.io. public string KubectlRegistry { get; set; } = KubectlContainerImageTags.Registry; - /// - /// Gets or sets the kubectl image name used by manifest applies. - /// Defaults to alpine/kubectl. - /// + /// Gets or sets the kubectl image name. Defaults to alpine/kubectl. public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; - /// - /// Gets or sets the kubectl image tag used by manifest applies. - /// Defaults to 1.36.0, aligned with the default k3s server version. - /// + /// Gets or sets the kubectl image tag. Defaults to 1.36.0. public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs index 19af80cef..c0f5eea16 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs @@ -12,7 +12,7 @@ public void AgentCountInOptionsCreatesK3sAgentResources() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2); + appBuilder.AddK3sCluster("k8s", agentCount: 2); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -29,7 +29,7 @@ public void AgentNodesAreChildrenOfCluster() { // Implements IResourceWithParent so they appear nested under k8s in the dashboard. var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + var cluster = appBuilder.AddK3sCluster("k8s", agentCount: 1); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -47,7 +47,7 @@ public void AgentNodesAreChildrenOfCluster() public void AgentCountZeroProducesNoAgentResources() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 0); + appBuilder.AddK3sCluster("k8s"); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -59,7 +59,7 @@ public void AgentCountZeroProducesNoAgentResources() public void AgentCountUpdatesClusterAgentCount() { var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 3); + var cluster = appBuilder.AddK3sCluster("k8s", agentCount: 3); Assert.Equal(3, cluster.Resource.AgentCount); } @@ -71,7 +71,7 @@ public void AgentNodesDoNotHaveWaitForDependencyOnCluster() // cluster health check waits for all nodes (including agents) to be Ready. // Instead, k3s agent retries connecting to the server independently. var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + appBuilder.AddK3sCluster("k8s", agentCount: 1); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -86,7 +86,7 @@ public void AgentNodesDoNotHaveWaitForDependencyOnCluster() public void AgentNodesUseSameImageAsServer() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + appBuilder.AddK3sCluster("k8s", agentCount: 1); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -101,7 +101,7 @@ public void AgentNodesUseSameImageAsServer() public void AgentNodesAreExcludedFromManifest() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + appBuilder.AddK3sCluster("k8s", agentCount: 1); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -114,7 +114,7 @@ public void AgentNodesAreExcludedFromManifest() public void AgentNodesHaveEnvironmentAnnotations() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + appBuilder.AddK3sCluster("k8s", agentCount: 1); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -141,7 +141,7 @@ public void DefaultClusterHasNoAgentNodes() public void AgentNodeNamesFollowConvention() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("mycluster", configure: opts => opts.AgentCount = 3); + appBuilder.AddK3sCluster("mycluster", agentCount: 3); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -155,7 +155,34 @@ public void AgentNodeNamesFollowConvention() public void NegativeAgentCountIsIgnored() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = -1); + appBuilder.AddK3sCluster("k8s", agentCount: -1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + // ── WithAgentCount ──────────────────────────────────────────────────────── + + [Fact] + public void WithAgentCountAddsCorrectNumberOfAgentResources() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s").WithAgentCount(3); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + Assert.Equal(3, agents.Count); + } + + [Fact] + public void WithAgentCountZeroIsNoOp() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s").WithAgentCount(0); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -163,6 +190,76 @@ public void NegativeAgentCountIsIgnored() Assert.Empty(model.Resources.OfType()); } + [Fact] + public void WithAgentCountNamesAgentsSequentially() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s").WithAgentCount(2); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var names = model.Resources.OfType().Select(r => r.Name).ToList(); + Assert.Contains("k8s-agent-0", names); + Assert.Contains("k8s-agent-1", names); + } + + [Fact] + public void WithAgentCountAgentsUseClusterImageTag() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s") + .WithK3sVersion("v1.30.0-k3s1") + .WithAgentCount(1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var img = Assert.Single(agent.Annotations.OfType()); + Assert.Equal("v1.30.0-k3s1", img.Tag); + } + + [Fact] + public void WithAgentCountIsEquivalentToConfigureAgentCount() + { + // WithAgentCount(2) should produce the same agent resources as + // AddK3sCluster with configure: cfg => cfg.AgentCount = 2 + var b1 = DistributedApplication.CreateBuilder(); + b1.AddK3sCluster("k8s", agentCount: 2); + using var app1 = b1.Build(); + var agents1 = app1.Services.GetRequiredService() + .Resources.OfType().ToList(); + + var b2 = DistributedApplication.CreateBuilder(); + b2.AddK3sCluster("k8s").WithAgentCount(2); + using var app2 = b2.Build(); + var agents2 = app2.Services.GetRequiredService() + .Resources.OfType().ToList(); + + Assert.Equal(agents1.Count, agents2.Count); + Assert.Equal( + agents1.Select(a => a.Name).OrderBy(n => n), + agents2.Select(a => a.Name).OrderBy(n => n)); + } + + [Fact] + public void WithAgentCountShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithAgentCount(2); + Assert.Throws(action); + } + + [Fact] + public void WithAgentCountShouldThrowWhenCountIsNegative() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.WithAgentCount(-1); + Assert.Throws(action); + } + [Fact] public void WithLifetimePersistentPropagatestoAgentNodes() { @@ -170,7 +267,7 @@ public void WithLifetimePersistentPropagatestoAgentNodes() // DCP uses it to compute container identity before BeforeStartEvent fires. var appBuilder = DistributedApplication.CreateBuilder(); appBuilder - .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2) + .AddK3sCluster("k8s", agentCount: 2) .WithLifetime(ContainerLifetime.Persistent); using var app = appBuilder.Build(); @@ -190,7 +287,7 @@ public void WithLifetimeSessionDoesNotAddPersistentAnnotationToAgents() { var appBuilder = DistributedApplication.CreateBuilder(); appBuilder - .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1) + .AddK3sCluster("k8s", agentCount: 1) .WithLifetime(ContainerLifetime.Session); using var app = appBuilder.Build(); diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterOptionsTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterOptionsTests.cs new file mode 100644 index 000000000..be5df9cf0 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterOptionsTests.cs @@ -0,0 +1,86 @@ +using CommunityToolkit.Aspire.Hosting; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sClusterOptionsTests +{ + // ── Default values ──────────────────────────────────────────────────────── + + [Fact] + public void DefaultsProduceSingleNodeEphemeralCluster() + { + var opts = new K3sClusterOptions(); + + Assert.Equal(0, opts.AgentCount); + Assert.Null(opts.ClusterCidr); + Assert.Null(opts.ServiceCidr); + Assert.Null(opts.ImageTag); + Assert.Empty(opts.DisabledComponents); + Assert.Empty(opts.ExtraArgs); + } + + [Fact] + public void HelmImageDefaultsMatchPackageBundledVersion() + { + var opts = new K3sClusterOptions(); + + Assert.Equal(HelmContainerImageTags.Registry, opts.HelmRegistry); + Assert.Equal(HelmContainerImageTags.Image, opts.HelmImage); + Assert.Equal(HelmContainerImageTags.Tag, opts.HelmTag); + } + + [Fact] + public void KubectlImageDefaultsMatchPackageBundledVersion() + { + var opts = new K3sClusterOptions(); + + Assert.Equal(KubectlContainerImageTags.Registry, opts.KubectlRegistry); + Assert.Equal(KubectlContainerImageTags.Image, opts.KubectlImage); + Assert.Equal(KubectlContainerImageTags.Tag, opts.KubectlTag); + } + + // ── Mutation ────────────────────────────────────────────────────────────── + + [Fact] + public void CanSetAllScalarProperties() + { + var opts = new K3sClusterOptions + { + AgentCount = 3, + ClusterCidr = "10.88.0.0/16", + ServiceCidr = "10.89.0.0/16", + ImageTag = "v1.32.3-k3s1", + HelmRegistry = "my.registry.io", + HelmImage = "my/helm", + HelmTag = "3.18.0", + KubectlRegistry = "my.registry.io", + KubectlImage = "my/kubectl", + KubectlTag = "1.37.0", + }; + + Assert.Equal(3, opts.AgentCount); + Assert.Equal("10.88.0.0/16", opts.ClusterCidr); + Assert.Equal("10.89.0.0/16", opts.ServiceCidr); + Assert.Equal("v1.32.3-k3s1", opts.ImageTag); + Assert.Equal("my.registry.io", opts.HelmRegistry); + Assert.Equal("my/helm", opts.HelmImage); + Assert.Equal("3.18.0", opts.HelmTag); + Assert.Equal("my.registry.io", opts.KubectlRegistry); + Assert.Equal("my/kubectl", opts.KubectlImage); + Assert.Equal("1.37.0", opts.KubectlTag); + } + + [Fact] + public void DisabledComponentsAndExtraArgsAccumulateItems() + { + var opts = new K3sClusterOptions(); + opts.DisabledComponents.Add("traefik"); + opts.DisabledComponents.Add("coredns"); + opts.ExtraArgs.Add("--write-kubeconfig-mode=644"); + + Assert.Equal(2, opts.DisabledComponents.Count); + Assert.Single(opts.ExtraArgs); + Assert.Contains("traefik", opts.DisabledComponents); + Assert.Contains("--write-kubeconfig-mode=644", opts.ExtraArgs); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 6ffa81726..994675867 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -166,20 +166,20 @@ public void WithDataVolumeUsesCustomName() } [Fact] - public void AddK3sClusterWithClusterCidrViaOptions() + public void AddK3sClusterWithAgentCountParam() { + // agentCount is now a direct nullable parameter on AddK3sCluster, + // equivalent to calling WithAgentCount() on the returned builder. var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => - { - opts.ClusterCidr = "10.99.0.0/16"; - }); + appBuilder.AddK3sCluster("k8s", agentCount: 2); using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); var resource = Assert.Single(appModel.Resources.OfType()); - Assert.NotNull(resource); + Assert.Equal(2, resource.AgentCount); + Assert.Equal(2, appModel.Resources.OfType().Count()); } [Fact] @@ -409,7 +409,7 @@ public void WithK3sVersionSyncsAllAgentImageTags() var appBuilder = DistributedApplication.CreateBuilder(); appBuilder - .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2) + .AddK3sCluster("k8s", agentCount: 2) .WithK3sVersion("v1.30.0-k3s1"); using var app = appBuilder.Build(); @@ -557,6 +557,145 @@ public void AddK3sClusterDefaultsToNoServiceCidrArg() Assert.DoesNotContain(ctx.Args, arg => arg is string s && s.StartsWith("--service-cidr=")); } + // ── IResourceWithConnectionString (K3sClusterResource) ─────────────────── + + [Fact] + public void ConnectionStringEnvironmentVariableIsKUBECONFIG() + { + var resource = new K3sClusterResource("k8s"); + Assert.Equal("KUBECONFIG", resource.ConnectionStringEnvironmentVariable); + } + + [Fact] + public async Task GetConnectionStringAsyncReturnsNullWhenDirectoryNotSet() + { + var resource = new K3sClusterResource("k8s") { KubeconfigDirectory = null }; + var result = await resource.GetConnectionStringAsync(); + Assert.Null(result); + } + + [Fact] + public async Task GetConnectionStringAsyncReturnsLocalKubeconfigPath() + { + var dir = Path.Combine(Path.GetTempPath(), $"k8s-{Guid.NewGuid():N}"); + var resource = new K3sClusterResource("k8s") { KubeconfigDirectory = dir }; + + var result = await resource.GetConnectionStringAsync(); + + Assert.Equal(Path.Combine(dir, "local", "kubeconfig.yaml"), result); + } + + // ── AddK3sCluster with agentCount param ────────────────────────────────── + + [Fact] + public void AddK3sClusterWithAgentCountCreatesAgents() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", agentCount: 2); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Equal(2, model.Resources.OfType().Count()); + } + + // ── WithHelmImage / WithKubectlImage ────────────────────────────────────── + + [Fact] + public void WithHelmImageSetsTagOnly() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s").WithHelmImage(tag: "3.18.0"); + + using var app = appBuilder.Build(); + var resource = Assert.Single(app.Services + .GetRequiredService() + .Resources.OfType()); + + var (_, _, tag) = resource.HelmImageInfo; + Assert.Equal("3.18.0", tag); + } + + [Fact] + public void WithHelmImageSetsAllComponents() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s") + .WithHelmImage(tag: "3.18.0", image: "my/helm", registry: "my.registry.io"); + + using var app = appBuilder.Build(); + var resource = Assert.Single(app.Services + .GetRequiredService() + .Resources.OfType()); + + var (registry, image, tag) = resource.HelmImageInfo; + Assert.Equal("my.registry.io", registry); + Assert.Equal("my/helm", image); + Assert.Equal("3.18.0", tag); + } + + [Fact] + public void WithHelmImageNullParametersPreserveCurrentValues() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var (origRegistry, origImage, _) = cluster.Resource.HelmImageInfo; + + cluster.WithHelmImage(tag: "3.18.0"); // only tag changed + + var (registry, image, tag) = cluster.Resource.HelmImageInfo; + Assert.Equal(origRegistry, registry); + Assert.Equal(origImage, image); + Assert.Equal("3.18.0", tag); + } + + [Fact] + public void WithHelmImageShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + Assert.Throws(() => builder.WithHelmImage("3.18.0")); + } + + [Fact] + public void WithKubectlImageSetsTagOnly() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s").WithKubectlImage(tag: "1.37.0"); + + using var app = appBuilder.Build(); + var resource = Assert.Single(app.Services + .GetRequiredService() + .Resources.OfType()); + + var (_, _, tag) = resource.KubectlImageInfo; + Assert.Equal("1.37.0", tag); + } + + [Fact] + public void WithKubectlImageSetsAllComponents() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s") + .WithKubectlImage(tag: "1.37.0", image: "my/kubectl", registry: "my.registry.io"); + + using var app = appBuilder.Build(); + var resource = Assert.Single(app.Services + .GetRequiredService() + .Resources.OfType()); + + var (registry, image, tag) = resource.KubectlImageInfo; + Assert.Equal("my.registry.io", registry); + Assert.Equal("my/kubectl", image); + Assert.Equal("1.37.0", tag); + } + + [Fact] + public void WithKubectlImageShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + Assert.Throws(() => builder.WithKubectlImage("1.37.0")); + } + // ── Options configure callback ──────────────────────────────────────────── [Fact] @@ -564,10 +703,7 @@ public void AddK3sClusterWithServiceCidrViaOptions() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => - { - opts.ServiceCidr = "10.99.0.0/16"; - }); + appBuilder.AddK3sCluster("k8s").WithServiceSubnet("10.99.0.0/16"); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -585,11 +721,9 @@ public void AddK3sClusterWithMultipleDisabledComponentsViaOptions() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => - { - opts.DisabledComponents.Add("traefik"); - opts.DisabledComponents.Add("coredns"); - }); + appBuilder.AddK3sCluster("k8s") + .WithDisabledComponent("traefik") + .WithDisabledComponent("coredns"); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); @@ -608,10 +742,7 @@ public void AddK3sClusterWithExtraArgsViaOptions() { var appBuilder = DistributedApplication.CreateBuilder(); - appBuilder.AddK3sCluster("k8s", configure: opts => - { - opts.ExtraArgs.Add("--write-kubeconfig-mode=644"); - }); + appBuilder.AddK3sCluster("k8s").WithExtraArg("--write-kubeconfig-mode=644"); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs index cb6d85b10..73786feb2 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sServiceEndpointResourceTests.cs @@ -340,6 +340,67 @@ public async Task ServiceEndpointHealthCheckDefaultsToNotReady() Assert.Equal(HealthStatus.Unhealthy, result.Status); } + // ── IResourceWithConnectionString (K3sServiceEndpointResource) ────────── + + [Fact] + public void ConnectionStringEnvironmentVariableFollowsServiceDiscoveryConvention() + { + var cluster = new K3sClusterResource("k8s"); + var ep = new K3sServiceEndpointResource("my-ep", "svc", 80, "default", cluster); + Assert.Equal("services__my-ep__url", ep.ConnectionStringEnvironmentVariable); + } + + [Fact] + public async Task GetConnectionStringAsyncReturnsNullWhenNotReady() + { + var cluster = new K3sClusterResource("k8s"); + var ep = new K3sServiceEndpointResource("ep", "svc", 80, "default", cluster); + // IsReady=false, HostPort=0 by default + var result = await ep.GetConnectionStringAsync(); + Assert.Null(result); + } + + [Fact] + public async Task GetConnectionStringAsyncReturnsLocalhostUrlWhenReady() + { + var cluster = new K3sClusterResource("k8s"); + var ep = new K3sServiceEndpointResource("ep", "svc", 80, "default", cluster) + { + IsReady = true, + HostPort = 9898, + }; + + var result = await ep.GetConnectionStringAsync(); + + Assert.Equal("http://localhost:9898", result); + } + + [Fact] + public async Task GetConnectionStringAsyncUsesSchemeWhenReady() + { + var cluster = new K3sClusterResource("k8s"); + var ep = new K3sServiceEndpointResource("ep", "svc", 443, "default", cluster) + { + Scheme = "https", + IsReady = true, + HostPort = 7777, + }; + + var result = await ep.GetConnectionStringAsync(); + + Assert.Equal("https://localhost:7777", result); + } + + [Fact] + public void ConnectionStringExpressionIsEmptyWhenNotReady() + { + var cluster = new K3sClusterResource("k8s"); + var ep = new K3sServiceEndpointResource("ep", "svc", 80, "default", cluster); + // Not ready — expression should be an empty string expression. + var expr = ep.ConnectionStringExpression; + Assert.NotNull(expr); + } + // ── Resource construction ───────────────────────────────────────────────── [Fact] From 067369859b5a253df0a32ed5d73d7e6f2e8edb57 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Tue, 2 Jun 2026 19:56:39 +0200 Subject: [PATCH 24/29] chore: fixing docs --- .../README.md | 271 +++++++++++------- .../CommunityToolkit.Aspire.Hosting.K3s.cs | 63 ++-- 2 files changed, 209 insertions(+), 125 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index 15cc59f53..7be04c162 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -1,191 +1,254 @@ # CommunityToolkit.Aspire.Hosting.K3s -Provides extension methods and resource definitions for the .NET Aspire AppHost to run a +Provides extension methods and resource definitions for Aspire Distribute Apps to run a [k3s](https://k3s.io/) lightweight Kubernetes cluster as part of the local development inner loop. The cluster, Helm chart installs, manifest applies, and service endpoint exposures all appear as first-class resources in the Aspire dashboard — no external tooling beyond a supported container runtime is required. -## Getting Started +## Prerequisites -### Prerequisites +A container runtime that supports privileged Linux containers: -- A container runtime that supports privileged Linux containers: - - **Docker Engine 20.10+** (Linux) or **Docker Desktop** (macOS / Windows) - - **Podman 4.0+** (Linux) — works with rootful Podman; rootless requires cgroup v2 delegation +- **Docker Engine 20.10+** (Linux) or **Docker Desktop** (macOS / Windows) +- **Podman 4.0+** (Linux, rootful only — rootless requires cgroup v2 delegation) -> **Note:** k3s requires `--privileged`, `--cgroupns=host`, and a writable cgroup -> filesystem mount. These flags are passed automatically by the integration. Whether your -> runtime honours them depends on your system configuration. +> k3s requires `--privileged`, `--cgroupns=host`, and a writable cgroup filesystem. These +> flags are passed automatically by the integration. -### Install the package +## Install -In your AppHost project: - -```sh +```dotnetcli dotnet add package CommunityToolkit.Aspire.Hosting.K3s ``` -### Quick start +## Quick start ```csharp var cluster = builder.AddK3sCluster("k8s"); -// Inject kubeconfig into a project — KUBECONFIG env var points to local/kubeconfig.yaml builder.AddProject("operator") .WaitFor(cluster) - .WithReference(cluster); + .WithReference(cluster); // injects KUBECONFIG automatically ``` -## Deploying Helm charts +## Cluster configuration -`AddHelmRelease` runs `helm upgrade --install` inside an `alpine/helm` container on the -DCP network. No host-side `helm` binary is required. The container exits with code 0 on -success, so use `WaitForCompletion` on any resource that depends on the chart being installed. +All options are available as fluent builder methods — no callback required. ```csharp -var argocd = cluster.AddHelmRelease("argocd", "argo-cd", - repo: "https://argoproj.github.io/argo-helm", - version: "7.8.0", - @namespace: "argocd") - .WithHelmValue("server.insecure", "true") - .WithHelmValuesFile("./deploy/argocd-values.yaml"); +var cluster = builder + .AddK3sCluster("k8s", + apiServerPort: 16443, // fixed host port for the API server (random by default) + agentCount: 2) // 1 server + 2 agent nodes + .WithK3sVersion("v1.32.3-k3s1") // pin k3s image tag + .WithPodSubnet("10.42.0.0/16") // --cluster-cidr + .WithServiceSubnet("10.43.0.0/16") // --service-cidr + .WithDisabledComponent("traefik") // --disable=traefik (repeatable) + .WithExtraArg("--write-kubeconfig-mode=644") // raw k3s server flag (repeatable) + .WithHelmImage(tag: "3.18.0") // override the alpine/helm version + .WithKubectlImage(tag: "1.37.0") // override the alpine/kubectl version + .WithDataVolume() // persist cluster state across restarts + .WithLifetime(ContainerLifetime.Persistent); +``` -builder.AddProject("api") - .WaitForCompletion(argocd) // wait for the chart install to finish +### Option reference + +| Method | Parameter / Effect | +|---|---| +| `AddK3sCluster(agentCount:)` | Number of worker nodes (0 = single-node). Equivalent to `WithAgentCount`. | +| `WithAgentCount(n)` | Same as above, fluent alternative. The health check waits for all `1 + n` nodes to be `Ready`. | +| `WithK3sVersion(tag)` | Overrides the k3s image tag, e.g. `v1.32.3-k3s1`. Synced to agents automatically. | +| `WithPodSubnet(cidr)` | Sets `--cluster-cidr`. Default: k3s built-in `10.42.0.0/16`. | +| `WithServiceSubnet(cidr)` | Sets `--service-cidr`. Default: k3s built-in `10.43.0.0/16`. | +| `WithDisabledComponent(c)` | Passes `--disable=`. Call multiple times for multiple components. | +| `WithExtraArg(arg)` | Appends a raw argument to `k3s server`. | +| `WithHelmImage(registry?, image?, tag?)` | Overrides the `alpine/helm` image used by `AddHelmRelease`. | +| `WithKubectlImage(registry?, image?, tag?)` | Overrides the `alpine/kubectl` image used by `AddK8sManifest`. | +| `WithDataVolume(name?)` | Mounts a named Docker volume at `/var/lib/rancher/k3s`. | +| `WithLifetime(lifetime)` | Sets `ContainerLifetime.Persistent` or `Session` for cluster and agents. | + +## Persistent cluster + +```csharp +var cluster = builder.AddK3sCluster("k8s") + .WithDataVolume() + .WithLifetime(ContainerLifetime.Persistent); +``` + +`WithDataVolume` persists the k3s database, certificates, and node tokens across AppHost +restarts. `WithLifetime(Persistent)` tells DCP to keep the Docker container alive between +runs, making subsequent starts much faster. + +> **Agent count and persistence**: with persistent containers, `agentCount` must stay +> constant across runs. Decreasing it leaves orphaned agent containers that will fail to +> rejoin the cluster; delete them manually with `docker rm -f`. + +## Deploying Helm charts + +`AddHelmRelease` runs `helm upgrade --install --wait` inside an `alpine/helm` container. +No host-side `helm` binary is required. + +```csharp +var podinfo = cluster.AddHelmRelease( + name: "podinfo", + chart: "podinfo", + repo: "https://stefanprodan.github.io/podinfo", + version: "6.7.1", + @namespace: "podinfo") + .WithHelmValue("replicaCount", "2") + .WithHelmValuesFile("./deploy/podinfo-values.yaml"); + +// Wait for the chart install to complete before starting the operator. +builder.AddProject("operator") + .WaitForCompletion(podinfo) .WithReference(cluster); ``` -### Helm override precedence +### Helm value precedence Values are applied in this order (last wins): -1. `WithHelmValuesFile` calls — in the order they are declared (`0-`, `1-`, … prefix) -2. `WithHelmValue` `--set` flags — always override values files +1. `WithHelmValuesFile` — in declaration order +2. `WithHelmValue` (`--set` flags) — always override files + +Use `WithHelmValuesFile` for structured overrides (values with commas, braces, or +backslashes). `WithHelmValue` is convenient for individual scalar overrides. ## Applying Kubernetes manifests `AddK8sManifest` runs `kubectl apply --server-side` inside an `alpine/kubectl` container. -No host-side `kubectl` binary is required. Kustomize overlays are auto-detected. +The apply mode is detected automatically from the path: + +| Path | Mode | +|---|---| +| Single `.yaml` / `.yml` file | `kubectl apply -f ` | +| Directory (no `kustomization.yaml`) | `kubectl apply -f ` (all YAML files, lexicographic order) | +| Directory containing `kustomization.yaml` | `kubectl apply -k ` (Kustomize) | ```csharp -// Plain YAML file or directory -var crd = cluster.AddK8sManifest("widget-crd", "./k8s/crds/"); +// Plain YAML +var appConfig = cluster.AddK8sManifest("app-config", "./k8s/app-config.yaml") + .WaitForCompletion(podinfo); -// Kustomize overlay — detected automatically when kustomization.yaml is present -var overlay = cluster.AddK8sManifest("prod-overlay", "./k8s/overlays/local"); +// Kustomize overlay — auto-detected; directory bind-mounted to preserve base references +var monitoring = cluster.AddK8sManifest("monitoring-config", "./k8s/monitoring") + .WaitForCompletion(podinfo) + .WaitForCompletion(appConfig); -// Gate dependent resources until the CRD is Established +// Wait for CRD to be Established before starting the operator builder.AddProject("operator") .WaitForCompletion(crd) .WithReference(cluster); ``` -For Kustomize overlays that reference base directories outside the overlay path, point -`AddK8sManifest` to the common root and specify the overlay path in `kustomization.yaml`. +## Exposing k8s services -## Exposing k8s services to the Aspire network - -`AddServiceEndpoint` starts an in-process KubernetesClient WebSocket port-forward bound -to `0.0.0.0:{allocatedPort}`. No NodePort configuration is required. +`AddServiceEndpoint` starts an in-process WebSocket port-forward bound to +`0.0.0.0:{allocatedPort}`. No NodePort or LoadBalancer configuration is required. +The endpoint transitions to `Running` only after the target service has a ready pod. ```csharp -var ui = cluster.AddServiceEndpoint("argocd-ui", - serviceName: "argocd-server", - servicePort: 443, - @namespace: "argocd") - .WaitForCompletion(argocd); // wait for chart install before port-forwarding - -// Host processes receive services__argocd-ui__url=https://localhost:{port} -builder.AddProject("consumer") - .WaitFor(ui) - .WithReference(ui); - -// DCP containers receive https://host.docker.internal:{port} -// --add-host=host.docker.internal:host-gateway is injected automatically on Linux +var podinfoWeb = cluster + .AddServiceEndpoint("podinfo-web", "podinfo", servicePort: 9898, @namespace: "podinfo") + .WaitForCompletion(podinfo); + +// Host processes receive http://localhost:{port} +builder.AddProject("api") + .WaitFor(podinfoWeb) + .WithReference(podinfoWeb); + +// DCP-network containers receive http://host.docker.internal:{port} +// --add-host=host.docker.internal:host-gateway is injected automatically builder.AddContainer("sidecar", "myorg/sidecar") - .WaitFor(ui) - .WithReference(ui); + .WaitFor(podinfoWeb) + .WithReference(podinfoWeb); ``` +The injected environment variable follows the Aspire service-discovery convention: +`services__{name}__url=http(s)://{host}:{port}`. + +Scheme is inferred from the port: 443 and 8443 → `https`, all others → `http`. Override +with the `scheme` parameter: `AddServiceEndpoint("ep", "svc", 8080, scheme: "https")`. + ## Kubeconfig injection -`WithReference(cluster)` selects the injection mode automatically: +Both `K3sClusterResource` and `K3sServiceEndpointResource` implement +`IResourceWithConnectionString`, so the standard Aspire `WithReference` overload handles +credential injection automatically. -| Consumer type | What is injected | -|---|---| -| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=…/.k3s/k8s/local/kubeconfig.yaml` | -| `ContainerResource` | Bind-mount of `container/kubeconfig.yaml` at `/var/k3s/` + `KUBECONFIG=/var/k3s/kubeconfig.yaml` | +```csharp +// Projects and executables receive KUBECONFIG pointing to the host-accessible variant +builder.AddProject("operator") + .WithReference(cluster); // KUBECONFIG=…/.k3s/k8s/local/kubeconfig.yaml + +// Containers receive a bind-mounted kubeconfig at /tmp/k3s-kubeconfig.yaml +builder.AddContainer("sidecar", "myorg/sidecar") + .WithReference(cluster); // KUBECONFIG=/tmp/k3s-kubeconfig.yaml + file bind-mount +``` -All standard Kubernetes tooling (`kubectl`, `helm`, KubernetesClient SDK) reads `KUBECONFIG` automatically — no custom bootstrap code required. +All standard Kubernetes tooling reads `KUBECONFIG` automatically: -Reading in .NET: ```csharp -// Works identically for both projects and containers — the SDK reads KUBECONFIG automatically. var config = KubernetesClientConfiguration.BuildConfigFromConfigFile( Environment.GetEnvironmentVariable("KUBECONFIG")); using var client = new Kubernetes(config); ``` -## Persistent cluster state +## TypeScript polyglot AppHost -```csharp -builder.AddK3sCluster("k8s") - .WithDataVolume() // persists /var/lib/rancher/k3s across AppHost restarts - .WithLifetime(ContainerLifetime.Persistent); -``` +All cluster options are available as fluent methods in the TypeScript SDK — no callback +or `K3sClusterOptions` type required. -Without `WithDataVolume` the cluster state is ephemeral — each AppHost start produces a -fresh cluster. With it, subsequent starts reuse the existing cluster and skip -reinitialisation, making startup significantly faster. +```typescript +import { createBuilder, ContainerLifetime } from './.aspire/modules/aspire.mjs'; -## Multi-node clusters +const builder = await createBuilder(); -```csharp -builder.AddK3sCluster("k8s", configure: opts => -{ - opts.AgentCount = 2; // 1 server + 2 agents +const cluster = await builder + .addK3sCluster('k8s', { agentCount: 2 }) + .withK3sVersion('v1.32.3-k3s1') + .withPodSubnet('10.42.0.0/16') + .withDisabledComponent('traefik') + .withHelmImage({ tag: '3.18.0' }) + .withDataVolume({ name: 'k8s-data' }) + .withLifetime(ContainerLifetime.Persistent); + +const podinfo = await cluster.addHelmRelease('podinfo', 'podinfo', { + repo: 'https://stefanprodan.github.io/podinfo', + version: '6.7.1', + namespace: 'podinfo', }); -``` -The health check waits for all nodes to reach `Ready` before the cluster is marked healthy. +const podinfoWeb = await cluster + .addServiceEndpoint('podinfo-web', 'podinfo', 9898, { namespace: 'podinfo' }) + .waitForCompletion(podinfo); -## Image overrides +// Standard withReference — injects KUBECONFIG or services__name__url +await builder.addProject('operator', '../MyOperator/MyOperator.csproj') + .withReference(cluster); -The `alpine/helm` and `alpine/kubectl` images are pinned but configurable: +await builder.addProject('api', '../MyApi/MyApi.csproj') + .withReference(podinfoWeb); -```csharp -builder.AddK3sCluster("k8s", configure: opts => -{ - opts.HelmImage = "my-registry/helm"; - opts.HelmTag = "3.18.0"; - opts.KubectlImage = "my-registry/k8s"; - opts.KubectlTag = "1.36.0"; -}); +await builder.build().run(); ``` ## Reaching Aspire services from k3s pods -k3s pods run on the internal pod network (`10.42.0.0/16`). k3s's Flannel CNI masquerades -pod traffic through the k3s container's DCP network IP, so pods can reach DCP services by -their host-mapped port — but not by their Aspire DNS name. Use Helm values or a ConfigMap -to inject the host-accessible address: +k3s pods run on the internal pod network (`10.42.0.0/16`). Flannel masquerades outbound +pod traffic through the k3s container's DCP network IP, so pods can reach DCP services +using `host.docker.internal` and the host-mapped port. ```csharp var postgres = builder.AddPostgres("db"); -// Resolve the host-mapped port at configuration time -var dbPort = postgres.GetEndpoint("tcp"); - cluster.AddHelmRelease("my-operator", "my-operator-chart") .WithHelmValue("database.host", "host.docker.internal") - .WithHelmValue("database.port", dbPort.Property(EndpointProperty.Port)); + .WithHelmValue("database.port", + postgres.GetEndpoint("tcp").Property(EndpointProperty.Port)); ``` -Inside the pod, `host.docker.internal` resolves to the Docker host because k3s runs as a -privileged container on the DCP network and Flannel masquerades outbound pod traffic -through it. - ## Additional information https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-k3s diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs index 4abe3607c..52c8f0375 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -10,59 +10,68 @@ namespace Aspire.Hosting { public static partial class K3sBuilderExtensions { - [AspireExport("addK3sCluster", Description = "Adds a k3s Kubernetes cluster resource")] - public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } + [AspireExport] + public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, int? agentCount = null) { throw null; } - [AspireExport("withDataVolume", Description = "Adds a named volume for the k3s cluster data directory so state survives AppHost restarts")] + [AspireExport] + public static ApplicationModel.IResourceBuilder WithAgentCount(this ApplicationModel.IResourceBuilder builder, int count) { throw null; } + + [AspireExport] public static ApplicationModel.IResourceBuilder WithDataVolume(this ApplicationModel.IResourceBuilder builder, string? name = null) { throw null; } - [AspireExport("withDisabledComponent", Description = "Disables a built-in k3s component")] + [AspireExport] public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } - [AspireExport("withExtraArg", Description = "Appends a raw argument to the k3s server command")] + [AspireExport] public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } - [AspireExport("withK3sVersion", Description = "Overrides the k3s server image version")] + [AspireExport] + public static ApplicationModel.IResourceBuilder WithHelmImage(this ApplicationModel.IResourceBuilder builder, string? tag = null, string? image = null, string? registry = null) { throw null; } + + [AspireExport] public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } - [AspireExport("withLifetime", Description = "Sets the container lifetime for the k3s cluster and all its agent nodes")] + [AspireExport] + public static ApplicationModel.IResourceBuilder WithKubectlImage(this ApplicationModel.IResourceBuilder builder, string? tag = null, string? image = null, string? registry = null) { throw null; } + + [AspireExport] public static ApplicationModel.IResourceBuilder WithLifetime(this ApplicationModel.IResourceBuilder builder, ApplicationModel.ContainerLifetime lifetime) { throw null; } - [AspireExport("withPodSubnet", Description = "Sets the pod subnet CIDR for the k3s cluster")] + [AspireExport] public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } - [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] + [AspireExport] + public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + + [AspireExportIgnore(Reason = "Marker only. The actual injection (KUBECONFIG env var, container bind-mount) is applied by the BeforeStartEvent subscriber registered in AddK3sCluster, which owns the behavior because the cluster knows what to inject.")] public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source) where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } - - [AspireExport("withServiceSubnet", Description = "Sets the service subnet CIDR for the k3s cluster")] - public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } } public static partial class K3sHelmBuilderExtensions { - [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] + [AspireExport] public static ApplicationModel.IResourceBuilder AddHelmRelease(this ApplicationModel.IResourceBuilder builder, string name, string chart, string? repo = null, string? version = null, string @namespace = "default") { throw null; } - [AspireExport("withHelmValue", Description = "Adds a --set key=value argument to the Helm release")] + [AspireExport] public static ApplicationModel.IResourceBuilder WithHelmValue(this ApplicationModel.IResourceBuilder builder, string key, string value) { throw null; } - [AspireExport("withHelmValuesFile", Description = "Injects a host-side YAML values file into the Helm installer container")] + [AspireExport] public static ApplicationModel.IResourceBuilder WithHelmValuesFile(this ApplicationModel.IResourceBuilder builder, string path) { throw null; } } public static partial class K3sManifestBuilderExtensions { - [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] + [AspireExport] public static ApplicationModel.IResourceBuilder AddK8sManifest(this ApplicationModel.IResourceBuilder builder, string name, string path) { throw null; } } public static partial class K3sServiceEndpointExtensions { - [AspireExport("addServiceEndpoint", Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] + [AspireExport] public static ApplicationModel.IResourceBuilder AddServiceEndpoint(this ApplicationModel.IResourceBuilder builder, string name, string serviceName, int servicePort, string @namespace = "default", string? scheme = null) { throw null; } - [AspireExport("withReference", Description = "Injects the k3s service URL into a dependent resource")] + [AspireExportIgnore(Reason = "Marker only. The actual injection (services__name__url env var, --add-host for containers) is applied by the BeforeStartEvent subscriber registered in AddK3sCluster, which owns the behavior.")] public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source) where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } } @@ -90,24 +99,34 @@ public K3sAgentResource(string name, K3sClusterResource cluster) : base(default! } [AspireExport(ExposeProperties = true)] - public sealed partial class K3sClusterResource : ContainerResource + public sealed partial class K3sClusterResource : ContainerResource, IResourceWithConnectionString { public K3sClusterResource(string name) : base(default!, default) { } public EndpointReference ApiEndpoint { get { throw null; } } + public string? ConnectionStringEnvironmentVariable { get { throw null; } } + + public ReferenceExpression ConnectionStringExpression { get { throw null; } } + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] public System.Collections.Generic.IReadOnlyDictionary HelmReleases { get { throw null; } } [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] public System.Collections.Generic.IReadOnlyList Manifests { get { throw null; } } + + public System.Threading.Tasks.ValueTask GetConnectionStringAsync(System.Threading.CancellationToken cancellationToken = default) { throw null; } } [AspireExport(ExposeProperties = true)] - public sealed partial class K3sServiceEndpointResource : Resource, IResourceWithParent, IResourceWithParent, IResource, IResourceWithWaitSupport + public sealed partial class K3sServiceEndpointResource : Resource, IResourceWithParent, IResourceWithParent, IResource, IResourceWithWaitSupport, IResourceWithConnectionString { public K3sServiceEndpointResource(string name, string serviceName, int servicePort, string @namespace, K3sClusterResource cluster) : base(default!) { } + public string? ConnectionStringEnvironmentVariable { get { throw null; } } + + public ReferenceExpression ConnectionStringExpression { get { throw null; } } + public int HostPort { get { throw null; } } public string Namespace { get { throw null; } } @@ -117,6 +136,8 @@ public K3sServiceEndpointResource(string name, string serviceName, int servicePo public string ServiceName { get { throw null; } } public int ServicePort { get { throw null; } } + + public System.Threading.Tasks.ValueTask GetConnectionStringAsync(System.Threading.CancellationToken cancellationToken = default) { throw null; } } [AspireExport(ExposeProperties = true)] @@ -158,4 +179,4 @@ public sealed partial class K3sClusterOptions public string? ServiceCidr { get { throw null; } set { } } } -} \ No newline at end of file +} From 6108bdafa807c96ba327919407f2a49f4ab827ac Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 8 Jun 2026 13:03:56 +0200 Subject: [PATCH 25/29] chore: increase aspire version to 13.4.2 --- Directory.Build.props | 4 ++-- .../CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index da7e99c2c..e009f7086 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,8 +12,8 @@ enable 13 - $(AspireMajorVersion).4.0 - -preview.1.26281.18 + $(AspireMajorVersion).4.2 + -preview.1.26303.6 9.0.0 10.0.8 4.8.1 diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index 50ee2b3d0..fe417ad67 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe From 76db209f57f3cad80d4d71bd71acef3b3a51394d Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 8 Jun 2026 13:05:57 +0200 Subject: [PATCH 26/29] chore: increase aspire version to 13.4.3 --- Directory.Build.props | 2 +- .../CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index e009f7086..594190daf 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,7 +12,7 @@ enable 13 - $(AspireMajorVersion).4.2 + $(AspireMajorVersion).4.3 -preview.1.26303.6 9.0.0 10.0.8 diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index fe417ad67..e09c45320 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe From 6ff7800c44d0e1bbe1c66fb66bc8386e7984049b Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 8 Jun 2026 13:19:19 +0200 Subject: [PATCH 27/29] chore: reverting to 13.4.2 and fixing action to support newer releases --- .github/actions/setup-runtimes-caching/action.yml | 15 ++++++++------- Directory.Build.props | 2 +- ...unityToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup-runtimes-caching/action.yml b/.github/actions/setup-runtimes-caching/action.yml index e72dc8ba1..ab0da9139 100644 --- a/.github/actions/setup-runtimes-caching/action.yml +++ b/.github/actions/setup-runtimes-caching/action.yml @@ -43,13 +43,14 @@ runs: EXPECTED_ASPIRE_CLI_VERSION: ${{ steps.aspire-version.outputs.cli-version }} run: | INSTALLED_ASPIRE_CLI_VERSION="$(aspire --version)" - case "${INSTALLED_ASPIRE_CLI_VERSION}" in - "${EXPECTED_ASPIRE_CLI_VERSION}"|"${EXPECTED_ASPIRE_CLI_VERSION}"+*) ;; - *) - echo "Expected Aspire CLI version ${EXPECTED_ASPIRE_CLI_VERSION}, but found ${INSTALLED_ASPIRE_CLI_VERSION}." - exit 1 - ;; - esac + # Strip build metadata (e.g. 13.4.3+becb48e → 13.4.3) before comparing so that + # a newer patch release installed by the action does not fail this check. + INSTALLED_BASE="${INSTALLED_ASPIRE_CLI_VERSION%%+*}" + # Pass when installed base version >= expected version (sort -V = version ordering). + if ! printf '%s\n' "${EXPECTED_ASPIRE_CLI_VERSION}" "${INSTALLED_BASE}" | sort -V -C; then + echo "Expected Aspire CLI version >= ${EXPECTED_ASPIRE_CLI_VERSION}, but found ${INSTALLED_ASPIRE_CLI_VERSION}." + exit 1 + fi - name: Set up Python uses: actions/setup-python@v5 diff --git a/Directory.Build.props b/Directory.Build.props index 594190daf..e009f7086 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,7 +12,7 @@ enable 13 - $(AspireMajorVersion).4.3 + $(AspireMajorVersion).4.2 -preview.1.26303.6 9.0.0 10.0.8 diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index e09c45320..fe417ad67 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe From ae9c54b1c2e9e45dd1e96a7301f0d987dbebc412 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 8 Jun 2026 13:26:49 +0200 Subject: [PATCH 28/29] chore: reverting version to 13.4.0 --- Directory.Build.props | 4 ++-- .../CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index e009f7086..da7e99c2c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,8 +12,8 @@ enable 13 - $(AspireMajorVersion).4.2 - -preview.1.26303.6 + $(AspireMajorVersion).4.0 + -preview.1.26281.18 9.0.0 10.0.8 4.8.1 diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index fe417ad67..50ee2b3d0 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe From 7f8f12abe497cdfac520388c0306becc72e51c67 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Tue, 9 Jun 2026 02:05:37 +0200 Subject: [PATCH 29/29] fix(k3s): use annotation.AllocatedEndpoint to resolve API server port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EndpointReference.IsAllocated caches its result on first call via a nullable bool field (??= pattern). For persistent containers in polyglot AppHosts, the health check can tick before DCP fires the endpoint allocation event, permanently caching false and making the port unresolvable for the lifetime of the process — leaving local/kubeconfig.yaml stale with port 6443. Replace the IsAllocated guard + GetValueAsync path with a direct read of EndpointAnnotation.AllocatedEndpoint.Port. This property is non-cached (checks IsValueSet on every call) and non-blocking (returns null if DCP has not yet allocated), so it picks up DCP's allocation on any subsequent health check tick regardless of when the first tick occurred. Port resolution order: 1. annotation.AllocatedEndpoint.Port — set by DCP on container start 2. annotation.Port — static apiServerPort from AddK3sCluster Remove the EndpointReference constructor parameter and the port hint file mechanism, both of which are unnecessary with this approach. --- .../K3sBuilderExtensions.cs | 2 +- .../K3sReadinessHealthCheck.cs | 57 ++++++++----------- .../K3sReadinessHealthCheckTests.cs | 51 +++++++++++------ 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index cb0f3a70e..da69cc309 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -184,7 +184,7 @@ public static IResourceBuilder AddK3sCluster( // state survive across health-check ticks. Using a factory (sp => new ...) would // create a fresh instance on every check, making _cachedClient dead state and // leaking a Kubernetes/HttpClient on every tick. - var healthCheck = new K3sReadinessHealthCheck(resource, resource.ApiEndpoint); + var healthCheck = new K3sReadinessHealthCheck(resource); builder.Services.AddHealthChecks().Add(new HealthCheckRegistration( $"k3s_{name}_ready", _ => healthCheck, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index 100a17aa9..17b1f11c0 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -22,7 +22,6 @@ namespace CommunityToolkit.Aspire.Hosting; /// internal sealed class K3sReadinessHealthCheck( K3sClusterResource resource, - EndpointReference endpoint, Func? kubernetesFactory = null) : IHealthCheck { private IKubernetes CreateClient(string path) => @@ -35,49 +34,43 @@ public async Task CheckHealthAsync( HealthCheckContext context, CancellationToken cancellationToken = default) { - int port; + // Resolve the API server port. We try two sources in order: + // + // 1. EndpointAnnotation.AllocatedEndpoint.Port — set by DCP when it allocates the + // endpoint. Non-blocking (returns null if not yet set) and NOT cached, so it picks + // up DCP's allocation on any tick after the event fires. + // We intentionally avoid EndpointReference.IsAllocated here because that property + // caches its result on first call; if the health check ticks before DCP fires the + // allocation event the cached false would persist forever. + // + // 2. EndpointAnnotation.Port — the statically configured host port (only non-null + // when the caller passed an explicit apiServerPort to AddK3sCluster). Works for + // both C# and polyglot AppHosts. + + var annotation = resource.Annotations + .OfType() + .FirstOrDefault(a => a.Name == K3sClusterResource.ApiServerEndpointName); - if (endpoint.IsAllocated) + int port; + if (annotation?.AllocatedEndpoint is { Port: > 0 } alloc) { - // Use the async reference-expression path — the same mechanism PostgreSQL uses - // for its connection string port. This resolves through DCP's actual allocation - // rather than the synchronous AllocatedEndpoint.Port shortcut, which for proxied - // HTTPS endpoints in Aspire 13.4.0+ may return the proxy port (= target port 6443) - // rather than the Docker host port that is actually reachable from the AppHost. - // Only call GetValueAsync when IsAllocated is true — in test/non-DCP contexts the - // call would block indefinitely waiting for an allocation that never arrives. - var portExpression = endpoint.Property(EndpointProperty.Port); - var portStr = await ((IValueProvider)portExpression) - .GetValueAsync(cancellationToken) - .ConfigureAwait(false); - - if (!int.TryParse(portStr, out port) || port <= 0) - port = endpoint.Port; // synchronous fallback within same allocation context + port = alloc.Port; } - else + else if (annotation?.Port is > 0) { - // EndpointReference.IsAllocated is false — either the endpoint has not yet been - // allocated, or DCP reconnected to a persistent container without firing the normal - // allocation event. Fall back to EndpointAnnotation.Port, which DCP updates even - // in the reconnect path. - var annotation = resource.Annotations - .OfType() - .FirstOrDefault(a => a.Name == K3sClusterResource.ApiServerEndpointName); - - if (annotation?.Port is not > 0) - return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); - port = annotation.Port!.Value; } + else + { + return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); + } return await CheckCoreAsync(port, cancellationToken).ConfigureAwait(false); } /// /// Core readiness check given an already-known API server . - /// Extracted so unit tests can exercise the full check path without requiring DCP - /// to allocate the endpoint (i.e. without - /// being true). + /// Extracted so unit tests can call it directly with an explicit port. /// internal async Task CheckCoreAsync(int port, CancellationToken cancellationToken = default) { diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs index a95249ec1..2a6e65743 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sReadinessHealthCheckTests.cs @@ -15,13 +15,12 @@ public class K3sReadinessHealthCheckTests // ── CheckHealthAsync — early exit (no DCP allocation) ──────────────────── [Fact] - public async Task CheckHealthAsync_WhenEndpointNotAllocated_ReturnsUnhealthy() + public async Task CheckHealthAsync_WhenNeitherAllocatedEndpointNorStaticPort_ReturnsUnhealthy() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - // In unit-test context there is no DCP, so the port expression resolves to null/empty - // and EndpointAnnotation.Port is also unset — both paths return Unhealthy. - var healthCheck = new K3sReadinessHealthCheck(cluster.Resource, cluster.Resource.ApiEndpoint); + // No DCP, no static port — both annotation.AllocatedEndpoint and annotation.Port are null. + var healthCheck = new K3sReadinessHealthCheck(cluster.Resource); var result = await healthCheck.CheckHealthAsync(null!); @@ -30,30 +29,50 @@ public async Task CheckHealthAsync_WhenEndpointNotAllocated_ReturnsUnhealthy() } [Fact] - public async Task CheckHealthAsync_WhenEndpointAnnotationHasPort_UsesAnnotationPortAsFallback() + public async Task CheckHealthAsync_WhenAllocatedEndpointSet_ProceedsToCheckCore() { - // Simulates a persistent container reconnect: endpoint.IsAllocated is false but - // DCP has already set EndpointAnnotation.Port on the resource. + // Simulates DCP having fired the endpoint allocation event. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + var annotation = cluster.Resource.Annotations + .OfType() + .First(a => a.Name == K3sClusterResource.ApiServerEndpointName); + // Set AllocatedEndpoint directly — the same property DCP sets when it allocates the port. + annotation.AllocatedEndpoint = new AllocatedEndpoint(annotation, "localhost", 32773); + + var healthCheck = new K3sReadinessHealthCheck(cluster.Resource); + + var result = await healthCheck.CheckHealthAsync(null!); + + // cluster/kubeconfig.yaml doesn't exist in the test dir → reaches CheckCoreAsync + // and returns "write kubeconfig", not "not yet allocated". + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.DoesNotContain("not yet allocated", result.Description); + Assert.Contains("write kubeconfig", result.Description); + } + + [Fact] + public async Task CheckHealthAsync_WhenStaticPortConfigured_ProceedsToCheckCore() + { + // Simulates AddK3sCluster("k8s", apiServerPort: 32773): AllocatedEndpoint is null + // but annotation.Port carries the explicit static port. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - // Manually set the allocated port on the EndpointAnnotation (DCP does this - // even when reconnecting to a persistent container). var annotation = cluster.Resource.Annotations .OfType() .First(a => a.Name == K3sClusterResource.ApiServerEndpointName); annotation.Port = 32773; - var (hc, dir, _) = MakeCheck(writeKubeconfig: false, nodeCount: 0, agentCount: 0); - // Re-create the health check using the cluster that has the annotation set. - var healthCheck = new K3sReadinessHealthCheck(cluster.Resource, cluster.Resource.ApiEndpoint); + var healthCheck = new K3sReadinessHealthCheck(cluster.Resource); var result = await healthCheck.CheckHealthAsync(null!); - // With no kubeconfig file it returns "Waiting for k3s to write kubeconfig" - // — proving it got past the IsAllocated check and reached CheckCoreAsync. + // cluster/kubeconfig.yaml doesn't exist → "write kubeconfig", not "not yet allocated". Assert.Equal(HealthStatus.Unhealthy, result.Status); Assert.DoesNotContain("not yet allocated", result.Description); + Assert.Contains("write kubeconfig", result.Description); } // ── CheckCoreAsync — file-system and Kubernetes paths ──────────────────── @@ -62,7 +81,7 @@ public async Task CheckHealthAsync_WhenEndpointAnnotationHasPort_UsesAnnotationP public async Task CheckCoreAsync_WhenKubeconfigDirectoryIsNull_ReturnsUnhealthy() { var cluster = new K3sClusterResource("k8s") { KubeconfigDirectory = null }; - var healthCheck = new K3sReadinessHealthCheck(cluster, cluster.ApiEndpoint, _ => Mock.Of()); + var healthCheck = new K3sReadinessHealthCheck(cluster, _ => Mock.Of()); var result = await healthCheck.CheckCoreAsync(port: 6443); @@ -367,7 +386,7 @@ private static (K3sReadinessHealthCheck check, string dir, Mock moc }); } - var check = new K3sReadinessHealthCheck(cluster, cluster.ApiEndpoint, _ => mockK8s.Object); + var check = new K3sReadinessHealthCheck(cluster, _ => mockK8s.Object); return (check, dir, mockK8s); } }