diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ff4df17e85..a2e8646484 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -171,6 +171,7 @@ jobs: - 'Cargo.lock' - 'Makefile.toml' - 'rust-toolchain.toml' + - 'dev/k8s/machine-a-tron-controller/**' - name: Calculate version id: version run: | @@ -816,6 +817,32 @@ jobs: timeout_minutes: 60 secrets: inherit + build-mat-k8s-controller: + if: >- + ${{ + always() + && github.event_name != 'schedule' + && needs.prepare.result == 'success' + && needs.prepare.outputs.source_files_changed == 'true' + }} + needs: + - prepare + uses: ./.github/workflows/docker-build.yml + with: + dockerfile_path: dev/k8s/machine-a-tron-controller/Dockerfile + context_path: dev/k8s/machine-a-tron-controller + image_name: ${{ needs.prepare.outputs.image_registry }}/mat-k8s-controller + image_tag: ${{ needs.prepare.outputs.version }} + additional_tags: ${{ needs.prepare.outputs.image_registry }}/mat-k8s-controller:${{ needs.prepare.outputs.major_minor_version }}-latest + platforms: linux/amd64,linux/arm64 + runner: linux-amd64-cpu4 + push: ${{ needs.prepare.outputs.publish_images == 'true' }} + load: false + scan: true + tag_latest: false + timeout_minutes: 30 + secrets: inherit + test-release-container-services: if: >- ${{ @@ -1834,6 +1861,7 @@ jobs: - build-release-container-x86_64 - build-release-container-aarch64 - build-machine-a-tron + - build-mat-k8s-controller - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb @@ -1938,6 +1966,7 @@ jobs: - build-release-container-x86_64 - build-release-container-aarch64 - build-machine-a-tron + - build-mat-k8s-controller - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb @@ -2017,6 +2046,8 @@ jobs: - security-secret-scan - lint-police - check-rest-core-proto-sync + - build-machine-a-tron + - build-mat-k8s-controller steps: - name: Decide pass/fail env: diff --git a/crates/machine-a-tron/README.md b/crates/machine-a-tron/README.md index ace4a2bceb..2ed87de295 100644 --- a/crates/machine-a-tron/README.md +++ b/crates/machine-a-tron/README.md @@ -9,7 +9,7 @@ periodically report health network observations. ## Usage -``` +```text target/debug/machine-a-tron -h Usage: machine-a-tron [OPTIONS] --relay-address [NICO_API] @@ -120,4 +120,4 @@ setup with the appropriate overrides for redfish so that it will send all reques > for all libredfish calls. If you want to go back to running the TUI locally, you'll want to manually edit the > generated nico-api-site-config.toml and drop the `override_target_host` line. You may also want to edit the > `mat.toml` in the same directory and set the host_count to 0 so that the in-cluster machine-a-tron doesn't run any -> mock machines. \ No newline at end of file +> mock machines. diff --git a/dev/k8s/machine-a-tron-controller/Dockerfile b/dev/k8s/machine-a-tron-controller/Dockerfile new file mode 100644 index 0000000000..08a1497907 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/Dockerfile @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build stage +FROM golang:1.26.4-alpine@sha256:3ad57304ad93bbec8548a0437ad9e06a455660655d9af011d58b993f6f615648 AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o mat-k8s-controller ./cmd/mat-k8s-controller + +# Runtime stage +FROM gcr.io/distroless/static:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 + +COPY --from=builder /app/mat-k8s-controller /mat-k8s-controller + +USER nonroot:nonroot + +ENTRYPOINT ["/mat-k8s-controller"] diff --git a/dev/k8s/machine-a-tron-controller/Makefile b/dev/k8s/machine-a-tron-controller/Makefile new file mode 100644 index 0000000000..818dc02b43 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/Makefile @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: all build test test-coverage lint fmt docker-build docker-build-latest clean run verify + +TAG ?= dev + +all: build + +build: + @mkdir -p bin + go build -o bin/mat-k8s-controller ./cmd/mat-k8s-controller + +test: + go test -v ./... + +test-coverage: + go test -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html + +lint: + golangci-lint run ./... + +fmt: + go fmt ./... + +docker-build: + docker build -t mat-k8s-controller:$(TAG) . + +docker-build-latest: + docker build -t mat-k8s-controller:latest . + +clean: + rm -rf bin/ coverage.out coverage.html + +run: + go run ./cmd/mat-k8s-controller --kubeconfig=$(KUBECONFIG) --log-level=debug + +verify: lint test diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md new file mode 100644 index 0000000000..9d6eabf62c --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -0,0 +1,114 @@ +# Machine-a-tron Kubernetes Controller + +Kubernetes controller that auto-discovers machine-a-tron pods and creates +Services for mock BMC endpoints. + +## Features + +- Auto-discovers machine-a-tron pods via `nvidia-infra-controller/mat-service=true` + label +- Creates ClusterIP Services with BMC IP for each mock BMC +- Supports Redfish (TCP 443) ports +- Multi-pod deployments with pod-specific routing +- Automatic cleanup of stale Services + +## Build + +```bash +docker build -t mat-k8s-controller:latest . +kind load docker-image mat-k8s-controller:latest --name +``` + +## Configuration + +| Flag | Env Var | Default | Description | +|------|---------|---------|-------------| +| `--namespace` | `NAMESPACE` | `nico-system` | Kubernetes namespace | +| `--discovery-selector` | `DISCOVERY_SELECTOR` | `nvidia-infra-controller/mat-service=true` | Label selector for discovery | +| `--sync-interval` | `SYNC_INTERVAL` | `30s` | Reconciliation interval | +| `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector for Services | +| `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `false` | Skip TLS verification (dev only) | +| `--log-level` | `LOG_LEVEL` | `info` | Log level | +| `--bmc-mock-port` | `BMC_MOCK_PORT` | `1266` | BMC mock service port | +| `--kubeconfig` | `KUBECONFIG` | (empty) | Path to kubeconfig (dev only, uses in-cluster config if empty) | + +## Helm Deployment + +Enable in parent chart: + +```yaml +mat-k8s-controller: + enabled: true + image: + pullPolicy: Never # For local images + config: + insecureSkipVerify: true # Only for dev with self-signed certs +``` + +## Service Structure + +Created Services have: + +**Labels:** + +- `app.kubernetes.io/managed-by: mat-k8s-controller` +- `nvidia-infra-controller/mat-id: ` +- `nvidia-infra-controller/mat-machine-type: host|dpu` +- `nvidia-infra-controller/pod-name: ` (multi-pod) + +**Annotations:** + +- `nvidia-infra-controller/mat-bmc-ip` +- `nvidia-infra-controller/mat-api-state` +- `nvidia-infra-controller/mat-power-state` +- `nvidia-infra-controller/mat-hardware-type` + +## Development + +```bash +make build +make test +make run KUBECONFIG="$HOME/.kube/config" +``` + +## Troubleshooting + +### ClusterIP already allocated + +BMC IP is outside ServiceCIDR or already in use. + +**Solutions:** + +1. Reserve a ServiceCIDR for machine-a-tron (K8s 1.29+) +2. Use a CIDR within the cluster's ServiceCIDR +3. Delete conflicting Services + +### ClusterIP change detected + +BMC IP changed but ClusterIP is immutable. Controller will delete and recreate +the Service. + +## Architecture + +```mermaid +flowchart LR + subgraph MAT[machine-a-tron pods] + MAT0[mat-0/bmc-mock] + MAT1[mat-1/bmc-mock] + end + + subgraph Controller + Discovery[Service Discovery] + Reconciler[Reconciler] + end + + subgraph K8s[Kubernetes Services] + SVC1[mat-bmc-host-xxx] + SVC2[mat-bmc-dpu-yyy] + end + + Discovery -->|discovers| MAT0 + Discovery -->|discovers| MAT1 + Reconciler -->|polls /machines/status| MAT + Reconciler -->|creates/updates/deletes| K8s +``` diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go new file mode 100644 index 0000000000..7c5977a186 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// mat-k8s-controller is a Kubernetes controller that reconciles Services +// for machine-a-tron mock BMC endpoints. +// +// It discovers machine-a-tron pods via their bmc-mock Services, polls each +// pod's /machines/status API, and creates/updates/deletes Kubernetes Services +// to expose Redfish (and optionally IPMI) endpoints for each mock BMC. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/rs/zerolog" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/controller" + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/matclient" +) + +func main() { + // Flags + namespace := flag.String("namespace", envOrDefault("NAMESPACE", "nico-system"), + "Kubernetes namespace for Services and machine-a-tron discovery") + discoverySelector := flag.String("discovery-selector", envOrDefault("DISCOVERY_SELECTOR", "nvidia-infra-controller/mat-service=true"), + "Label selector for discovering machine-a-tron bmc-mock Services") + syncInterval := flag.Duration("sync-interval", parseDurationOrDefault("SYNC_INTERVAL", 30*time.Second), + "Interval between reconciliation passes") + kubeconfig := flag.String("kubeconfig", os.Getenv("KUBECONFIG"), + "Path to kubeconfig file (uses in-cluster config if empty, development only)") + targetSelector := flag.String("target-selector", envOrDefault("TARGET_SELECTOR", "app.kubernetes.io/name=nico-machine-a-tron"), + "Pod selector for Services (comma-separated key=value pairs)") + insecureSkipVerify := flag.Bool("insecure-skip-verify", envBoolOrDefault("INSECURE_SKIP_VERIFY", false), + "Skip TLS certificate verification (use only for development with self-signed certs)") + logLevel := flag.String("log-level", envOrDefault("LOG_LEVEL", "info"), + "Log level (debug, info, warn, error)") + bmcMockPort := flag.Int("bmc-mock-port", envIntOrDefault("BMC_MOCK_PORT", 1266), + "Port number for bmc-mock service") + + flag.Parse() + + // Validate sync interval + if *syncInterval <= 0 { + fmt.Fprintf(os.Stderr, "error: sync-interval must be positive, got %v\n", *syncInterval) + os.Exit(1) + } + + // Setup logger + level, err := zerolog.ParseLevel(*logLevel) + if err != nil { + level = zerolog.InfoLevel + } + logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}). + Level(level). + With(). + Timestamp(). + Str("component", "mat-k8s-controller"). + Logger() + + logger.Info(). + Str("namespace", *namespace). + Str("discovery_selector", *discoverySelector). + Dur("sync_interval", *syncInterval). + Str("target_selector", *targetSelector). + Int("bmc_mock_port", *bmcMockPort). + Bool("insecure_skip_verify", *insecureSkipVerify). + Msg("starting controller") + + // Create Kubernetes client + var k8sConfig *rest.Config + if *kubeconfig != "" { + k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) + } else { + k8sConfig, err = rest.InClusterConfig() + } + if err != nil { + logger.Fatal().Err(err).Msg("failed to create Kubernetes config") + } + + // Increase rate limits for bulk operations + k8sConfig.QPS = 100 + k8sConfig.Burst = 200 + + clientset, err := kubernetes.NewForConfig(k8sConfig) + if err != nil { + logger.Fatal().Err(err).Msg("failed to create Kubernetes clientset") + } + + // Create machine-a-tron client options + clientOpts := []matclient.Option{matclient.WithLogger(logger)} + if *insecureSkipVerify { + clientOpts = append(clientOpts, matclient.WithInsecureSkipVerify()) + } + + // Parse target selector + selector := parseSelector(*targetSelector) + + // Create service builder + builder := &controller.ServiceBuilder{ + Namespace: *namespace, + BaseSelector: selector, + } + + // Create discovery and reconciler + discovery := controller.NewMatPodDiscovery(clientset, *namespace, *bmcMockPort, *discoverySelector) + k8sClient := controller.NewRealK8sServiceClient(clientset) + reconciler := controller.NewReconciler(discovery, builder, k8sClient, clientOpts, logger) + + // Setup signal handling + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + go func() { + sig := <-sigCh + logger.Info().Str("signal", sig.String()).Msg("received shutdown signal") + cancel() + }() + + // Run reconciliation loop + ticker := time.NewTicker(*syncInterval) + defer ticker.Stop() + + // Run initial reconciliation + runReconcile(ctx, reconciler, logger) + + for { + select { + case <-ctx.Done(): + logger.Info().Msg("shutting down") + return + case <-ticker.C: + runReconcile(ctx, reconciler, logger) + } + } +} + +func runReconcile(ctx context.Context, r *controller.Reconciler, logger zerolog.Logger) { + start := time.Now() + result := r.Reconcile(ctx) + elapsed := time.Since(start) + + logEvent := logger.Info(). + Int("created", result.Created). + Int("updated", result.Updated). + Int("deleted", result.Deleted). + Dur("elapsed", elapsed) + + if len(result.Errors) > 0 { + logEvent = logger.Error(). + Int("created", result.Created). + Int("updated", result.Updated). + Int("deleted", result.Deleted). + Int("errors", len(result.Errors)). + Dur("elapsed", elapsed) + + for _, err := range result.Errors { + logger.Error().Err(err).Msg("reconciliation error") + } + } + + logEvent.Msg("reconciliation complete") +} + +func envOrDefault(key, defaultValue string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultValue +} + +func envBoolOrDefault(key string, defaultValue bool) bool { + if v := os.Getenv(key); v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + } + return defaultValue +} + +func envIntOrDefault(key string, defaultValue int) int { + if v := os.Getenv(key); v != "" { + if i, err := strconv.Atoi(v); err == nil { + return i + } + } + return defaultValue +} + +func parseDurationOrDefault(envKey string, defaultValue time.Duration) time.Duration { + if v := os.Getenv(envKey); v != "" { + if d, err := time.ParseDuration(v); err == nil { + return d + } + } + return defaultValue +} + +func parseSelector(s string) map[string]string { + result := make(map[string]string) + if s == "" { + return result + } + + for _, pair := range strings.Split(s, ",") { + if kv := strings.SplitN(pair, "=", 2); len(kv) == 2 { + result[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + } + return result +} diff --git a/dev/k8s/machine-a-tron-controller/go.mod b/dev/k8s/machine-a-tron-controller/go.mod new file mode 100644 index 0000000000..4bba495f13 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/go.mod @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +module github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller + +go 1.26.4 + +require ( + github.com/rs/zerolog v1.34.0 + github.com/stretchr/testify v1.10.0 + k8s.io/api v0.32.4 + k8s.io/apimachinery v0.32.4 + k8s.io/client-go v0.32.4 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/dev/k8s/machine-a-tron-controller/go.sum b/dev/k8s/machine-a-tron-controller/go.sum new file mode 100644 index 0000000000..476d95621c --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/go.sum @@ -0,0 +1,167 @@ +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.4 h1:kw8Y/G8E7EpNy7gjB8gJZl3KJkNz8HM2YHrZPtAZsF4= +k8s.io/api v0.32.4/go.mod h1:5MYFvLvweRhyKylM3Es/6uh/5hGp0dg82vP34KifX4g= +k8s.io/apimachinery v0.32.4 h1:8EEksaxA7nd7xWJkkwLDN4SvWS5ot9g6Z/VZb3ju25I= +k8s.io/apimachinery v0.32.4/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.4 h1:zaGJS7xoYOYumoWIFXlcVrsiYioRPrXGO7dBfVC5R6M= +k8s.io/client-go v0.32.4/go.mod h1:k0jftcyYnEtwlFW92xC7MTtFv5BNcZBr+zn9jPlT9Ic= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go new file mode 100644 index 0000000000..3bfcbd9139 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -0,0 +1,603 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package controller implements the Kubernetes Service reconciliation logic +// for machine-a-tron mock BMC endpoints. +package controller + +import ( + "context" + "fmt" + "strconv" + "sync" + "sync/atomic" + + "github.com/rs/zerolog" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/matclient" +) + +const ( + // LabelManagedBy identifies the controller managing the resource. + LabelManagedBy = "app.kubernetes.io/managed-by" + // LabelManagedByValue is the value for the managed-by label. + LabelManagedByValue = "mat-k8s-controller" + + // LabelPodName is the label that identifies which machine-a-tron pod owns this service. + LabelPodName = "nvidia-infra-controller/pod-name" + + // LabelMatID is the machine-a-tron ID label. + LabelMatID = "nvidia-infra-controller/mat-id" + // LabelMachineID is the NICo machine ID label. + LabelMachineID = "nvidia-infra-controller/mat-machine-id" + // LabelMachineType distinguishes host vs dpu. + LabelMachineType = "nvidia-infra-controller/mat-machine-type" + // LabelParentMatID links DPUs to their parent host. + LabelParentMatID = "nvidia-infra-controller/mat-parent-id" + + // AnnotationBMCIP is the BMC IP address annotation. + AnnotationBMCIP = "nvidia-infra-controller/mat-bmc-ip" + // AnnotationAPIState is the API state annotation. + AnnotationAPIState = "nvidia-infra-controller/mat-api-state" + // AnnotationPowerState is the power state annotation. + AnnotationPowerState = "nvidia-infra-controller/mat-power-state" + // AnnotationHardwareType is the hardware type annotation. + AnnotationHardwareType = "nvidia-infra-controller/mat-hardware-type" + // AnnotationRedfishListenPort is the Redfish listen port annotation. + AnnotationRedfishListenPort = "nvidia-infra-controller/mat-redfish-listen-port" + // AnnotationIPMIListenPort is the IPMI listen port annotation. + AnnotationIPMIListenPort = "nvidia-infra-controller/mat-ipmi-listen-port" + + // MachineTypeHost is the machine type for hosts. + MachineTypeHost = "host" + // MachineTypeDPU is the machine type for DPUs. + MachineTypeDPU = "dpu" + + // PortNameRedfish is the name of the Redfish port. + PortNameRedfish = "redfish" + // PortNameIPMI is the name of the IPMI port. + PortNameIPMI = "ipmi" + + // DefaultConcurrency is the default number of concurrent workers for K8s API calls. + DefaultConcurrency = 50 +) + +// ServiceBuilder builds Kubernetes Services from machine status. +type ServiceBuilder struct { + Namespace string + BaseSelector map[string]string +} + +// BuildServiceName generates a consistent service name for a machine. +func BuildServiceName(machineType, matID string) string { + shortID := matID + if len(matID) > 12 { + shortID = matID[:12] + } + return fmt.Sprintf("mat-bmc-%s-%s", machineType, shortID) +} + +// BuildService creates a Kubernetes Service for a machine's BMC. +// podName is used to create a pod-specific selector for multi-pod deployments. +func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineType, parentMatID, podName string) *corev1.Service { + name := BuildServiceName(machineType, machine.MatID) + + labels := map[string]string{ + LabelManagedBy: LabelManagedByValue, + LabelMatID: machine.MatID, + LabelMachineType: machineType, + } + if machine.MachineID != nil { + labels[LabelMachineID] = *machine.MachineID + } + if parentMatID != "" { + labels[LabelParentMatID] = parentMatID + } + + annotations := map[string]string{ + AnnotationAPIState: machine.APIState, + AnnotationPowerState: machine.PowerState, + AnnotationRedfishListenPort: strconv.Itoa(int(machine.BMC.Redfish.ListenPort)), + } + if machine.BMC.IP != nil { + annotations[AnnotationBMCIP] = *machine.BMC.IP + } + if machine.HardwareType != nil { + annotations[AnnotationHardwareType] = *machine.HardwareType + } + + ports := []corev1.ServicePort{ + { + Name: PortNameRedfish, + Protocol: corev1.ProtocolTCP, + Port: int32(machine.BMC.Redfish.ReachablePort), + TargetPort: intstr.FromInt32(int32(machine.BMC.Redfish.ListenPort)), + }, + } + + // Add IPMI port if available + if machine.BMC.IPMI != nil { + ports = append(ports, corev1.ServicePort{ + Name: PortNameIPMI, + Protocol: corev1.ProtocolUDP, + Port: int32(machine.BMC.IPMI.ReachablePort), + TargetPort: intstr.FromInt32(int32(machine.BMC.IPMI.ListenPort)), + }) + annotations[AnnotationIPMIListenPort] = strconv.Itoa(int(machine.BMC.IPMI.ListenPort)) + } + + // Build selector - include pod name for multi-pod deployments + selector := make(map[string]string) + for k, v := range b.BaseSelector { + selector[k] = v + } + if podName != "" { + selector[LabelPodName] = podName + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: b.Namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: selector, + Ports: ports, + }, + } + + // Set ClusterIP to BMC IP for direct addressing + if machine.BMC.IP != nil { + svc.Spec.ClusterIP = *machine.BMC.IP + } + + return svc +} + +// BuildServicesFromStatus builds Services for all machines in the status response. +// podName is used to create pod-specific selectors for multi-pod deployments. +func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatusResponse, podName string) []*corev1.Service { + var services []*corev1.Service + + for _, machine := range status.Machines { + // Build service for the host + svc := b.BuildService(&machine, MachineTypeHost, "", podName) + services = append(services, svc) + + // Build services for DPUs + for _, dpu := range machine.DPUs { + dpuSvc := b.BuildService(&dpu, MachineTypeDPU, machine.MatID, podName) + services = append(services, dpuSvc) + } + } + + return services +} + +// ServiceDiff represents the differences between desired and existing services. +type ServiceDiff struct { + Create []*corev1.Service + Update []*corev1.Service + Recreate []*corev1.Service // Services that need delete+create due to immutable field changes + Delete []string +} + +// ComputeServiceDiff calculates the differences between desired and existing services. +func ComputeServiceDiff(desired []*corev1.Service, existing []*corev1.Service) ServiceDiff { + diff := ServiceDiff{} + + existingMap := make(map[string]*corev1.Service) + for _, svc := range existing { + existingMap[svc.Name] = svc + } + + desiredMap := make(map[string]*corev1.Service) + for _, svc := range desired { + desiredMap[svc.Name] = svc + } + + // Find services to create or update + for _, svc := range desired { + existingSvc, exists := existingMap[svc.Name] + if !exists { + diff.Create = append(diff.Create, svc) + } else if needsUpdate(svc, existingSvc) { + svc.ResourceVersion = existingSvc.ResourceVersion + // Check if ClusterIP is changing (immutable field) + if svc.Spec.ClusterIP != "" && existingSvc.Spec.ClusterIP != "" && + svc.Spec.ClusterIP != existingSvc.Spec.ClusterIP { + // ClusterIP changed - need to delete and recreate + diff.Recreate = append(diff.Recreate, svc) + } else { + // Preserve existing ClusterIP if not explicitly set + if svc.Spec.ClusterIP == "" { + svc.Spec.ClusterIP = existingSvc.Spec.ClusterIP + } + diff.Update = append(diff.Update, svc) + } + } + } + + // Find services to delete (managed by us but no longer desired) + for _, existing := range existing { + if _, wanted := desiredMap[existing.Name]; !wanted { + // Only delete if we manage this service + if existing.Labels[LabelManagedBy] == LabelManagedByValue { + diff.Delete = append(diff.Delete, existing.Name) + } + } + } + + return diff +} + +// needsUpdate checks if a service needs to be updated. +func needsUpdate(desired, existing *corev1.Service) bool { + // Check ports + if len(desired.Spec.Ports) != len(existing.Spec.Ports) { + return true + } + for i, port := range desired.Spec.Ports { + if i >= len(existing.Spec.Ports) { + return true + } + existingPort := existing.Spec.Ports[i] + if port.Name != existingPort.Name || + port.Port != existingPort.Port || + port.Protocol != existingPort.Protocol || + port.TargetPort.IntValue() != existingPort.TargetPort.IntValue() { + return true + } + } + + // Check selector + if len(desired.Spec.Selector) != len(existing.Spec.Selector) { + return true + } + for k, v := range desired.Spec.Selector { + if existing.Spec.Selector[k] != v { + return true + } + } + + // Check labels + if len(desired.Labels) != len(existing.Labels) { + return true + } + for k, v := range desired.Labels { + if existing.Labels[k] != v { + return true + } + } + + // Check annotations + if len(desired.Annotations) != len(existing.Annotations) { + return true + } + for k, v := range desired.Annotations { + if existing.Annotations[k] != v { + return true + } + } + + // Check ClusterIP change + if desired.Spec.ClusterIP != "" && existing.Spec.ClusterIP != "" && + desired.Spec.ClusterIP != existing.Spec.ClusterIP { + return true + } + + return false +} + +// K8sServiceClient defines the interface for Kubernetes service operations. +type K8sServiceClient interface { + List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) + Create(ctx context.Context, svc *corev1.Service) error + Update(ctx context.Context, svc *corev1.Service) error + Delete(ctx context.Context, namespace, name string) error +} + +// ReconcileResult holds the results of a reconciliation cycle. +type ReconcileResult struct { + Created int + Updated int + Deleted int + Recreated int + Errors []error +} + +// Reconciler reconciles Kubernetes Services with machine-a-tron machine status. +type Reconciler struct { + discovery *MatPodDiscovery + serviceBuilder *ServiceBuilder + k8sClient K8sServiceClient + clientOpts []matclient.Option + logger zerolog.Logger + concurrency int +} + +// NewReconciler creates a new Reconciler. +func NewReconciler( + discovery *MatPodDiscovery, + serviceBuilder *ServiceBuilder, + k8sClient K8sServiceClient, + clientOpts []matclient.Option, + logger zerolog.Logger, +) *Reconciler { + return &Reconciler{ + discovery: discovery, + serviceBuilder: serviceBuilder, + k8sClient: k8sClient, + clientOpts: clientOpts, + logger: logger, + concurrency: DefaultConcurrency, + } +} + +// SetConcurrency sets the number of concurrent workers for K8s API calls. +func (r *Reconciler) SetConcurrency(n int) { + if n > 0 { + r.concurrency = n + } +} + +// Reconcile performs a full reconciliation cycle. +func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { + result := ReconcileResult{} + + // Discover machine-a-tron instances + instances, err := r.discovery.Discover(ctx) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("discovering instances: %w", err)) + return result + } + + if len(instances) == 0 { + r.logger.Warn().Msg("no machine-a-tron instances discovered") + return result + } + + r.logger.Debug(). + Int("count", len(instances)). + Msg("discovered machine-a-tron instances") + + // Collect all desired services from all instances + var allDesired []*corev1.Service + fetchFailed := false + + for _, instance := range instances { + client, err := matclient.NewClient(instance.URL, r.clientOpts...) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("creating client for %s: %w", instance.URL, err)) + fetchFailed = true + continue + } + + r.logger.Debug(). + Str("url", instance.URL). + Msg("fetching machine status") + + status, err := client.GetMachinesStatus(ctx) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("fetching status from %s: %w", instance.URL, err)) + fetchFailed = true + continue + } + + r.logger.Debug(). + Str("url", instance.URL). + Str("pod", instance.PodName). + Int("machines", len(status.Machines)). + Msg("fetched machine status") + + services := r.serviceBuilder.BuildServicesFromStatus(status, instance.PodName) + allDesired = append(allDesired, services...) + } + + r.logger.Info(). + Int("total_services", len(allDesired)). + Msg("built desired services from all instances") + + // List existing services + existing, err := r.k8sClient.List(ctx, r.serviceBuilder.Namespace, + fmt.Sprintf("%s=%s", LabelManagedBy, LabelManagedByValue)) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("listing existing services: %w", err)) + return result + } + + // Compute and apply diff + diff := ComputeServiceDiff(allDesired, existing) + + r.logger.Info(). + Int("create", len(diff.Create)). + Int("update", len(diff.Update)). + Int("delete", len(diff.Delete)). + Int("recreate", len(diff.Recreate)). + Msg("computed service diff") + + // Process deletes first (needed for recreate to work) + // Skip deletions if any fetch failed to prevent spurious Service removal + if fetchFailed { + r.logger.Warn().Msg("skipping deletions due to partial status-fetch failures") + } + + // Process deletes concurrently + if !fetchFailed && len(diff.Delete) > 0 { + deleted := r.processDeletesConcurrently(ctx, diff.Delete) + result.Deleted = deleted + } + + // Process recreates (delete then create for immutable field changes like ClusterIP) + // Skip recreates if any fetch failed to prevent spurious Service removal + if !fetchFailed && len(diff.Recreate) > 0 { + recreated := r.processRecreatesConcurrently(ctx, diff.Recreate, &result) + result.Recreated = recreated + } + + // Process creates concurrently + if len(diff.Create) > 0 { + created := r.processCreatesConcurrently(ctx, diff.Create, &result) + result.Created = created + } + + // Process updates concurrently + if len(diff.Update) > 0 { + updated := r.processUpdatesConcurrently(ctx, diff.Update, &result) + result.Updated = updated + } + + return result +} + +// processDeletesConcurrently deletes services using a worker pool. +func (r *Reconciler) processDeletesConcurrently(ctx context.Context, names []string) int { + var deleted int64 + var wg sync.WaitGroup + sem := make(chan struct{}, r.concurrency) + + for i, name := range names { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(names)). + Msg("delete progress") + } + + wg.Add(1) + sem <- struct{}{} + + go func(name string) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { + r.logger.Error().Err(err).Str("service", name).Msg("failed to delete service") + } else { + atomic.AddInt64(&deleted, 1) + } + }(name) + } + + wg.Wait() + return int(deleted) +} + +// processRecreatesConcurrently handles services that need delete+create. +func (r *Reconciler) processRecreatesConcurrently(ctx context.Context, services []*corev1.Service, result *ReconcileResult) int { + var recreated int64 + var wg sync.WaitGroup + sem := make(chan struct{}, r.concurrency) + + for i, svc := range services { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(services)). + Msg("recreate progress") + } + + wg.Add(1) + sem <- struct{}{} + + go func(svc *corev1.Service) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, svc.Name); err != nil { + r.logger.Error().Err(err).Str("service", svc.Name).Msg("failed to delete service for recreate") + return + } + // Clear ResourceVersion for create + svc.ResourceVersion = "" + if err := r.k8sClient.Create(ctx, svc); err != nil { + r.logger.Error().Err(err).Str("service", svc.Name).Msg("failed to create service after delete") + } else { + atomic.AddInt64(&recreated, 1) + } + }(svc) + } + + wg.Wait() + return int(recreated) +} + +// processCreatesConcurrently creates services using a worker pool. +func (r *Reconciler) processCreatesConcurrently(ctx context.Context, services []*corev1.Service, result *ReconcileResult) int { + var created int64 + var wg sync.WaitGroup + var errMu sync.Mutex + sem := make(chan struct{}, r.concurrency) + + for i, svc := range services { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(services)). + Int64("created", atomic.LoadInt64(&created)). + Msg("create progress") + } + + wg.Add(1) + sem <- struct{}{} + + go func(svc *corev1.Service) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Create(ctx, svc); err != nil { + errMu.Lock() + result.Errors = append(result.Errors, fmt.Errorf("creating service %s: %w", svc.Name, err)) + errMu.Unlock() + } else { + atomic.AddInt64(&created, 1) + } + }(svc) + } + + wg.Wait() + return int(created) +} + +// processUpdatesConcurrently updates services using a worker pool. +func (r *Reconciler) processUpdatesConcurrently(ctx context.Context, services []*corev1.Service, result *ReconcileResult) int { + var updated int64 + var wg sync.WaitGroup + var errMu sync.Mutex + sem := make(chan struct{}, r.concurrency) + + for i, svc := range services { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(services)). + Int64("updated", atomic.LoadInt64(&updated)). + Msg("update progress") + } + + wg.Add(1) + sem <- struct{}{} + + go func(svc *corev1.Service) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Update(ctx, svc); err != nil { + errMu.Lock() + result.Errors = append(result.Errors, fmt.Errorf("updating service %s: %w", svc.Name, err)) + errMu.Unlock() + } else { + atomic.AddInt64(&updated, 1) + } + }(svc) + } + + wg.Wait() + return int(updated) +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go new file mode 100644 index 0000000000..fca963f730 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go @@ -0,0 +1,545 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/matclient" +) + +func TestBuildServiceName(t *testing.T) { + tests := []struct { + machineType string + matID string + want string + }{ + { + machineType: MachineTypeHost, + matID: "12345678-1234-1234-1234-123456789abc", + want: "mat-bmc-host-12345678-123", + }, + { + machineType: MachineTypeDPU, + matID: "abcdefgh-1234-1234-1234-123456789abc", + want: "mat-bmc-dpu-abcdefgh-123", + }, + { + machineType: MachineTypeHost, + matID: "short", + want: "mat-bmc-host-short", + }, + } + + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + got := BuildServiceName(tt.machineType, tt.matID) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestServiceBuilder_BuildService(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + BaseSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + machine := &matclient.MachineStatus{ + MatID: "host-uuid-12345678", + MachineID: ptr("nico-machine-id"), + HardwareType: ptr("GB200"), + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + } + + svc := builder.BuildService(machine, MachineTypeHost, "", "") + + // Check basic metadata + assert.Equal(t, "mat-bmc-host-host-uuid-12", svc.Name) + assert.Equal(t, "test-ns", svc.Namespace) + + // Check labels + assert.Equal(t, LabelManagedByValue, svc.Labels[LabelManagedBy]) + assert.Equal(t, "host-uuid-12345678", svc.Labels[LabelMatID]) + assert.Equal(t, "nico-machine-id", svc.Labels[LabelMachineID]) + assert.Equal(t, MachineTypeHost, svc.Labels[LabelMachineType]) + + // Check annotations + assert.Equal(t, "192.168.1.100", svc.Annotations[AnnotationBMCIP]) + assert.Equal(t, "Ready", svc.Annotations[AnnotationAPIState]) + assert.Equal(t, "On", svc.Annotations[AnnotationPowerState]) + assert.Equal(t, "GB200", svc.Annotations[AnnotationHardwareType]) + assert.Equal(t, "8443", svc.Annotations[AnnotationRedfishListenPort]) + + // Check ports + require.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, PortNameRedfish, svc.Spec.Ports[0].Name) + assert.Equal(t, corev1.ProtocolTCP, svc.Spec.Ports[0].Protocol) + assert.Equal(t, int32(443), svc.Spec.Ports[0].Port) + assert.Equal(t, intstr.FromInt32(8443), svc.Spec.Ports[0].TargetPort) + + // Check selector + assert.Equal(t, builder.BaseSelector, svc.Spec.Selector) +} + +func TestServiceBuilder_BuildService_WithIPMI(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + BaseSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + machine := &matclient.MachineStatus{ + MatID: "host-uuid-12345678", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + IPMI: &matclient.EndpointStatus{ + ReachablePort: 623, + ListenPort: 16023, + }, + }, + } + + svc := builder.BuildService(machine, MachineTypeHost, "", "") + + // Check we have both ports + require.Len(t, svc.Spec.Ports, 2) + + // Find ports by name + var redfishPort, ipmiPort *corev1.ServicePort + for i := range svc.Spec.Ports { + switch svc.Spec.Ports[i].Name { + case PortNameRedfish: + redfishPort = &svc.Spec.Ports[i] + case PortNameIPMI: + ipmiPort = &svc.Spec.Ports[i] + } + } + + require.NotNil(t, redfishPort) + assert.Equal(t, corev1.ProtocolTCP, redfishPort.Protocol) + assert.Equal(t, int32(443), redfishPort.Port) + assert.Equal(t, intstr.FromInt32(8443), redfishPort.TargetPort) + + require.NotNil(t, ipmiPort) + assert.Equal(t, corev1.ProtocolUDP, ipmiPort.Protocol) + assert.Equal(t, int32(623), ipmiPort.Port) + assert.Equal(t, intstr.FromInt32(16023), ipmiPort.TargetPort) + + // Check IPMI annotation + assert.Equal(t, "16023", svc.Annotations[AnnotationIPMIListenPort]) +} + +func TestServiceBuilder_BuildService_DPU(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + BaseSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + dpu := &matclient.MachineStatus{ + MatID: "dpu-uuid-12345678", + MachineID: ptr("dpu-machine-id"), + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.101"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8444, + }, + }, + } + + svc := builder.BuildService(dpu, MachineTypeDPU, "parent-host-uuid", "") + + // Check DPU-specific labels + assert.Equal(t, MachineTypeDPU, svc.Labels[LabelMachineType]) + assert.Equal(t, "parent-host-uuid", svc.Labels[LabelParentMatID]) +} + +func TestServiceBuilder_BuildService_BMCIPAsClusterIP(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + BaseSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + machine := &matclient.MachineStatus{ + MatID: "host-uuid-12345678", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("10.100.0.20"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + } + + svc := builder.BuildService(machine, MachineTypeHost, "", "") + + // Check BMC IP is used directly as ClusterIP + assert.Equal(t, "10.100.0.20", svc.Spec.ClusterIP) +} + +func TestServiceBuilder_BuildServicesFromStatus(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + BaseSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + status := &matclient.MachinesStatusResponse{ + Machines: []matclient.MachineStatus{ + { + MatID: "host-1", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8443}, + }, + DPUs: []matclient.MachineStatus{ + { + MatID: "dpu-1", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8444}, + }, + }, + { + MatID: "dpu-2", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8445}, + }, + }, + }, + }, + { + MatID: "host-2", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8446}, + }, + }, + }, + } + + services := builder.BuildServicesFromStatus(status, "") + + // Should have 4 services: 2 hosts + 2 DPUs + assert.Len(t, services, 4) + + // Verify parent links for DPUs + dpuServices := make([]*corev1.Service, 0) + for _, svc := range services { + if svc.Labels[LabelMachineType] == MachineTypeDPU { + dpuServices = append(dpuServices, svc) + } + } + + assert.Len(t, dpuServices, 2) + for _, svc := range dpuServices { + assert.Equal(t, "host-1", svc.Labels[LabelParentMatID]) + } +} + +func TestComputeServiceDiff(t *testing.T) { + tests := []struct { + name string + desired []*corev1.Service + existing []*corev1.Service + wantCreateCount int + wantUpdateCount int + wantDeleteCount int + }{ + { + name: "empty to empty", + desired: nil, + existing: nil, + wantCreateCount: 0, + wantUpdateCount: 0, + wantDeleteCount: 0, + }, + { + name: "create new service", + desired: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + existing: nil, + wantCreateCount: 1, + }, + { + name: "delete stale service", + desired: nil, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantDeleteCount: 1, + }, + { + name: "no changes needed", + desired: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantCreateCount: 0, + wantUpdateCount: 0, + wantDeleteCount: 0, + }, + { + name: "update port change", + desired: []*corev1.Service{ + func() *corev1.Service { + svc := makeTestService("svc-1", "mat-id-1") + svc.Spec.Ports[0].TargetPort = intstr.FromInt32(9999) + return svc + }(), + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantUpdateCount: 1, + }, + { + name: "update annotation change", + desired: []*corev1.Service{ + func() *corev1.Service { + svc := makeTestService("svc-1", "mat-id-1") + svc.Annotations[AnnotationAPIState] = "NotReady" + return svc + }(), + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantUpdateCount: 1, + }, + { + name: "mixed operations", + desired: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), // keep + makeTestService("svc-2", "mat-id-2"), // create + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), // keep + makeTestService("svc-3", "mat-id-3"), // delete + }, + wantCreateCount: 1, + wantDeleteCount: 1, + }, + { + name: "skip non-managed services on delete", + desired: nil, + existing: []*corev1.Service{ + func() *corev1.Service { + svc := makeTestService("svc-1", "mat-id-1") + svc.Labels[LabelManagedBy] = "someone-else" + return svc + }(), + }, + wantDeleteCount: 0, // should not delete services we don't manage + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + diff := ComputeServiceDiff(tt.desired, tt.existing) + + assert.Len(t, diff.Create, tt.wantCreateCount) + assert.Len(t, diff.Update, tt.wantUpdateCount) + assert.Len(t, diff.Delete, tt.wantDeleteCount) + }) + } +} + +func TestReconcileLogic(t *testing.T) { + ctx := context.Background() + + mockK8s := &mockK8sClient{ + services: make(map[string]*corev1.Service), + } + + builder := &ServiceBuilder{ + Namespace: "test-ns", + BaseSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + status := &matclient.MachinesStatusResponse{ + Machines: []matclient.MachineStatus{ + { + MatID: "host-1234", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + }, + }, + } + + // First reconcile: should create service + result := runTestReconcile(ctx, builder, mockK8s, status) + assert.Equal(t, 1, result.Created) + assert.Equal(t, 0, result.Updated) + assert.Equal(t, 0, result.Deleted) + assert.Empty(t, result.Errors) + assert.Len(t, mockK8s.services, 1) + + // Second reconcile with same state: no changes + result = runTestReconcile(ctx, builder, mockK8s, status) + assert.Equal(t, 0, result.Created) + assert.Equal(t, 0, result.Updated) + assert.Equal(t, 0, result.Deleted) + + // Third reconcile with empty status: should delete + result = runTestReconcile(ctx, builder, mockK8s, &matclient.MachinesStatusResponse{}) + assert.Equal(t, 0, result.Created) + assert.Equal(t, 0, result.Updated) + assert.Equal(t, 1, result.Deleted) + assert.Empty(t, mockK8s.services) +} + +// runTestReconcile tests the reconciliation logic without discovery. +func runTestReconcile(ctx context.Context, builder *ServiceBuilder, k8s *mockK8sClient, status *matclient.MachinesStatusResponse) ReconcileResult { + result := ReconcileResult{} + + desired := builder.BuildServicesFromStatus(status, "") + + selector := LabelManagedBy + "=" + LabelManagedByValue + existing, _ := k8s.List(ctx, builder.Namespace, selector) + + diff := ComputeServiceDiff(desired, existing) + + for _, svc := range diff.Create { + if err := k8s.Create(ctx, svc); err != nil { + result.Errors = append(result.Errors, err) + } else { + result.Created++ + } + } + + for _, svc := range diff.Update { + if err := k8s.Update(ctx, svc); err != nil { + result.Errors = append(result.Errors, err) + } else { + result.Updated++ + } + } + + for _, name := range diff.Delete { + if err := k8s.Delete(ctx, builder.Namespace, name); err != nil { + result.Errors = append(result.Errors, err) + } else { + result.Deleted++ + } + } + + return result +} + +// makeTestService creates a test service with standard labels. +func makeTestService(name, matID string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-ns", + Labels: map[string]string{ + LabelManagedBy: LabelManagedByValue, + LabelMatID: matID, + LabelMachineType: MachineTypeHost, + }, + Annotations: map[string]string{ + AnnotationAPIState: "Ready", + AnnotationPowerState: "On", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: PortNameRedfish, + Protocol: corev1.ProtocolTCP, + Port: 443, + TargetPort: intstr.FromInt32(8443), + }, + }, + }, + } +} + +// Mock implementations + +type mockK8sClient struct { + services map[string]*corev1.Service +} + +func (m *mockK8sClient) List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) { + result := make([]*corev1.Service, 0) + for _, svc := range m.services { + if svc.Labels[LabelManagedBy] == LabelManagedByValue { + result = append(result, svc) + } + } + return result, nil +} + +func (m *mockK8sClient) Create(ctx context.Context, svc *corev1.Service) error { + m.services[svc.Name] = svc.DeepCopy() + return nil +} + +func (m *mockK8sClient) Update(ctx context.Context, svc *corev1.Service) error { + m.services[svc.Name] = svc.DeepCopy() + return nil +} + +func (m *mockK8sClient) Delete(ctx context.Context, namespace, name string) error { + delete(m.services, name) + return nil +} + +func ptr[T any](v T) *T { + return &v +} + diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go new file mode 100644 index 0000000000..152e0502b4 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// DefaultDiscoverySelector is the default label selector for discovering +// machine-a-tron bmc-mock Services. +const DefaultDiscoverySelector = "nvidia-infra-controller/mat-service=true" + +// DiscoveredInstance represents a discovered machine-a-tron instance. +type DiscoveredInstance struct { + URL string + PodName string +} + +// MatPodDiscovery discovers machine-a-tron pods via their bmc-mock Services. +type MatPodDiscovery struct { + clientset kubernetes.Interface + namespace string + port int + labelSelector string +} + +// NewMatPodDiscovery creates a new MatPodDiscovery. +// If labelSelector is empty, uses DefaultDiscoverySelector. +func NewMatPodDiscovery(clientset kubernetes.Interface, namespace string, port int, labelSelector string) *MatPodDiscovery { + if labelSelector == "" { + labelSelector = DefaultDiscoverySelector + } + return &MatPodDiscovery{ + clientset: clientset, + namespace: namespace, + port: port, + labelSelector: labelSelector, + } +} + +// Discover finds all machine-a-tron bmc-mock services and returns their URLs with pod names. +func (d *MatPodDiscovery) Discover(ctx context.Context) ([]DiscoveredInstance, error) { + services, err := d.clientset.CoreV1().Services(d.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: d.labelSelector, + }) + if err != nil { + return nil, fmt.Errorf("listing services with selector %q: %w", d.labelSelector, err) + } + + var instances []DiscoveredInstance + for _, svc := range services.Items { + url := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", svc.Name, d.namespace, d.port) + podName := svc.Labels[LabelPodName] + instances = append(instances, DiscoveredInstance{ + URL: url, + PodName: podName, + }) + } + + return instances, nil +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/k8s_client.go b/dev/k8s/machine-a-tron-controller/pkg/controller/k8s_client.go new file mode 100644 index 0000000000..ec54dde3f5 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/k8s_client.go @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// RealK8sServiceClient implements K8sServiceClient using the real Kubernetes API. +type RealK8sServiceClient struct { + clientset kubernetes.Interface +} + +// NewRealK8sServiceClient creates a new RealK8sServiceClient. +func NewRealK8sServiceClient(clientset kubernetes.Interface) *RealK8sServiceClient { + return &RealK8sServiceClient{clientset: clientset} +} + +// List returns all Services matching the label selector. +func (c *RealK8sServiceClient) List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) { + list, err := c.clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return nil, err + } + + result := make([]*corev1.Service, len(list.Items)) + for i := range list.Items { + result[i] = &list.Items[i] + } + return result, nil +} + +// Create creates a new Service. +func (c *RealK8sServiceClient) Create(ctx context.Context, svc *corev1.Service) error { + _, err := c.clientset.CoreV1().Services(svc.Namespace).Create(ctx, svc, metav1.CreateOptions{}) + return err +} + +// Update updates an existing Service. +func (c *RealK8sServiceClient) Update(ctx context.Context, svc *corev1.Service) error { + _, err := c.clientset.CoreV1().Services(svc.Namespace).Update(ctx, svc, metav1.UpdateOptions{}) + return err +} + +// Delete deletes a Service by name. +func (c *RealK8sServiceClient) Delete(ctx context.Context, namespace, name string) error { + return c.clientset.CoreV1().Services(namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go new file mode 100644 index 0000000000..8aec987a47 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package matclient provides an HTTP client for the machine-a-tron API. +package matclient + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/rs/zerolog" +) + +const ( + // maxResponseSize is the maximum allowed response body size (10 MB). + maxResponseSize = 10 * 1024 * 1024 + // maxErrorBodySize is the maximum error body size to include in error messages. + maxErrorBodySize = 1024 +) + +// Client is an HTTP client for the machine-a-tron API. +type Client struct { + baseURL string + httpClient *http.Client + logger zerolog.Logger + insecureSkipVerify bool +} + +// Option configures a Client. +type Option func(*Client) + +// WithHTTPClient sets a custom HTTP client. +func WithHTTPClient(c *http.Client) Option { + return func(client *Client) { + client.httpClient = c + } +} + +// WithLogger sets the logger. +func WithLogger(logger zerolog.Logger) Option { + return func(client *Client) { + client.logger = logger + } +} + +// WithInsecureSkipVerify disables TLS certificate verification. +// Use only for development with self-signed certificates. +func WithInsecureSkipVerify() Option { + return func(client *Client) { + client.insecureSkipVerify = true + } +} + +// NewClient creates a new machine-a-tron API client. +// baseURL must be an absolute HTTP(S) URL with a host. +func NewClient(baseURL string, opts ...Option) (*Client, error) { + parsed, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("invalid base URL: %w", err) + } + + // Validate URL is absolute with proper scheme and host + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return nil, fmt.Errorf("invalid base URL: scheme must be http or https, got %q", parsed.Scheme) + } + if parsed.Host == "" { + return nil, fmt.Errorf("invalid base URL: missing host") + } + if parsed.RawQuery != "" { + return nil, fmt.Errorf("invalid base URL: query string not allowed") + } + if parsed.Fragment != "" { + return nil, fmt.Errorf("invalid base URL: fragment not allowed") + } + + // Normalize: rebuild from parsed components, trim trailing slash + parsed.Path = strings.TrimSuffix(parsed.Path, "/") + normalizedURL := parsed.String() + + c := &Client{ + baseURL: normalizedURL, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + logger: zerolog.Nop(), + } + + for _, opt := range opts { + opt(c) + } + + // Apply insecure TLS after all options to preserve existing transport settings + if c.insecureSkipVerify { + c.applyInsecureTLS() + } + + return c, nil +} + +// applyInsecureTLS configures the client to skip TLS verification while preserving +// existing transport settings. +func (c *Client) applyInsecureTLS() { + transport, ok := c.httpClient.Transport.(*http.Transport) + if !ok || transport == nil { + transport = http.DefaultTransport.(*http.Transport).Clone() + } else { + transport = transport.Clone() + } + + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.InsecureSkipVerify = true //nolint:gosec // Intentional for dev/test with self-signed certs + transport.TLSClientConfig.MinVersion = tls.VersionTLS12 + + c.httpClient.Transport = transport +} + +// GetMachinesStatus fetches the current machine status from machine-a-tron. +func (c *Client) GetMachinesStatus(ctx context.Context) (*MachinesStatusResponse, error) { + reqURL := c.baseURL + "/machines/status" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Accept", "application/json") + + c.logger.Debug().Str("url", reqURL).Msg("fetching machine status") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("executing request: %w", err) + } + defer resp.Body.Close() + + // Limit response body size to prevent memory exhaustion + limitedReader := io.LimitReader(resp.Body, maxResponseSize) + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodySize)) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, truncateString(string(body), maxErrorBodySize)) + } + + var result MachinesStatusResponse + if err := json.NewDecoder(limitedReader).Decode(&result); err != nil { + return nil, fmt.Errorf("decoding response: %w", err) + } + + c.logger.Debug().Int("machine_count", len(result.Machines)).Msg("fetched machine status") + + return &result, nil +} + +// truncateString truncates a string to maxLen and adds "..." if truncated. +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen-3] + "..." +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/client_test.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/client_test.go new file mode 100644 index 0000000000..ba901d001a --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/client_test.go @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package matclient + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClient_GetMachinesStatus(t *testing.T) { + tests := []struct { + name string + serverResponse *MachinesStatusResponse + serverStatus int + wantErr bool + errContains string + }{ + { + name: "empty machines list", + serverResponse: &MachinesStatusResponse{ + Machines: []MachineStatus{}, + }, + serverStatus: http.StatusOK, + }, + { + name: "single host with DPU", + serverResponse: &MachinesStatusResponse{ + Machines: []MachineStatus{ + { + MatID: "host-uuid-1", + MachineID: ptr("machine-123"), + APIState: "Ready", + PowerState: "On", + BMC: BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + DPUs: []MachineStatus{ + { + MatID: "dpu-uuid-1", + MachineID: ptr("dpu-456"), + APIState: "Ready", + PowerState: "On", + BMC: BMCStatus{ + IP: ptr("192.168.1.101"), + Redfish: EndpointStatus{ + ReachablePort: 443, + ListenPort: 8444, + }, + }, + }, + }, + }, + }, + }, + serverStatus: http.StatusOK, + }, + { + name: "server error", + serverStatus: http.StatusInternalServerError, + wantErr: true, + errContains: "unexpected status 500", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/machines/status", r.URL.Path) + assert.Equal(t, http.MethodGet, r.Method) + assert.Equal(t, "application/json", r.Header.Get("Accept")) + + w.WriteHeader(tt.serverStatus) + if tt.serverResponse != nil { + err := json.NewEncoder(w).Encode(tt.serverResponse) + require.NoError(t, err) + } + })) + defer server.Close() + + client, err := NewClient(server.URL, WithLogger(zerolog.Nop())) + require.NoError(t, err) + + resp, err := client.GetMachinesStatus(context.Background()) + + if tt.wantErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + assert.Equal(t, tt.serverResponse, resp) + }) + } +} + +func TestNewClient_InvalidURL(t *testing.T) { + _, err := NewClient("://invalid") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid base URL") +} + +func ptr[T any](v T) *T { + return &v +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/types.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/types.go new file mode 100644 index 0000000000..be4ef5951c --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/types.go @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package matclient provides an HTTP client for the machine-a-tron API. +package matclient + +// MachinesStatusResponse is the top-level response from GET /machines/status. +type MachinesStatusResponse struct { + Machines []MachineStatus `json:"machines"` +} + +// MachineStatus represents the status of a single machine (host or DPU). +type MachineStatus struct { + // MatID is the machine-a-tron internal identifier (UUID). + MatID string `json:"mat_id"` + // MachineID is the observed NICo machine ID, if known. + MachineID *string `json:"machine_id,omitempty"` + // HardwareType is the hardware model (e.g., "GB200", "DGX"). + HardwareType *string `json:"hardware_type,omitempty"` + // MatState is the machine-a-tron FSM state. + MatState *string `json:"mat_state,omitempty"` + // APIState is the observed NICo API state. + APIState string `json:"api_state"` + // PowerState is the current power state ("On", "Off", etc.). + PowerState string `json:"power_state"` + // MachineIP is the machine's management IP, if known. + MachineIP *string `json:"machine_ip,omitempty"` + // BMC contains BMC endpoint information. + BMC BMCStatus `json:"bmc"` + // DPUs contains nested DPU statuses for host machines. + DPUs []MachineStatus `json:"dpus,omitempty"` +} + +// BMCStatus contains the BMC endpoint configuration. +type BMCStatus struct { + // IP is the BMC IP address, if assigned. + IP *string `json:"ip,omitempty"` + // Redfish contains the Redfish endpoint configuration. + Redfish EndpointStatus `json:"redfish"` + // IPMI contains the IPMI endpoint configuration, if enabled. + IPMI *EndpointStatus `json:"ipmi,omitempty"` +} + +// EndpointStatus describes port mapping for an endpoint. +type EndpointStatus struct { + // ReachablePort is the external port clients should connect to (e.g., 443 for Redfish). + ReachablePort uint16 `json:"reachable_port"` + // ListenPort is the internal port the mock BMC listens on. + ListenPort uint16 `json:"listen_port"` +} diff --git a/helm/charts/nico-machine-a-tron/Chart.yaml b/helm/charts/nico-machine-a-tron/Chart.yaml index 5f6ceac8d6..2d64884f9a 100644 --- a/helm/charts/nico-machine-a-tron/Chart.yaml +++ b/helm/charts/nico-machine-a-tron/Chart.yaml @@ -10,3 +10,8 @@ keywords: - mock - testing - development +dependencies: + - name: mat-k8s-controller + version: "0.1.0" + condition: mat-k8s-controller.enabled + repository: "" diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 933d1c4042..4fb8cf0b70 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -98,7 +98,7 @@ pods: compute: hwType: wiwynn_gb200_nvl hostCount: 252 # 18 trays × 14 racks - dpuPerHostCount: 2 # 2 BF3 per tray → 504 DPU BMCs + dpuPerHostCount: 2 # 2 BF3 per tray -> 504 DPU BMCs oobDhcpRelayAddress: "10.100.0.1" switches: hwType: nvidia_switch_nd5200_ld diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/Chart.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/Chart.yaml new file mode 100644 index 0000000000..2fab72d0c5 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: mat-k8s-controller +description: Kubernetes controller for machine-a-tron BMC service reconciliation +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl new file mode 100644 index 0000000000..b23e8eea21 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl @@ -0,0 +1,77 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "mat-k8s-controller.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "mat-k8s-controller.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mat-k8s-controller.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "mat-k8s-controller.labels" -}} +helm.sh/chart: {{ include "mat-k8s-controller.chart" . }} +{{ include "mat-k8s-controller.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "mat-k8s-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "mat-k8s-controller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "mat-k8s-controller.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "mat-k8s-controller.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Namespace - inherited from parent release +*/}} +{{- define "mat-k8s-controller.namespace" -}} +{{- .Release.Namespace }} +{{- end }} + +{{/* +Discovery selector - finds machine-a-tron bmc-mock Services +*/}} +{{- define "mat-k8s-controller.discoverySelector" -}} +{{- default "nvidia-infra-controller/mat-service=true" .Values.config.discoverySelector }} +{{- end }} + +{{/* +Target selector - matches machine-a-tron pods +*/}} +{{- define "mat-k8s-controller.targetSelector" -}} +{{- default "app.kubernetes.io/name=nico-machine-a-tron" .Values.config.targetSelector }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml new file mode 100644 index 0000000000..5ece9c4514 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml @@ -0,0 +1,44 @@ +{{- if .Values.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mat-k8s-controller.fullname" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "mat-k8s-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "mat-k8s-controller.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "mat-k8s-controller.serviceAccountName" . }} + containers: + - name: controller + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --namespace={{ include "mat-k8s-controller.namespace" . }} + - --discovery-selector={{ include "mat-k8s-controller.discoverySelector" . }} + - --sync-interval={{ .Values.config.syncInterval }} + - --insecure-skip-verify={{ .Values.config.insecureSkipVerify }} + - --log-level={{ .Values.config.logLevel }} + - --target-selector={{ include "mat-k8s-controller.targetSelector" . }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + securityContext: + fsGroup: 65534 + runAsNonRoot: true +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/rbac.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/rbac.yaml new file mode 100644 index 0000000000..9793d493f1 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/rbac.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "mat-k8s-controller.fullname" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "mat-k8s-controller.fullname" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "mat-k8s-controller.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "mat-k8s-controller.serviceAccountName" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/serviceaccount.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/serviceaccount.yaml new file mode 100644 index 0000000000..9433af86a6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mat-k8s-controller.serviceAccountName" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml new file mode 100644 index 0000000000..7e3f186477 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -0,0 +1,53 @@ +## mat-k8s-controller - Kubernetes controller for machine-a-tron services +## +## This controller auto-discovers machine-a-tron pods via their bmc-mock Services +## and creates/updates/deletes Kubernetes Services for each endpoint. + +## Enable/disable the controller +enabled: false + +image: + repository: mat-k8s-controller + tag: latest + pullPolicy: IfNotPresent + +## Resource limits +resources: + limits: + cpu: "1" + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +## Controller configuration +config: + ## Label selector for discovering machine-a-tron Services + ## Default: nvidia-infra-controller/mat-service=true,app.kubernetes.io/instance= + discoverySelector: "" + + ## Pod selector for created Services (routes traffic to mat pods) + ## Default: app.kubernetes.io/name=nico-machine-a-tron,app.kubernetes.io/instance= + targetSelector: "" + + ## Reconciliation interval + syncInterval: "30s" + + ## Skip TLS certificate verification + ## WARNING: Only enable for development with self-signed certificates + insecureSkipVerify: false + + ## Log level (debug, info, warn, error) + logLevel: "info" + +serviceAccount: + create: true + name: "" + annotations: {} + +rbac: + create: true + +## Override names (useful when release name differs from chart name) +nameOverride: "" +fullnameOverride: "" diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml index 54f2353f2a..6bbc29db89 100644 --- a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml +++ b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml @@ -1,4 +1,4 @@ -{{- if .Values.bmcServices.enabled }} +{{- if and .Values.bmcServices.enabled (not (index .Values "mat-k8s-controller" "enabled")) }} {{- $root := . }} {{- $namespace := include "nico-machine-a-tron.namespace" . }} {{- $baseName := include "nico-machine-a-tron.name" . }} @@ -107,7 +107,7 @@ metadata: labels: {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} app.kubernetes.io/component: bmc-service - nvidia-infra-controller/bmc-ip: "{{ $currentIP }}" + nvidia-infra-controller/mat-bmc-ip: "{{ $currentIP }}" nvidia-infra-controller/bmc-index: "{{ $bmcIndex }}" nvidia-infra-controller/pod-name: {{ $podName | quote }} annotations: diff --git a/helm/charts/nico-machine-a-tron/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/templates/deployment.yaml index 05074aa06f..b041a0fb63 100644 --- a/helm/charts/nico-machine-a-tron/templates/deployment.yaml +++ b/helm/charts/nico-machine-a-tron/templates/deployment.yaml @@ -14,7 +14,7 @@ Count pods that should be deployed: {{- $activePods = add $activePods 1 }} {{- end }} {{- else }} -{{- if $podConfig.machines }} +{{- if and $podConfig.machines (gt (len $podConfig.machines) 0) }} {{- $activePods = add $activePods 1 }} {{- end }} {{- end }} @@ -24,14 +24,14 @@ Count pods that should be deployed: {{/* Skip conditions: - When bmcServices.enabled: skip pods without CIDR -- Otherwise: skip pods without machines +- Otherwise: skip pods without machines (or empty machines) */}} {{- if $root.Values.bmcServices.enabled }} {{- if not $podConfig.cidr }} {{- continue }} {{- end }} {{- else }} -{{- if not $podConfig.machines }} +{{- if not (and $podConfig.machines (gt (len $podConfig.machines) 0)) }} {{- continue }} {{- end }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/service.yaml b/helm/charts/nico-machine-a-tron/templates/service.yaml index a8faab7fc6..ffbf9bb2b0 100644 --- a/helm/charts/nico-machine-a-tron/templates/service.yaml +++ b/helm/charts/nico-machine-a-tron/templates/service.yaml @@ -45,6 +45,7 @@ metadata: labels: {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} app: {{ $baseName }} + nvidia-infra-controller/mat-service: "true" {{- if gt $activePods 1 }} nvidia-infra-controller/pod-name: {{ $podName | quote }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index 7fe7603473..ae24b88d63 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -354,3 +354,29 @@ configFiles: # oob_dhcp_relay_address = "10.100.0.1" # admin_dhcp_relay_address = "192.168.176.1" matConfigs: {} + +## ============================================================================= +## Machine-a-tron Kubernetes Controller +## ============================================================================= +## Discovers machine-a-tron pods and creates K8s Services for each mock BMC. +## Enable for multi-pod deployments to dynamically manage BMC Services. +## +## Auto-discovered: +## - Finds bmc-mock Services via label: nvidia-infra-controller/mat-service=true +## - Polls /machines/status from each discovered instance +## +## Note: BMC IP must be within Kubernetes ServiceCIDR (default 10.96.0.0/12) +## for ClusterIP assignment. See README troubleshooting for ClusterIP conflicts. +## +mat-k8s-controller: + enabled: false + + image: + repository: mat-k8s-controller + tag: latest + pullPolicy: IfNotPresent + + config: + syncInterval: "30s" + insecureSkipVerify: false + logLevel: "info"