From 5930f3622a0cd062116302c27884396c0c93531f Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 16 Jul 2026 10:44:01 -0600 Subject: [PATCH 01/17] feat(machine-a-tron): k8s controller --- dev/k8s/machine-a-tron-controller/Dockerfile | 25 + dev/k8s/machine-a-tron-controller/Makefile | 54 ++ dev/k8s/machine-a-tron-controller/README.md | 217 +++++++ .../cmd/mat-k8s-controller/main.go | 212 +++++++ dev/k8s/machine-a-tron-controller/go.mod | 58 ++ dev/k8s/machine-a-tron-controller/go.sum | 167 ++++++ .../pkg/controller/controller.go | 374 ++++++++++++ .../pkg/controller/controller_test.go | 559 ++++++++++++++++++ .../pkg/controller/k8s_client.go | 55 ++ .../pkg/matclient/client.go | 96 +++ .../pkg/matclient/client_test.go | 119 ++++ .../pkg/matclient/types.go | 50 ++ 12 files changed, 1986 insertions(+) create mode 100644 dev/k8s/machine-a-tron-controller/Dockerfile create mode 100644 dev/k8s/machine-a-tron-controller/Makefile create mode 100644 dev/k8s/machine-a-tron-controller/README.md create mode 100644 dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go create mode 100644 dev/k8s/machine-a-tron-controller/go.mod create mode 100644 dev/k8s/machine-a-tron-controller/go.sum create mode 100644 dev/k8s/machine-a-tron-controller/pkg/controller/controller.go create mode 100644 dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go create mode 100644 dev/k8s/machine-a-tron-controller/pkg/controller/k8s_client.go create mode 100644 dev/k8s/machine-a-tron-controller/pkg/matclient/client.go create mode 100644 dev/k8s/machine-a-tron-controller/pkg/matclient/client_test.go create mode 100644 dev/k8s/machine-a-tron-controller/pkg/matclient/types.go diff --git a/dev/k8s/machine-a-tron-controller/Dockerfile b/dev/k8s/machine-a-tron-controller/Dockerfile new file mode 100644 index 0000000000..f62797f4a8 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/Dockerfile @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +FROM golang:1.26.4-alpine AS builder + +WORKDIR /build + +# Copy go mod files first for caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source +COPY . . + +# Build static binary +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o mat-k8s-controller ./cmd/mat-k8s-controller + +# Runtime image +FROM gcr.io/distroless/static:nonroot + +COPY --from=builder /build/mat-k8s-controller /mat-k8s-controller + +USER nonroot:nonroot + +ENTRYPOINT ["/mat-k8s-controller"] diff --git a/dev/k8s/machine-a-tron-controller/Makefile b/dev/k8s/machine-a-tron-controller/Makefile new file mode 100644 index 0000000000..7f9f3dd5b2 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/Makefile @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: build test lint docker-build clean + +BINARY := mat-k8s-controller +DOCKER_IMAGE := mat-k8s-controller +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") + +# Build the binary +build: + CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/mat-k8s-controller + +# Run all tests +test: + go test -v -race ./... + +# Run tests with coverage +test-coverage: + go test -v -race -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html + +# Run linter +lint: + golangci-lint run ./... + +# Format code +fmt: + go fmt ./... + +# Build Docker image +docker-build: + docker build -t $(DOCKER_IMAGE):$(VERSION) . + +# Build Docker image with latest tag +docker-build-latest: docker-build + docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest + +# Clean build artifacts +clean: + rm -f $(BINARY) coverage.out coverage.html + +# Run locally (requires kubeconfig and machine-a-tron running) +run: + go run ./cmd/mat-k8s-controller --log-level debug + +# Verify dependencies +verify: + go mod verify + go mod tidy + @if [ -n "$$(git status --porcelain go.mod go.sum)" ]; then \ + echo "go.mod or go.sum changed after tidy"; \ + exit 1; \ + fi diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md new file mode 100644 index 0000000000..9eafd0b614 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -0,0 +1,217 @@ +# Machine-a-tron Kubernetes Controller + +A Kubernetes (K8s) controller that reconciles K8s services for Machine-a-tron (MAT) mock BMC endpoints. + +## Overview + +When Machine-a-tron runs in Kubernetes, mock BMCs need stable network exposure so that +NICo components can reach Redfish and IPMI/SOL endpoints. +This controller uses MAT machines status and reconciles one K8s service per mock BMC. + +Machine-a-tron itself remains Kubernetes-agnostic. This controller bridges the gap. + +## Features + +- Polls machine-a-tron `GET /machines/status` API +- Creates one K8s service per mock BMC (hosts and DPUs) +- Exposes Redfish TCP (port 443 -> internal listen port) +- Exposes IPMI UDP when enabled (port 623 -> internal listen port) +- Labels and annotates K8s services with machine/BMC identity +- Updates K8s services when machine status changes +- Deletes stale K8s services when machines disappear +- Supports static ClusterIP assignment for predictable addressing + +## Installation + +### From Source + +```bash +go build -o mat-k8s-controller ./cmd/mat-k8s-controller +``` + +### Docker + +```bash +docker build -t mat-k8s-controller . +``` + +### Kubernetes + +Deploy alongside Machine-a-tron using the Helm chart (coming soon). + +## Configuration + +The controller accepts configuration via flags or environment variables: + +| Flag | Env Var | Default | Description | +|------|---------|---------|-------------| +| `--mat-url` | `MAT_URL` | `http://machine-a-tron:8080` | Machine-a-tron base URL | +| `--namespace` | `NAMESPACE` | `nico-mat` | Kubernetes namespace for services | +| `--sync-interval` | `SYNC_INTERVAL` | `30s` | Interval between reconciliation passes | +| `--kubeconfig` | `KUBECONFIG` | (in-cluster) | Path to kubeconfig file | +| `--target-selector` | `TARGET_SELECTOR` | `app=machine-a-tron` | Pod selector for services | +| `--cluster-ip-prefix` | `CLUSTER_IP_PREFIX` | (none) | Prefix for static ClusterIP | +| `--log-level` | `LOG_LEVEL` | `info` | Log level (debug, info, warn, error) | + +### Static ClusterIP Assignment + +When `--cluster-ip-prefix` is set and a BMC has an assigned IP, the controller will +assign a predictable ClusterIP using the last two octets of the BMC IP. For example: + +- Prefix: `10.96` +- BMC IP: `172.20.0.20` +- ClusterIP: `10.96.0.20` + +This enables predictable addressing for testing and development. + +## Service Structure + +Each created Service has: + +### Labels + +| Label | Description | +|-------|-------------| +| `app.kubernetes.io/managed-by` | Always `mat-k8s-controller` | +| `machine-a-tron.nvidia.com/mat-id` | Machine-a-tron internal UUID | +| `machine-a-tron.nvidia.com/machine-id` | NICo machine ID (if known) | +| `machine-a-tron.nvidia.com/machine-type` | `host` or `dpu` | +| `machine-a-tron.nvidia.com/parent-mat-id` | Parent host mat-id (DPUs only) | + +### Annotations + +| Annotation | Description | +|------------|-------------| +| `machine-a-tron.nvidia.com/bmc-ip` | BMC IP address | +| `machine-a-tron.nvidia.com/api-state` | Machine API state | +| `machine-a-tron.nvidia.com/power-state` | Machine power state | +| `machine-a-tron.nvidia.com/hardware-type` | Hardware type (e.g., GB200) | +| `machine-a-tron.nvidia.com/redfish-listen-port` | Internal Redfish listen port | +| `machine-a-tron.nvidia.com/ipmi-listen-port` | Internal IPMI listen port (if enabled) | + +### Ports + +| Name | Protocol | Port | Description | +|------|----------|------|-------------| +| `redfish` | TCP | 443 | Redfish API | +| `ipmi` | UDP | 623 | IPMI (if enabled) | + +## Example + +Given this machine-a-tron status: + +```json +{ + "machines": [ + { + "mat_id": "abc12345-...", + "machine_id": "machine-001", + "api_state": "Ready", + "power_state": "On", + "bmc": { + "ip": "172.20.0.20", + "redfish": { + "reachable_port": 443, + "listen_port": 8443 + } + }, + "dpus": [ + { + "mat_id": "def67890-...", + "bmc": { + "ip": "172.20.0.21", + "redfish": { + "reachable_port": 443, + "listen_port": 8444 + } + } + } + ] + } + ] +} +``` + +The controller creates two Services: + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: mat-bmc-host-abc12345 + labels: + app.kubernetes.io/managed-by: mat-k8s-controller + machine-a-tron.nvidia.com/mat-id: abc12345-... + machine-a-tron.nvidia.com/machine-type: host +spec: + type: ClusterIP + selector: + app: machine-a-tron + ports: + - name: redfish + protocol: TCP + port: 443 + targetPort: 8443 +--- +apiVersion: v1 +kind: Service +metadata: + name: mat-bmc-dpu-def67890 + labels: + app.kubernetes.io/managed-by: mat-k8s-controller + machine-a-tron.nvidia.com/mat-id: def67890-... + machine-a-tron.nvidia.com/machine-type: dpu + machine-a-tron.nvidia.com/parent-mat-id: abc12345-... +spec: + type: ClusterIP + selector: + app: machine-a-tron + ports: + - name: redfish + protocol: TCP + port: 443 + targetPort: 8444 +``` + +## Development + +### Running Tests + +```bash +go test ./... +``` + +### Running Locally + +```bash +# With kubeconfig +./mat-k8s-controller --kubeconfig ~/.kube/config --mat-url http://localhost:8080 + +# In-cluster (when deployed as a pod) +./mat-k8s-controller +``` + +## Architecture + +```mermaid +flowchart LR + subgraph MAT[machine-a-tron] + API["/machines/status"] + end + + subgraph K8s[Kubernetes] + Controller[mat-k8s-controller] + subgraph Services + HostSvc[mat-bmc-host-xxx] + DpuSvc[mat-bmc-dpu-yyy] + end + end + + Controller -->|polls| API + Controller -->|creates/updates/deletes| Services +``` + +## Related + +- [GitHub Issue #3380](https://github.com/NVIDIA/infra-controller/issues/3380) - Original feature request +- [GitHub Issue #3379](https://github.com/NVIDIA/infra-controller/issues/3379) - Machine-a-tron status API (prerequisite) diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go new file mode 100644 index 0000000000..e908dda00c --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// mat-k8s-controller is a Kubernetes controller that reconciles Services +// for machine-a-tron mock BMC endpoints. +// +// It polls the machine-a-tron /machines/status API and creates/updates/deletes +// Kubernetes Services to expose Redfish (and optionally IPMI) endpoints for +// each mock BMC. +package main + +import ( + "context" + "flag" + "os" + "os/signal" + "syscall" + "time" + + "github.com/rs/zerolog" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/controller" + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/matclient" +) + +func main() { + // Flags + matURL := flag.String("mat-url", envOrDefault("MAT_URL", "http://machine-a-tron:8080"), + "Machine-a-tron base URL") + namespace := flag.String("namespace", envOrDefault("NAMESPACE", "default"), + "Kubernetes namespace for Services") + syncInterval := flag.Duration("sync-interval", parseDurationOrDefault("SYNC_INTERVAL", 10*time.Second), + "Interval between reconciliation passes") + kubeconfig := flag.String("kubeconfig", os.Getenv("KUBECONFIG"), + "Path to kubeconfig (uses in-cluster config if empty)") + targetSelector := flag.String("target-selector", envOrDefault("TARGET_SELECTOR", "app=machine-a-tron"), + "Pod selector for Services (comma-separated key=value pairs)") + clusterIPPrefix := flag.String("cluster-ip-prefix", os.Getenv("CLUSTER_IP_PREFIX"), + "Prefix for static ClusterIP assignment (e.g., 10.96)") + logLevel := flag.String("log-level", envOrDefault("LOG_LEVEL", "info"), + "Log level (debug, info, warn, error)") + + flag.Parse() + + // Setup logger + level, err := zerolog.ParseLevel(*logLevel) + if err != nil { + level = zerolog.InfoLevel + } + logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}). + Level(level). + With(). + Timestamp(). + Str("component", "mat-k8s-controller"). + Logger() + + logger.Info(). + Str("mat_url", *matURL). + Str("namespace", *namespace). + Dur("sync_interval", *syncInterval). + Str("target_selector", *targetSelector). + Str("cluster_ip_prefix", *clusterIPPrefix). + Msg("starting controller") + + // Create machine-a-tron client + matClient, err := matclient.NewClient(*matURL, matclient.WithLogger(logger)) + if err != nil { + logger.Fatal().Err(err).Msg("failed to create machine-a-tron client") + } + + // Create Kubernetes client + var k8sConfig *rest.Config + if *kubeconfig != "" { + k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) + } else { + k8sConfig, err = rest.InClusterConfig() + } + if err != nil { + logger.Fatal().Err(err).Msg("failed to create Kubernetes config") + } + + clientset, err := kubernetes.NewForConfig(k8sConfig) + if err != nil { + logger.Fatal().Err(err).Msg("failed to create Kubernetes clientset") + } + + // Parse target selector + selector := parseSelector(*targetSelector) + + // Create service builder + builder := &controller.ServiceBuilder{ + Namespace: *namespace, + TargetSelector: selector, + ClusterIPPrefix: *clusterIPPrefix, + } + + // Create reconciler + k8sClient := controller.NewRealK8sServiceClient(clientset) + reconciler := controller.NewReconciler(matClient, builder, k8sClient) + + // Setup signal handling + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + go func() { + sig := <-sigCh + logger.Info().Str("signal", sig.String()).Msg("received shutdown signal") + cancel() + }() + + // Run reconciliation loop + ticker := time.NewTicker(*syncInterval) + defer ticker.Stop() + + // Run initial reconciliation + runReconcile(ctx, reconciler, logger) + + for { + select { + case <-ctx.Done(): + logger.Info().Msg("shutting down") + return + case <-ticker.C: + runReconcile(ctx, reconciler, logger) + } + } +} + +func runReconcile(ctx context.Context, r *controller.Reconciler, logger zerolog.Logger) { + start := time.Now() + result := r.Reconcile(ctx) + elapsed := time.Since(start) + + logEvent := logger.Info(). + Int("created", result.Created). + Int("updated", result.Updated). + Int("deleted", result.Deleted). + Dur("elapsed", elapsed) + + if len(result.Errors) > 0 { + logEvent = logger.Error(). + Int("created", result.Created). + Int("updated", result.Updated). + Int("deleted", result.Deleted). + Int("errors", len(result.Errors)). + Dur("elapsed", elapsed) + + for _, err := range result.Errors { + logger.Error().Err(err).Msg("reconciliation error") + } + } + + logEvent.Msg("reconciliation complete") +} + +func envOrDefault(key, defaultValue string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultValue +} + +func parseDurationOrDefault(envKey string, defaultValue time.Duration) time.Duration { + if v := os.Getenv(envKey); v != "" { + if d, err := time.ParseDuration(v); err == nil { + return d + } + } + return defaultValue +} + +func parseSelector(s string) map[string]string { + result := make(map[string]string) + if s == "" { + return result + } + + // Simple parser for key=value,key2=value2 format + pairs := splitPairs(s, ',') + for _, pair := range pairs { + kv := splitPairs(pair, '=') + if len(kv) == 2 { + result[kv[0]] = kv[1] + } + } + return result +} + +func splitPairs(s string, sep rune) []string { + var result []string + var current string + for _, r := range s { + if r == sep { + if current != "" { + result = append(result, current) + } + current = "" + } else { + current += string(r) + } + } + if current != "" { + result = append(result, current) + } + return result +} diff --git a/dev/k8s/machine-a-tron-controller/go.mod b/dev/k8s/machine-a-tron-controller/go.mod new file mode 100644 index 0000000000..ccf7d728a1 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/go.mod @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +module github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller + +go 1.26.4 + +require ( + github.com/rs/zerolog v1.34.0 + github.com/stretchr/testify v1.10.0 + k8s.io/api v0.32.4 + k8s.io/apimachinery v0.32.4 + k8s.io/client-go v0.32.4 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/dev/k8s/machine-a-tron-controller/go.sum b/dev/k8s/machine-a-tron-controller/go.sum new file mode 100644 index 0000000000..922bf09481 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/go.sum @@ -0,0 +1,167 @@ +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.4 h1:kw8Y/G8E7EpNy7gjB8gJZl3KJkNz8HM2YHrZPtAZsF4= +k8s.io/api v0.32.4/go.mod h1:5MYFvLvweRhyKylM3Es/6uh/5hGp0dg82vP34KifX4g= +k8s.io/apimachinery v0.32.4 h1:8EEksaxA7nd7xWJkkwLDN4SvWS5ot9g6Z/VZb3ju25I= +k8s.io/apimachinery v0.32.4/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.4 h1:zaGJS7xoYOYumoWIFXlcVrsiYioRPrXGO7dBfVC5R6M= +k8s.io/client-go v0.32.4/go.mod h1:k0jftcyYnEtwlFW92xC7MTtFv5BNcZBr+zn9jPlT9Ic= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go new file mode 100644 index 0000000000..8df18301d8 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -0,0 +1,374 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package controller implements the Kubernetes Service reconciliation logic +// for machine-a-tron mock BMC endpoints. +package controller + +import ( + "context" + "fmt" + "strconv" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/matclient" +) + +const ( + // LabelManagedBy identifies the controller managing the resource. + LabelManagedBy = "app.kubernetes.io/managed-by" + // LabelManagedByValue is the value for the managed-by label. + LabelManagedByValue = "mat-k8s-controller" + + // LabelMatID is the machine-a-tron ID label. + LabelMatID = "machine-a-tron.nvidia.com/mat-id" + // LabelMachineID is the NICo machine ID label. + LabelMachineID = "machine-a-tron.nvidia.com/machine-id" + // LabelMachineType indicates if this is a host or DPU. + LabelMachineType = "machine-a-tron.nvidia.com/machine-type" + // LabelParentMatID is the parent host's mat-id for DPU services. + LabelParentMatID = "machine-a-tron.nvidia.com/parent-mat-id" + + // AnnotationBMCIP stores the BMC IP address. + AnnotationBMCIP = "machine-a-tron.nvidia.com/bmc-ip" + // AnnotationAPIState stores the machine's API state. + AnnotationAPIState = "machine-a-tron.nvidia.com/api-state" + // AnnotationPowerState stores the machine's power state. + AnnotationPowerState = "machine-a-tron.nvidia.com/power-state" + // AnnotationHardwareType stores the hardware type. + AnnotationHardwareType = "machine-a-tron.nvidia.com/hardware-type" + // AnnotationRedfishListenPort stores the internal Redfish listen port. + AnnotationRedfishListenPort = "machine-a-tron.nvidia.com/redfish-listen-port" + // AnnotationIPMIListenPort stores the internal IPMI listen port. + AnnotationIPMIListenPort = "machine-a-tron.nvidia.com/ipmi-listen-port" + + // MachineTypeHost indicates a host machine. + MachineTypeHost = "host" + // MachineTypeDPU indicates a DPU. + MachineTypeDPU = "dpu" + + // PortNameRedfish is the name of the Redfish port in the Service. + PortNameRedfish = "redfish" + // PortNameIPMI is the name of the IPMI port in the Service. + PortNameIPMI = "ipmi" +) + +// ServiceBuilder builds Kubernetes Services from machine status. +type ServiceBuilder struct { + // Namespace is the target namespace for Services. + Namespace string + // TargetSelector is the pod selector that Services should target. + // This should match the machine-a-tron pod labels. + TargetSelector map[string]string + // ClusterIPPrefix is a prefix for static ClusterIP assignment. + // If set along with BMC IP, Services get predictable IPs. + // Format: "10.96.{last-two-octets-of-bmc-ip}" + ClusterIPPrefix string +} + +// BuildServiceName generates a consistent service name for a machine. +func BuildServiceName(machineType, matID string) string { + // Use first 8 chars of mat-id for reasonable length + shortID := matID + if len(matID) > 8 { + shortID = matID[:8] + } + return fmt.Sprintf("mat-bmc-%s-%s", machineType, shortID) +} + +// BuildService creates a Kubernetes Service for a machine's BMC. +func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineType, parentMatID string) *corev1.Service { + name := BuildServiceName(machineType, machine.MatID) + + labels := map[string]string{ + LabelManagedBy: LabelManagedByValue, + LabelMatID: machine.MatID, + LabelMachineType: machineType, + } + + annotations := map[string]string{ + AnnotationAPIState: machine.APIState, + AnnotationPowerState: machine.PowerState, + AnnotationRedfishListenPort: strconv.Itoa(int(machine.BMC.Redfish.ListenPort)), + } + + if machine.MachineID != nil { + labels[LabelMachineID] = *machine.MachineID + } + + if parentMatID != "" { + labels[LabelParentMatID] = parentMatID + } + + if machine.BMC.IP != nil { + annotations[AnnotationBMCIP] = *machine.BMC.IP + } + + if machine.HardwareType != nil { + annotations[AnnotationHardwareType] = *machine.HardwareType + } + + if machine.BMC.IPMI != nil { + annotations[AnnotationIPMIListenPort] = strconv.Itoa(int(machine.BMC.IPMI.ListenPort)) + } + + ports := []corev1.ServicePort{ + { + Name: PortNameRedfish, + Protocol: corev1.ProtocolTCP, + Port: int32(machine.BMC.Redfish.ReachablePort), + TargetPort: intstr.FromInt32(int32(machine.BMC.Redfish.ListenPort)), + }, + } + + if machine.BMC.IPMI != nil { + ports = append(ports, corev1.ServicePort{ + Name: PortNameIPMI, + Protocol: corev1.ProtocolUDP, + Port: int32(machine.BMC.IPMI.ReachablePort), + TargetPort: intstr.FromInt32(int32(machine.BMC.IPMI.ListenPort)), + }) + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: b.Namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: b.TargetSelector, + Ports: ports, + }, + } + + // Set static ClusterIP if configured and BMC IP is known + if b.ClusterIPPrefix != "" && machine.BMC.IP != nil { + clusterIP := buildStaticClusterIP(b.ClusterIPPrefix, *machine.BMC.IP) + if clusterIP != "" { + svc.Spec.ClusterIP = clusterIP + } + } + + return svc +} + +// BuildServicesFromStatus generates all Services from a machines status response. +func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatusResponse) []*corev1.Service { + var services []*corev1.Service + + for i := range status.Machines { + machine := &status.Machines[i] + + // Build service for the host BMC + svc := b.BuildService(machine, MachineTypeHost, "") + services = append(services, svc) + + // Build services for DPU BMCs + for j := range machine.DPUs { + dpu := &machine.DPUs[j] + dpuSvc := b.BuildService(dpu, MachineTypeDPU, machine.MatID) + services = append(services, dpuSvc) + } + } + + return services +} + +// buildStaticClusterIP constructs a ClusterIP from a prefix and BMC IP. +// For example, with prefix "10.96" and bmcIP "172.20.0.20", +// it produces "10.96.0.20" (uses last two octets of BMC IP). +func buildStaticClusterIP(prefix, bmcIP string) string { + // Parse last two octets from BMC IP + // This is a simple implementation; production might need more robust parsing + var a, b, c, d int + n, err := fmt.Sscanf(bmcIP, "%d.%d.%d.%d", &a, &b, &c, &d) + if err != nil || n != 4 { + return "" + } + + return fmt.Sprintf("%s.%d.%d", prefix, c, d) +} + +// ServiceDiff represents changes between desired and existing services. +type ServiceDiff struct { + Create []*corev1.Service + Update []*corev1.Service + Delete []string // service names to delete +} + +// ComputeServiceDiff calculates what changes need to be made. +func ComputeServiceDiff(desired []*corev1.Service, existing []*corev1.Service) ServiceDiff { + var diff ServiceDiff + + existingByName := make(map[string]*corev1.Service) + for _, svc := range existing { + existingByName[svc.Name] = svc + } + + desiredNames := make(map[string]bool) + for _, svc := range desired { + desiredNames[svc.Name] = true + + existingSvc, exists := existingByName[svc.Name] + if !exists { + diff.Create = append(diff.Create, svc) + continue + } + + if needsUpdate(svc, existingSvc) { + // Copy ResourceVersion for update + svc.ResourceVersion = existingSvc.ResourceVersion + // Preserve ClusterIP if not explicitly set + if svc.Spec.ClusterIP == "" { + svc.Spec.ClusterIP = existingSvc.Spec.ClusterIP + } + diff.Update = append(diff.Update, svc) + } + } + + // Find services to delete (exist but not in desired) + for name, svc := range existingByName { + if !desiredNames[name] && isManagedByController(svc) { + diff.Delete = append(diff.Delete, name) + } + } + + return diff +} + +// needsUpdate checks if a service needs to be updated. +func needsUpdate(desired, existing *corev1.Service) bool { + // Check port changes + if len(desired.Spec.Ports) != len(existing.Spec.Ports) { + return true + } + + existingPorts := make(map[string]corev1.ServicePort) + for _, p := range existing.Spec.Ports { + existingPorts[p.Name] = p + } + + for _, dp := range desired.Spec.Ports { + ep, ok := existingPorts[dp.Name] + if !ok { + return true + } + if dp.Port != ep.Port || dp.TargetPort != ep.TargetPort || dp.Protocol != ep.Protocol { + return true + } + } + + // Check label changes (except for managed-by which should always be set) + for k, v := range desired.Labels { + if existing.Labels[k] != v { + return true + } + } + + // Check annotation changes + for k, v := range desired.Annotations { + if existing.Annotations[k] != v { + return true + } + } + + return false +} + +// isManagedByController checks if a service is managed by this controller. +func isManagedByController(svc *corev1.Service) bool { + return svc.Labels[LabelManagedBy] == LabelManagedByValue +} + +// Reconciler handles the reconciliation loop. +type Reconciler struct { + matClient *matclient.Client + serviceBuilder *ServiceBuilder + k8sClient K8sServiceClient +} + +// K8sServiceClient is an interface for Kubernetes Service operations. +type K8sServiceClient interface { + List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) + Create(ctx context.Context, svc *corev1.Service) error + Update(ctx context.Context, svc *corev1.Service) error + Delete(ctx context.Context, namespace, name string) error +} + +// NewReconciler creates a new Reconciler. +func NewReconciler(matClient *matclient.Client, builder *ServiceBuilder, k8sClient K8sServiceClient) *Reconciler { + return &Reconciler{ + matClient: matClient, + serviceBuilder: builder, + k8sClient: k8sClient, + } +} + +// ReconcileResult contains the result of a reconciliation. +type ReconcileResult struct { + Created int + Updated int + Deleted int + Errors []error +} + +// Reconcile performs a single reconciliation pass. +func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { + result := ReconcileResult{} + + // Fetch current machine status + status, err := r.matClient.GetMachinesStatus(ctx) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("fetching machine status: %w", err)) + return result + } + + // Build desired services + desired := r.serviceBuilder.BuildServicesFromStatus(status) + + // List existing managed services + selector := fmt.Sprintf("%s=%s", LabelManagedBy, LabelManagedByValue) + existing, err := r.k8sClient.List(ctx, r.serviceBuilder.Namespace, selector) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("listing existing services: %w", err)) + return result + } + + // Compute diff + diff := ComputeServiceDiff(desired, existing) + + // Apply creates + for _, svc := range diff.Create { + if err := r.k8sClient.Create(ctx, svc); err != nil { + result.Errors = append(result.Errors, fmt.Errorf("creating service %s: %w", svc.Name, err)) + } else { + result.Created++ + } + } + + // Apply updates + for _, svc := range diff.Update { + if err := r.k8sClient.Update(ctx, svc); err != nil { + result.Errors = append(result.Errors, fmt.Errorf("updating service %s: %w", svc.Name, err)) + } else { + result.Updated++ + } + } + + // Apply deletes + for _, name := range diff.Delete { + if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { + result.Errors = append(result.Errors, fmt.Errorf("deleting service %s: %w", name, err)) + } else { + result.Deleted++ + } + } + + return result +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go new file mode 100644 index 0000000000..f3f8064aa2 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go @@ -0,0 +1,559 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/NVIDIA/infra-controller/dev/k8s/machine-a-tron-controller/pkg/matclient" +) + +func TestBuildServiceName(t *testing.T) { + tests := []struct { + machineType string + matID string + want string + }{ + { + machineType: MachineTypeHost, + matID: "12345678-1234-1234-1234-123456789abc", + want: "mat-bmc-host-12345678", + }, + { + machineType: MachineTypeDPU, + matID: "abcdefgh-1234-1234-1234-123456789abc", + want: "mat-bmc-dpu-abcdefgh", + }, + { + machineType: MachineTypeHost, + matID: "short", + want: "mat-bmc-host-short", + }, + } + + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + got := BuildServiceName(tt.machineType, tt.matID) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestServiceBuilder_BuildService(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + TargetSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + machine := &matclient.MachineStatus{ + MatID: "host-uuid-12345678", + MachineID: ptr("nico-machine-id"), + HardwareType: ptr("GB200"), + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + } + + svc := builder.BuildService(machine, MachineTypeHost, "") + + // Check basic metadata + assert.Equal(t, "mat-bmc-host-host-uui", svc.Name) + assert.Equal(t, "test-ns", svc.Namespace) + + // Check labels + assert.Equal(t, LabelManagedByValue, svc.Labels[LabelManagedBy]) + assert.Equal(t, "host-uuid-12345678", svc.Labels[LabelMatID]) + assert.Equal(t, "nico-machine-id", svc.Labels[LabelMachineID]) + assert.Equal(t, MachineTypeHost, svc.Labels[LabelMachineType]) + + // Check annotations + assert.Equal(t, "192.168.1.100", svc.Annotations[AnnotationBMCIP]) + assert.Equal(t, "Ready", svc.Annotations[AnnotationAPIState]) + assert.Equal(t, "On", svc.Annotations[AnnotationPowerState]) + assert.Equal(t, "GB200", svc.Annotations[AnnotationHardwareType]) + assert.Equal(t, "8443", svc.Annotations[AnnotationRedfishListenPort]) + + // Check ports + require.Len(t, svc.Spec.Ports, 1) + assert.Equal(t, PortNameRedfish, svc.Spec.Ports[0].Name) + assert.Equal(t, corev1.ProtocolTCP, svc.Spec.Ports[0].Protocol) + assert.Equal(t, int32(443), svc.Spec.Ports[0].Port) + assert.Equal(t, intstr.FromInt32(8443), svc.Spec.Ports[0].TargetPort) + + // Check selector + assert.Equal(t, builder.TargetSelector, svc.Spec.Selector) +} + +func TestServiceBuilder_BuildService_WithIPMI(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + TargetSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + machine := &matclient.MachineStatus{ + MatID: "host-uuid-12345678", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + IPMI: &matclient.EndpointStatus{ + ReachablePort: 623, + ListenPort: 16023, + }, + }, + } + + svc := builder.BuildService(machine, MachineTypeHost, "") + + // Check we have both ports + require.Len(t, svc.Spec.Ports, 2) + + // Find ports by name + var redfishPort, ipmiPort *corev1.ServicePort + for i := range svc.Spec.Ports { + switch svc.Spec.Ports[i].Name { + case PortNameRedfish: + redfishPort = &svc.Spec.Ports[i] + case PortNameIPMI: + ipmiPort = &svc.Spec.Ports[i] + } + } + + require.NotNil(t, redfishPort) + assert.Equal(t, corev1.ProtocolTCP, redfishPort.Protocol) + assert.Equal(t, int32(443), redfishPort.Port) + assert.Equal(t, intstr.FromInt32(8443), redfishPort.TargetPort) + + require.NotNil(t, ipmiPort) + assert.Equal(t, corev1.ProtocolUDP, ipmiPort.Protocol) + assert.Equal(t, int32(623), ipmiPort.Port) + assert.Equal(t, intstr.FromInt32(16023), ipmiPort.TargetPort) + + // Check IPMI annotation + assert.Equal(t, "16023", svc.Annotations[AnnotationIPMIListenPort]) +} + +func TestServiceBuilder_BuildService_DPU(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + TargetSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + dpu := &matclient.MachineStatus{ + MatID: "dpu-uuid-12345678", + MachineID: ptr("dpu-machine-id"), + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.101"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8444, + }, + }, + } + + svc := builder.BuildService(dpu, MachineTypeDPU, "parent-host-uuid") + + // Check DPU-specific labels + assert.Equal(t, MachineTypeDPU, svc.Labels[LabelMachineType]) + assert.Equal(t, "parent-host-uuid", svc.Labels[LabelParentMatID]) +} + +func TestServiceBuilder_BuildService_StaticClusterIP(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + TargetSelector: map[string]string{ + "app": "machine-a-tron", + }, + ClusterIPPrefix: "10.96", + } + + machine := &matclient.MachineStatus{ + MatID: "host-uuid-12345678", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("172.20.0.20"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + } + + svc := builder.BuildService(machine, MachineTypeHost, "") + + // Check static ClusterIP was set + assert.Equal(t, "10.96.0.20", svc.Spec.ClusterIP) +} + +func TestServiceBuilder_BuildServicesFromStatus(t *testing.T) { + builder := &ServiceBuilder{ + Namespace: "test-ns", + TargetSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + status := &matclient.MachinesStatusResponse{ + Machines: []matclient.MachineStatus{ + { + MatID: "host-1", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8443}, + }, + DPUs: []matclient.MachineStatus{ + { + MatID: "dpu-1", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8444}, + }, + }, + { + MatID: "dpu-2", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8445}, + }, + }, + }, + }, + { + MatID: "host-2", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + Redfish: matclient.EndpointStatus{ReachablePort: 443, ListenPort: 8446}, + }, + }, + }, + } + + services := builder.BuildServicesFromStatus(status) + + // Should have 4 services: 2 hosts + 2 DPUs + assert.Len(t, services, 4) + + // Verify parent links for DPUs + dpuServices := make([]*corev1.Service, 0) + for _, svc := range services { + if svc.Labels[LabelMachineType] == MachineTypeDPU { + dpuServices = append(dpuServices, svc) + } + } + + assert.Len(t, dpuServices, 2) + for _, svc := range dpuServices { + assert.Equal(t, "host-1", svc.Labels[LabelParentMatID]) + } +} + +func TestComputeServiceDiff(t *testing.T) { + tests := []struct { + name string + desired []*corev1.Service + existing []*corev1.Service + wantCreateCount int + wantUpdateCount int + wantDeleteCount int + }{ + { + name: "empty to empty", + desired: nil, + existing: nil, + wantCreateCount: 0, + wantUpdateCount: 0, + wantDeleteCount: 0, + }, + { + name: "create new service", + desired: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + existing: nil, + wantCreateCount: 1, + }, + { + name: "delete stale service", + desired: nil, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantDeleteCount: 1, + }, + { + name: "no changes needed", + desired: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantCreateCount: 0, + wantUpdateCount: 0, + wantDeleteCount: 0, + }, + { + name: "update port change", + desired: []*corev1.Service{ + func() *corev1.Service { + svc := makeTestService("svc-1", "mat-id-1") + svc.Spec.Ports[0].TargetPort = intstr.FromInt32(9999) + return svc + }(), + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantUpdateCount: 1, + }, + { + name: "update annotation change", + desired: []*corev1.Service{ + func() *corev1.Service { + svc := makeTestService("svc-1", "mat-id-1") + svc.Annotations[AnnotationAPIState] = "NotReady" + return svc + }(), + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), + }, + wantUpdateCount: 1, + }, + { + name: "mixed operations", + desired: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), // keep + makeTestService("svc-2", "mat-id-2"), // create + }, + existing: []*corev1.Service{ + makeTestService("svc-1", "mat-id-1"), // keep + makeTestService("svc-3", "mat-id-3"), // delete + }, + wantCreateCount: 1, + wantDeleteCount: 1, + }, + { + name: "skip non-managed services on delete", + desired: nil, + existing: []*corev1.Service{ + func() *corev1.Service { + svc := makeTestService("svc-1", "mat-id-1") + svc.Labels[LabelManagedBy] = "someone-else" + return svc + }(), + }, + wantDeleteCount: 0, // should not delete services we don't manage + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + diff := ComputeServiceDiff(tt.desired, tt.existing) + + assert.Len(t, diff.Create, tt.wantCreateCount) + assert.Len(t, diff.Update, tt.wantUpdateCount) + assert.Len(t, diff.Delete, tt.wantDeleteCount) + }) + } +} + +func TestReconciler_Reconcile(t *testing.T) { + ctx := context.Background() + + mockClient := &mockK8sClient{ + services: make(map[string]*corev1.Service), + } + + builder := &ServiceBuilder{ + Namespace: "test-ns", + TargetSelector: map[string]string{ + "app": "machine-a-tron", + }, + } + + matClient := &mockMatClient{ + status: &matclient.MachinesStatusResponse{ + Machines: []matclient.MachineStatus{ + { + MatID: "host-1234", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + }, + }, + }, + } + + reconciler := NewReconciler(nil, builder, mockClient) + reconciler.matClient = nil // We'll use the mock + + // First reconcile: should create service + result := reconcileWithMockMatClient(ctx, reconciler, mockClient, matClient.status) + assert.Equal(t, 1, result.Created) + assert.Equal(t, 0, result.Updated) + assert.Equal(t, 0, result.Deleted) + assert.Empty(t, result.Errors) + assert.Len(t, mockClient.services, 1) + + // Second reconcile with same state: no changes + result = reconcileWithMockMatClient(ctx, reconciler, mockClient, matClient.status) + assert.Equal(t, 0, result.Created) + assert.Equal(t, 0, result.Updated) + assert.Equal(t, 0, result.Deleted) + + // Third reconcile with empty status: should delete + result = reconcileWithMockMatClient(ctx, reconciler, mockClient, &matclient.MachinesStatusResponse{}) + assert.Equal(t, 0, result.Created) + assert.Equal(t, 0, result.Updated) + assert.Equal(t, 1, result.Deleted) + assert.Empty(t, mockClient.services) +} + +// Helper function to run reconcile with mock mat client +func reconcileWithMockMatClient(ctx context.Context, r *Reconciler, k8s *mockK8sClient, status *matclient.MachinesStatusResponse) ReconcileResult { + result := ReconcileResult{} + + desired := r.serviceBuilder.BuildServicesFromStatus(status) + + selector := LabelManagedBy + "=" + LabelManagedByValue + existing, _ := k8s.List(ctx, r.serviceBuilder.Namespace, selector) + + diff := ComputeServiceDiff(desired, existing) + + for _, svc := range diff.Create { + if err := k8s.Create(ctx, svc); err != nil { + result.Errors = append(result.Errors, err) + } else { + result.Created++ + } + } + + for _, svc := range diff.Update { + if err := k8s.Update(ctx, svc); err != nil { + result.Errors = append(result.Errors, err) + } else { + result.Updated++ + } + } + + for _, name := range diff.Delete { + if err := k8s.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { + result.Errors = append(result.Errors, err) + } else { + result.Deleted++ + } + } + + return result +} + +// makeTestService creates a test service with standard labels. +func makeTestService(name, matID string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-ns", + Labels: map[string]string{ + LabelManagedBy: LabelManagedByValue, + LabelMatID: matID, + LabelMachineType: MachineTypeHost, + }, + Annotations: map[string]string{ + AnnotationAPIState: "Ready", + AnnotationPowerState: "On", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: PortNameRedfish, + Protocol: corev1.ProtocolTCP, + Port: 443, + TargetPort: intstr.FromInt32(8443), + }, + }, + }, + } +} + +// Mock implementations + +type mockK8sClient struct { + services map[string]*corev1.Service +} + +func (m *mockK8sClient) List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) { + result := make([]*corev1.Service, 0) + for _, svc := range m.services { + if svc.Labels[LabelManagedBy] == LabelManagedByValue { + result = append(result, svc) + } + } + return result, nil +} + +func (m *mockK8sClient) Create(ctx context.Context, svc *corev1.Service) error { + m.services[svc.Name] = svc.DeepCopy() + return nil +} + +func (m *mockK8sClient) Update(ctx context.Context, svc *corev1.Service) error { + m.services[svc.Name] = svc.DeepCopy() + return nil +} + +func (m *mockK8sClient) Delete(ctx context.Context, namespace, name string) error { + delete(m.services, name) + return nil +} + +type mockMatClient struct { + status *matclient.MachinesStatusResponse + err error +} + +func (m *mockMatClient) GetMachinesStatus(ctx context.Context) (*matclient.MachinesStatusResponse, error) { + return m.status, m.err +} + +func ptr[T any](v T) *T { + return &v +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/k8s_client.go b/dev/k8s/machine-a-tron-controller/pkg/controller/k8s_client.go new file mode 100644 index 0000000000..ec54dde3f5 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/k8s_client.go @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// RealK8sServiceClient implements K8sServiceClient using the real Kubernetes API. +type RealK8sServiceClient struct { + clientset kubernetes.Interface +} + +// NewRealK8sServiceClient creates a new RealK8sServiceClient. +func NewRealK8sServiceClient(clientset kubernetes.Interface) *RealK8sServiceClient { + return &RealK8sServiceClient{clientset: clientset} +} + +// List returns all Services matching the label selector. +func (c *RealK8sServiceClient) List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) { + list, err := c.clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return nil, err + } + + result := make([]*corev1.Service, len(list.Items)) + for i := range list.Items { + result[i] = &list.Items[i] + } + return result, nil +} + +// Create creates a new Service. +func (c *RealK8sServiceClient) Create(ctx context.Context, svc *corev1.Service) error { + _, err := c.clientset.CoreV1().Services(svc.Namespace).Create(ctx, svc, metav1.CreateOptions{}) + return err +} + +// Update updates an existing Service. +func (c *RealK8sServiceClient) Update(ctx context.Context, svc *corev1.Service) error { + _, err := c.clientset.CoreV1().Services(svc.Namespace).Update(ctx, svc, metav1.UpdateOptions{}) + return err +} + +// Delete deletes a Service by name. +func (c *RealK8sServiceClient) Delete(ctx context.Context, namespace, name string) error { + return c.clientset.CoreV1().Services(namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go new file mode 100644 index 0000000000..31c7be4d03 --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package matclient provides an HTTP client for the machine-a-tron API. +package matclient + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/rs/zerolog" +) + +// Client is an HTTP client for the machine-a-tron API. +type Client struct { + baseURL string + httpClient *http.Client + logger zerolog.Logger +} + +// Option configures a Client. +type Option func(*Client) + +// WithHTTPClient sets a custom HTTP client. +func WithHTTPClient(c *http.Client) Option { + return func(client *Client) { + client.httpClient = c + } +} + +// WithLogger sets the logger. +func WithLogger(logger zerolog.Logger) Option { + return func(client *Client) { + client.logger = logger + } +} + +// NewClient creates a new machine-a-tron API client. +func NewClient(baseURL string, opts ...Option) (*Client, error) { + if _, err := url.Parse(baseURL); err != nil { + return nil, fmt.Errorf("invalid base URL: %w", err) + } + + c := &Client{ + baseURL: baseURL, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + logger: zerolog.Nop(), + } + + for _, opt := range opts { + opt(c) + } + + return c, nil +} + +// GetMachinesStatus fetches the current machine status from machine-a-tron. +func (c *Client) GetMachinesStatus(ctx context.Context) (*MachinesStatusResponse, error) { + reqURL := c.baseURL + "/machines/status" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Accept", "application/json") + + c.logger.Debug().Str("url", reqURL).Msg("fetching machine status") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("executing request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + var result MachinesStatusResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decoding response: %w", err) + } + + c.logger.Debug().Int("machine_count", len(result.Machines)).Msg("fetched machine status") + + return &result, nil +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/client_test.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/client_test.go new file mode 100644 index 0000000000..ba901d001a --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/client_test.go @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package matclient + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClient_GetMachinesStatus(t *testing.T) { + tests := []struct { + name string + serverResponse *MachinesStatusResponse + serverStatus int + wantErr bool + errContains string + }{ + { + name: "empty machines list", + serverResponse: &MachinesStatusResponse{ + Machines: []MachineStatus{}, + }, + serverStatus: http.StatusOK, + }, + { + name: "single host with DPU", + serverResponse: &MachinesStatusResponse{ + Machines: []MachineStatus{ + { + MatID: "host-uuid-1", + MachineID: ptr("machine-123"), + APIState: "Ready", + PowerState: "On", + BMC: BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, + }, + }, + DPUs: []MachineStatus{ + { + MatID: "dpu-uuid-1", + MachineID: ptr("dpu-456"), + APIState: "Ready", + PowerState: "On", + BMC: BMCStatus{ + IP: ptr("192.168.1.101"), + Redfish: EndpointStatus{ + ReachablePort: 443, + ListenPort: 8444, + }, + }, + }, + }, + }, + }, + }, + serverStatus: http.StatusOK, + }, + { + name: "server error", + serverStatus: http.StatusInternalServerError, + wantErr: true, + errContains: "unexpected status 500", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/machines/status", r.URL.Path) + assert.Equal(t, http.MethodGet, r.Method) + assert.Equal(t, "application/json", r.Header.Get("Accept")) + + w.WriteHeader(tt.serverStatus) + if tt.serverResponse != nil { + err := json.NewEncoder(w).Encode(tt.serverResponse) + require.NoError(t, err) + } + })) + defer server.Close() + + client, err := NewClient(server.URL, WithLogger(zerolog.Nop())) + require.NoError(t, err) + + resp, err := client.GetMachinesStatus(context.Background()) + + if tt.wantErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + assert.Equal(t, tt.serverResponse, resp) + }) + } +} + +func TestNewClient_InvalidURL(t *testing.T) { + _, err := NewClient("://invalid") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid base URL") +} + +func ptr[T any](v T) *T { + return &v +} diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/types.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/types.go new file mode 100644 index 0000000000..be4ef5951c --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/types.go @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package matclient provides an HTTP client for the machine-a-tron API. +package matclient + +// MachinesStatusResponse is the top-level response from GET /machines/status. +type MachinesStatusResponse struct { + Machines []MachineStatus `json:"machines"` +} + +// MachineStatus represents the status of a single machine (host or DPU). +type MachineStatus struct { + // MatID is the machine-a-tron internal identifier (UUID). + MatID string `json:"mat_id"` + // MachineID is the observed NICo machine ID, if known. + MachineID *string `json:"machine_id,omitempty"` + // HardwareType is the hardware model (e.g., "GB200", "DGX"). + HardwareType *string `json:"hardware_type,omitempty"` + // MatState is the machine-a-tron FSM state. + MatState *string `json:"mat_state,omitempty"` + // APIState is the observed NICo API state. + APIState string `json:"api_state"` + // PowerState is the current power state ("On", "Off", etc.). + PowerState string `json:"power_state"` + // MachineIP is the machine's management IP, if known. + MachineIP *string `json:"machine_ip,omitempty"` + // BMC contains BMC endpoint information. + BMC BMCStatus `json:"bmc"` + // DPUs contains nested DPU statuses for host machines. + DPUs []MachineStatus `json:"dpus,omitempty"` +} + +// BMCStatus contains the BMC endpoint configuration. +type BMCStatus struct { + // IP is the BMC IP address, if assigned. + IP *string `json:"ip,omitempty"` + // Redfish contains the Redfish endpoint configuration. + Redfish EndpointStatus `json:"redfish"` + // IPMI contains the IPMI endpoint configuration, if enabled. + IPMI *EndpointStatus `json:"ipmi,omitempty"` +} + +// EndpointStatus describes port mapping for an endpoint. +type EndpointStatus struct { + // ReachablePort is the external port clients should connect to (e.g., 443 for Redfish). + ReachablePort uint16 `json:"reachable_port"` + // ListenPort is the internal port the mock BMC listens on. + ListenPort uint16 `json:"listen_port"` +} From 6d1ccdc49081b06783e9c69127197fb91abbba88 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 22 Jul 2026 12:45:33 -0600 Subject: [PATCH 02/17] feat(helm): machine-a-tron controller subchart --- helm/charts/nico-machine-a-tron/Chart.yaml | 5 ++ .../charts/mat-k8s-controller/Chart.yaml | 6 ++ .../mat-k8s-controller/templates/_helpers.tpl | 77 +++++++++++++++++++ .../templates/deployment.yaml | 47 +++++++++++ .../mat-k8s-controller/templates/rbac.yaml | 29 +++++++ .../templates/serviceaccount.yaml | 13 ++++ .../charts/mat-k8s-controller/values.yaml | 57 ++++++++++++++ .../templates/bmc-services.yaml | 2 +- helm/charts/nico-machine-a-tron/values.yaml | 59 ++++++++++++++ 9 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/Chart.yaml create mode 100644 helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl create mode 100644 helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml create mode 100644 helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/rbac.yaml create mode 100644 helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/serviceaccount.yaml create mode 100644 helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml diff --git a/helm/charts/nico-machine-a-tron/Chart.yaml b/helm/charts/nico-machine-a-tron/Chart.yaml index 5f6ceac8d6..2d64884f9a 100644 --- a/helm/charts/nico-machine-a-tron/Chart.yaml +++ b/helm/charts/nico-machine-a-tron/Chart.yaml @@ -10,3 +10,8 @@ keywords: - mock - testing - development +dependencies: + - name: mat-k8s-controller + version: "0.1.0" + condition: mat-k8s-controller.enabled + repository: "" diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/Chart.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/Chart.yaml new file mode 100644 index 0000000000..2fab72d0c5 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: mat-k8s-controller +description: Kubernetes controller for machine-a-tron BMC service reconciliation +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl new file mode 100644 index 0000000000..b9a380c34d --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl @@ -0,0 +1,77 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "mat-k8s-controller.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "mat-k8s-controller.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mat-k8s-controller.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "mat-k8s-controller.labels" -}} +helm.sh/chart: {{ include "mat-k8s-controller.chart" . }} +{{ include "mat-k8s-controller.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "mat-k8s-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "mat-k8s-controller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "mat-k8s-controller.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "mat-k8s-controller.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Namespace - inherited from parent release +*/}} +{{- define "mat-k8s-controller.namespace" -}} +{{- .Release.Namespace }} +{{- end }} + +{{/* +Machine-a-tron URL - derived from parent chart's bmc-mock service +*/}} +{{- define "mat-k8s-controller.matUrl" -}} +{{- printf "https://%s-bmc-mock.%s.svc.cluster.local:1266" .Release.Name .Release.Namespace }} +{{- end }} + +{{/* +Target selector - matches parent chart's machine-a-tron pods +*/}} +{{- define "mat-k8s-controller.targetSelector" -}} +app.kubernetes.io/name=nico-machine-a-tron +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml new file mode 100644 index 0000000000..e3dd5085f6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml @@ -0,0 +1,47 @@ +{{- if .Values.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mat-k8s-controller.fullname" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "mat-k8s-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "mat-k8s-controller.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "mat-k8s-controller.serviceAccountName" . }} + containers: + - name: controller + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --mat-url={{ include "mat-k8s-controller.matUrl" . }} + - --namespace={{ include "mat-k8s-controller.namespace" . }} + - --sync-interval={{ .Values.config.syncInterval }} + - --insecure-skip-verify={{ .Values.config.insecureSkipVerify }} + - --log-level={{ .Values.config.logLevel }} + - --target-selector={{ include "mat-k8s-controller.targetSelector" . }} + {{- if .Values.config.clusterIpPrefix }} + - --cluster-ip-prefix={{ .Values.config.clusterIpPrefix }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + securityContext: + fsGroup: 65534 + runAsNonRoot: true +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/rbac.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/rbac.yaml new file mode 100644 index 0000000000..9793d493f1 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/rbac.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.enabled .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "mat-k8s-controller.fullname" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "mat-k8s-controller.fullname" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "mat-k8s-controller.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "mat-k8s-controller.serviceAccountName" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/serviceaccount.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/serviceaccount.yaml new file mode 100644 index 0000000000..9433af86a6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mat-k8s-controller.serviceAccountName" . }} + namespace: {{ include "mat-k8s-controller.namespace" . }} + labels: + {{- include "mat-k8s-controller.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml new file mode 100644 index 0000000000..4d03991e4e --- /dev/null +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -0,0 +1,57 @@ +## ============================================================================= +## mat-k8s-controller - Kubernetes controller for machine-a-tron BMC services +## ============================================================================= +## +## This controller dynamically creates/updates/deletes Kubernetes Services +## based on machine-a-tron's /machines/status API. +## +## Inherited from parent chart: +## - matUrl: derived from release name + bmc-mock service +## - namespace: inherited from release namespace +## - targetSelector: matches parent's machine-a-tron pods +## +## ============================================================================= + +## Enable/disable the controller +enabled: false + +image: + repository: mat-k8s-controller + tag: latest + pullPolicy: IfNotPresent + +## Resource limits +resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi + +## Controller configuration +config: + ## Reconciliation interval + syncInterval: "30s" + + ## Skip TLS certificate verification (for self-signed certs) + insecureSkipVerify: true + + ## Log level (debug, info, warn, error) + logLevel: "info" + + ## Optional: Prefix for static ClusterIP assignment + ## When set, controller assigns ClusterIPs using BMC IP's last two octets + ## Example: prefix "10.96" + BMC IP "172.20.0.20" = ClusterIP "10.96.0.20" + ## Leave empty for Kubernetes auto-assigned ClusterIPs + clusterIpPrefix: "" + +## Service account configuration +serviceAccount: + create: true + name: "" + annotations: {} + +## RBAC configuration +rbac: + create: true diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml index 54f2353f2a..369359d347 100644 --- a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml +++ b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml @@ -1,4 +1,4 @@ -{{- if .Values.bmcServices.enabled }} +{{- if and .Values.bmcServices.enabled (not (index .Values "mat-k8s-controller" "enabled")) }} {{- $root := . }} {{- $namespace := include "nico-machine-a-tron.namespace" . }} {{- $baseName := include "nico-machine-a-tron.name" . }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index 7fe7603473..9ed30d95fc 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -354,3 +354,62 @@ configFiles: # oob_dhcp_relay_address = "10.100.0.1" # admin_dhcp_relay_address = "192.168.176.1" matConfigs: {} + +## ============================================================================= +## Machine-a-tron Kubernetes Controller (Subchart) +## ============================================================================= +## When enabled, the controller dynamically creates BMC services based on +## machine-a-tron's /machines/status API instead of static Helm-generated services. +## +## Benefits: +## - Services reflect actual machine state (annotations show power/API state) +## - Automatic cleanup when machines are removed +## - No need to pre-calculate BMC counts and CIDRs +## +## Requirements: +## - bmcServices.enabled must also be true (for ServiceCIDR allocation) +## - Controller image must be available in the cluster +## +## ============================================================================= +## Machine-a-tron Kubernetes Controller (Subchart) +## ============================================================================= +## When enabled, the controller dynamically creates BMC services based on +## machine-a-tron's /machines/status API instead of static Helm-generated services. +## +## Configuration is inherited from parent chart: +## - matUrl: derived from release name + bmc-mock service +## - namespace: inherited from release namespace +## - clusterIpPrefix: extracted from pods.*.cidr +## +## Requirements: +## - bmcServices.enabled must also be true (for ServiceCIDR allocation) +## - Controller image must be available in the cluster +## +## ============================================================================= +## Machine-a-tron Kubernetes Controller (Subchart) +## ============================================================================= +## When enabled, the controller dynamically creates BMC services based on +## machine-a-tron's /machines/status API instead of static Helm-generated services. +## +## Inherited automatically: +## - matUrl: https://-bmc-mock..svc.cluster.local:1266 +## - namespace: release namespace +## - targetSelector: app.kubernetes.io/name=nico-machine-a-tron +## +## Requirements: +## - Controller image must be available in the cluster +## +mat-k8s-controller: + enabled: false + + image: + repository: mat-k8s-controller + tag: latest + pullPolicy: IfNotPresent + + config: + syncInterval: "30s" + insecureSkipVerify: true + logLevel: "info" + ## Optional: set for predictable ClusterIPs (e.g., "10.96") + clusterIpPrefix: "" From f1fea7e1de7cac7d74ac1d7c1a31825efe5f92fa Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 22 Jul 2026 12:46:12 -0600 Subject: [PATCH 03/17] fix(mat-ctrl): machine-a-tron controller update --- dev/k8s/machine-a-tron-controller/README.md | 211 +++++------------- .../cmd/mat-k8s-controller/main.go | 28 ++- .../pkg/matclient/client.go | 14 ++ 3 files changed, 93 insertions(+), 160 deletions(-) diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md index 9eafd0b614..05798211d2 100644 --- a/dev/k8s/machine-a-tron-controller/README.md +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -1,194 +1,95 @@ # Machine-a-tron Kubernetes Controller -A Kubernetes (K8s) controller that reconciles K8s services for Machine-a-tron (MAT) mock BMC endpoints. +A Kubernetes controller that reconciles Services for machine-a-tron mock BMC endpoints. ## Overview -When Machine-a-tron runs in Kubernetes, mock BMCs need stable network exposure so that +When machine-a-tron runs in Kubernetes, mock BMCs need stable network exposure so that NICo components can reach Redfish and IPMI/SOL endpoints. -This controller uses MAT machines status and reconciles one K8s service per mock BMC. +This controller polls machine-a-tron status and reconciles one Service per mock BMC. Machine-a-tron itself remains Kubernetes-agnostic. This controller bridges the gap. ## Features - Polls machine-a-tron `GET /machines/status` API -- Creates one K8s service per mock BMC (hosts and DPUs) -- Exposes Redfish TCP (port 443 -> internal listen port) -- Exposes IPMI UDP when enabled (port 623 -> internal listen port) -- Labels and annotates K8s services with machine/BMC identity -- Updates K8s services when machine status changes -- Deletes stale K8s services when machines disappear -- Supports static ClusterIP assignment for predictable addressing +- Creates one Service per mock BMC (hosts and DPUs) +- Exposes Redfish TCP (port 443 → internal listen port) +- Exposes IPMI UDP when enabled (port 623 → internal listen port) +- Labels and annotates Services with machine/BMC identity +- Updates Services when machine status changes +- Deletes stale Services when machines disappear -## Installation - -### From Source +## Build ```bash +# Build binary go build -o mat-k8s-controller ./cmd/mat-k8s-controller -``` -### Docker +# Build container +docker build -t mat-k8s-controller:latest . -```bash -docker build -t mat-k8s-controller . +# Load to kind cluster +kind load docker-image mat-k8s-controller:latest --name ``` -### Kubernetes - -Deploy alongside Machine-a-tron using the Helm chart (coming soon). - ## Configuration -The controller accepts configuration via flags or environment variables: - | Flag | Env Var | Default | Description | |------|---------|---------|-------------| -| `--mat-url` | `MAT_URL` | `http://machine-a-tron:8080` | Machine-a-tron base URL | -| `--namespace` | `NAMESPACE` | `nico-mat` | Kubernetes namespace for services | -| `--sync-interval` | `SYNC_INTERVAL` | `30s` | Interval between reconciliation passes | -| `--kubeconfig` | `KUBECONFIG` | (in-cluster) | Path to kubeconfig file | -| `--target-selector` | `TARGET_SELECTOR` | `app=machine-a-tron` | Pod selector for services | -| `--cluster-ip-prefix` | `CLUSTER_IP_PREFIX` | (none) | Prefix for static ClusterIP | -| `--log-level` | `LOG_LEVEL` | `info` | Log level (debug, info, warn, error) | +| `--mat-url` | `MAT_URL` | `https://nico-machine-a-tron-bmc-mock:1266` | Machine-a-tron base URL | +| `--namespace` | `NAMESPACE` | `nico-system` | Namespace for Services | +| `--sync-interval` | `SYNC_INTERVAL` | `30s` | Reconciliation interval | +| `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector | +| `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `true` | Skip TLS verification | +| `--log-level` | `LOG_LEVEL` | `info` | Log level | +| `--cluster-ip-prefix` | `CLUSTER_IP_PREFIX` | (none) | Static ClusterIP prefix | -### Static ClusterIP Assignment +## Helm Deployment -When `--cluster-ip-prefix` is set and a BMC has an assigned IP, the controller will -assign a predictable ClusterIP using the last two octets of the BMC IP. For example: +The controller is a subchart of `nico-machine-a-tron`. When enabled, it dynamically +manages BMC Services instead of static Helm-generated ones. -- Prefix: `10.96` -- BMC IP: `172.20.0.20` -- ClusterIP: `10.96.0.20` +```bash +helm upgrade --install nico-machine-a-tron ./helm/charts/nico-machine-a-tron \ + --namespace nico-system \ + --create-namespace \ + --set machineATron.enableIpmiSimulation=true \ + --set mat-k8s-controller.enabled=true \ + --set mat-k8s-controller.image.pullPolicy=Never +``` -This enables predictable addressing for testing and development. +Configuration is inherited from the parent chart: +- `matUrl` → derived from release name +- `namespace` → release namespace +- `targetSelector` → matches machine-a-tron pods ## Service Structure -Each created Service has: - -### Labels - -| Label | Description | -|-------|-------------| -| `app.kubernetes.io/managed-by` | Always `mat-k8s-controller` | -| `machine-a-tron.nvidia.com/mat-id` | Machine-a-tron internal UUID | -| `machine-a-tron.nvidia.com/machine-id` | NICo machine ID (if known) | -| `machine-a-tron.nvidia.com/machine-type` | `host` or `dpu` | -| `machine-a-tron.nvidia.com/parent-mat-id` | Parent host mat-id (DPUs only) | - -### Annotations - -| Annotation | Description | -|------------|-------------| -| `machine-a-tron.nvidia.com/bmc-ip` | BMC IP address | -| `machine-a-tron.nvidia.com/api-state` | Machine API state | -| `machine-a-tron.nvidia.com/power-state` | Machine power state | -| `machine-a-tron.nvidia.com/hardware-type` | Hardware type (e.g., GB200) | -| `machine-a-tron.nvidia.com/redfish-listen-port` | Internal Redfish listen port | -| `machine-a-tron.nvidia.com/ipmi-listen-port` | Internal IPMI listen port (if enabled) | - -### Ports - -| Name | Protocol | Port | Description | -|------|----------|------|-------------| -| `redfish` | TCP | 443 | Redfish API | -| `ipmi` | UDP | 623 | IPMI (if enabled) | - -## Example - -Given this machine-a-tron status: - -```json -{ - "machines": [ - { - "mat_id": "abc12345-...", - "machine_id": "machine-001", - "api_state": "Ready", - "power_state": "On", - "bmc": { - "ip": "172.20.0.20", - "redfish": { - "reachable_port": 443, - "listen_port": 8443 - } - }, - "dpus": [ - { - "mat_id": "def67890-...", - "bmc": { - "ip": "172.20.0.21", - "redfish": { - "reachable_port": 443, - "listen_port": 8444 - } - } - } - ] - } - ] -} -``` +Each Service has: -The controller creates two Services: - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: mat-bmc-host-abc12345 - labels: - app.kubernetes.io/managed-by: mat-k8s-controller - machine-a-tron.nvidia.com/mat-id: abc12345-... - machine-a-tron.nvidia.com/machine-type: host -spec: - type: ClusterIP - selector: - app: machine-a-tron - ports: - - name: redfish - protocol: TCP - port: 443 - targetPort: 8443 ---- -apiVersion: v1 -kind: Service -metadata: - name: mat-bmc-dpu-def67890 - labels: - app.kubernetes.io/managed-by: mat-k8s-controller - machine-a-tron.nvidia.com/mat-id: def67890-... - machine-a-tron.nvidia.com/machine-type: dpu - machine-a-tron.nvidia.com/parent-mat-id: abc12345-... -spec: - type: ClusterIP - selector: - app: machine-a-tron - ports: - - name: redfish - protocol: TCP - port: 443 - targetPort: 8444 -``` - -## Development +**Labels:** +- `app.kubernetes.io/managed-by: mat-k8s-controller` +- `machine-a-tron.nvidia.com/mat-id` +- `machine-a-tron.nvidia.com/machine-type` (host/dpu) -### Running Tests +**Annotations:** +- `machine-a-tron.nvidia.com/bmc-ip` +- `machine-a-tron.nvidia.com/api-state` +- `machine-a-tron.nvidia.com/power-state` -```bash -go test ./... -``` +**Ports:** +- `redfish` TCP 443 → targetPort (always) +- `ipmi` UDP 623 → targetPort (when IPMI enabled) -### Running Locally +## Development ```bash -# With kubeconfig -./mat-k8s-controller --kubeconfig ~/.kube/config --mat-url http://localhost:8080 +# Run tests +go test ./... -# In-cluster (when deployed as a pod) -./mat-k8s-controller +# Run locally +./mat-k8s-controller --kubeconfig ~/.kube/config --mat-url https://localhost:1266 ``` ## Architecture @@ -213,5 +114,5 @@ flowchart LR ## Related -- [GitHub Issue #3380](https://github.com/NVIDIA/infra-controller/issues/3380) - Original feature request -- [GitHub Issue #3379](https://github.com/NVIDIA/infra-controller/issues/3379) - Machine-a-tron status API (prerequisite) +- [GitHub Issue #3380](https://github.com/NVIDIA/infra-controller/issues/3380) - Feature request +- [GitHub Issue #3379](https://github.com/NVIDIA/infra-controller/issues/3379) - Status API diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index e908dda00c..475f65e859 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -14,6 +14,7 @@ import ( "flag" "os" "os/signal" + "strconv" "syscall" "time" @@ -28,18 +29,20 @@ import ( func main() { // Flags - matURL := flag.String("mat-url", envOrDefault("MAT_URL", "http://machine-a-tron:8080"), + matURL := flag.String("mat-url", envOrDefault("MAT_URL", "https://nico-machine-a-tron-bmc-mock:1266"), "Machine-a-tron base URL") - namespace := flag.String("namespace", envOrDefault("NAMESPACE", "default"), + namespace := flag.String("namespace", envOrDefault("NAMESPACE", "nico-system"), "Kubernetes namespace for Services") - syncInterval := flag.Duration("sync-interval", parseDurationOrDefault("SYNC_INTERVAL", 10*time.Second), + syncInterval := flag.Duration("sync-interval", parseDurationOrDefault("SYNC_INTERVAL", 30*time.Second), "Interval between reconciliation passes") kubeconfig := flag.String("kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig (uses in-cluster config if empty)") - targetSelector := flag.String("target-selector", envOrDefault("TARGET_SELECTOR", "app=machine-a-tron"), + targetSelector := flag.String("target-selector", envOrDefault("TARGET_SELECTOR", "app.kubernetes.io/name=nico-machine-a-tron"), "Pod selector for Services (comma-separated key=value pairs)") clusterIPPrefix := flag.String("cluster-ip-prefix", os.Getenv("CLUSTER_IP_PREFIX"), "Prefix for static ClusterIP assignment (e.g., 10.96)") + insecureSkipVerify := flag.Bool("insecure-skip-verify", envBoolOrDefault("INSECURE_SKIP_VERIFY", true), + "Skip TLS certificate verification (for self-signed certs)") logLevel := flag.String("log-level", envOrDefault("LOG_LEVEL", "info"), "Log level (debug, info, warn, error)") @@ -63,10 +66,16 @@ func main() { Dur("sync_interval", *syncInterval). Str("target_selector", *targetSelector). Str("cluster_ip_prefix", *clusterIPPrefix). + Bool("insecure_skip_verify", *insecureSkipVerify). Msg("starting controller") // Create machine-a-tron client - matClient, err := matclient.NewClient(*matURL, matclient.WithLogger(logger)) + clientOpts := []matclient.Option{matclient.WithLogger(logger)} + if *insecureSkipVerify { + clientOpts = append(clientOpts, matclient.WithInsecureSkipVerify()) + } + + matClient, err := matclient.NewClient(*matURL, clientOpts...) if err != nil { logger.Fatal().Err(err).Msg("failed to create machine-a-tron client") } @@ -166,6 +175,15 @@ func envOrDefault(key, defaultValue string) string { return defaultValue } +func envBoolOrDefault(key string, defaultValue bool) bool { + if v := os.Getenv(key); v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + } + return defaultValue +} + func parseDurationOrDefault(envKey string, defaultValue time.Duration) time.Duration { if v := os.Getenv(envKey); v != "" { if d, err := time.ParseDuration(v); err == nil { diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go index 31c7be4d03..b25ec6aa63 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go @@ -6,6 +6,7 @@ package matclient import ( "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -40,6 +41,19 @@ func WithLogger(logger zerolog.Logger) Option { } } +// WithInsecureSkipVerify disables TLS certificate verification. +// Use only for development with self-signed certificates. +func WithInsecureSkipVerify() Option { + return func(client *Client) { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // Intentional for dev/test with self-signed certs + }, + } + client.httpClient.Transport = transport + } +} + // NewClient creates a new machine-a-tron API client. func NewClient(baseURL string, opts ...Option) (*Client, error) { if _, err := url.Parse(baseURL); err != nil { From 18c368174ee69b3286476864bc370dfdeb82a2b9 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 22 Jul 2026 13:13:39 -0600 Subject: [PATCH 04/17] fix(helm): machine-a-tron controller chart --- .../mat-k8s-controller/templates/_helpers.tpl | 10 +++++-- .../charts/mat-k8s-controller/values.yaml | 29 ++++++++++--------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl index b9a380c34d..68fd9e9e65 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl @@ -9,10 +9,10 @@ Expand the name of the chart. Create a default fully qualified app name. */}} {{- define "mat-k8s-controller.fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride }} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} {{- end }} {{- end }} @@ -63,15 +63,19 @@ Namespace - inherited from parent release {{- end }} {{/* -Machine-a-tron URL - derived from parent chart's bmc-mock service +Machine-a-tron URL - configurable, defaults to release-based naming */}} {{- define "mat-k8s-controller.matUrl" -}} +{{- if .Values.config.matUrl }} +{{- .Values.config.matUrl }} +{{- else }} {{- printf "https://%s-bmc-mock.%s.svc.cluster.local:1266" .Release.Name .Release.Namespace }} {{- end }} +{{- end }} {{/* Target selector - matches parent chart's machine-a-tron pods */}} {{- define "mat-k8s-controller.targetSelector" -}} -app.kubernetes.io/name=nico-machine-a-tron +{{- default "app.kubernetes.io/name=nico-machine-a-tron" .Values.config.targetSelector }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml index 4d03991e4e..5ce419777f 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -1,16 +1,7 @@ -## ============================================================================= ## mat-k8s-controller - Kubernetes controller for machine-a-tron BMC services -## ============================================================================= ## ## This controller dynamically creates/updates/deletes Kubernetes Services ## based on machine-a-tron's /machines/status API. -## -## Inherited from parent chart: -## - matUrl: derived from release name + bmc-mock service -## - namespace: inherited from release namespace -## - targetSelector: matches parent's machine-a-tron pods -## -## ============================================================================= ## Enable/disable the controller enabled: false @@ -31,18 +22,24 @@ resources: ## Controller configuration config: + ## Machine-a-tron URL (optional) + ## Default: https://-bmc-mock..svc.cluster.local:1266 + matUrl: "" + + ## Target selector for Services (optional) + ## Default: app.kubernetes.io/name=nico-machine-a-tron + targetSelector: "" + ## Reconciliation interval syncInterval: "30s" - + ## Skip TLS certificate verification (for self-signed certs) insecureSkipVerify: true - + ## Log level (debug, info, warn, error) logLevel: "info" - + ## Optional: Prefix for static ClusterIP assignment - ## When set, controller assigns ClusterIPs using BMC IP's last two octets - ## Example: prefix "10.96" + BMC IP "172.20.0.20" = ClusterIP "10.96.0.20" ## Leave empty for Kubernetes auto-assigned ClusterIPs clusterIpPrefix: "" @@ -55,3 +52,7 @@ serviceAccount: ## RBAC configuration rbac: create: true + +## Override names (useful when release name differs from chart name) +nameOverride: "" +fullnameOverride: "" From 4ebbfd9b2f084eb3b56d68f0ebe231bb31c18c82 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 22 Jul 2026 19:23:57 -0600 Subject: [PATCH 05/17] fix: machine-a-tron controller clusterip logic --- dev/k8s/machine-a-tron-controller/README.md | 95 ++++++++++++++++++- .../cmd/mat-k8s-controller/main.go | 8 +- .../pkg/controller/controller.go | 29 +----- .../pkg/controller/controller_test.go | 9 +- .../templates/deployment.yaml | 3 - .../charts/mat-k8s-controller/values.yaml | 8 -- 6 files changed, 103 insertions(+), 49 deletions(-) diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md index 05798211d2..53f2d41079 100644 --- a/dev/k8s/machine-a-tron-controller/README.md +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -16,6 +16,7 @@ Machine-a-tron itself remains Kubernetes-agnostic. This controller bridges the g - Creates one Service per mock BMC (hosts and DPUs) - Exposes Redfish TCP (port 443 → internal listen port) - Exposes IPMI UDP when enabled (port 623 → internal listen port) +- Uses BMC IP directly as Service ClusterIP - Labels and annotates Services with machine/BMC identity - Updates Services when machine status changes - Deletes stale Services when machines disappear @@ -43,7 +44,47 @@ kind load docker-image mat-k8s-controller:latest --name | `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector | | `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `true` | Skip TLS verification | | `--log-level` | `LOG_LEVEL` | `info` | Log level | -| `--cluster-ip-prefix` | `CLUSTER_IP_PREFIX` | (none) | Static ClusterIP prefix | + +## ClusterIP Assignment + +The controller uses BMC IP directly as Service ClusterIP. This requires machine-a-tron's +`oobDhcpRelayAddress` to be within the Kubernetes ServiceCIDR range. + +### Setup + +1. Configure machine-a-tron with `oobDhcpRelayAddress` from K8s ServiceCIDR: + ```yaml + pods: + pod-0: + machines: + compute: + oobDhcpRelayAddress: "10.100.0.1" # Must be in K8s ServiceCIDR + ``` + +2. Add the network to NICo configuration: + ```toml + [networks.MAT-BMC-SERVICES] + type = "underlay" + prefix = "10.100.0.0/20" + ``` + +3. Result: BMC IP `10.100.0.5` → Service ClusterIP `10.100.0.5` + +### CIDR Reservation (Recommended) + +To avoid conflicts with other Services, reserve a CIDR range for machine-a-tron. +Kubernetes 1.29+ supports multiple ServiceCIDRs with the `MultiCIDRServiceAllocator` +feature gate: + +```yaml +apiVersion: networking.k8s.io/v1beta1 +kind: ServiceCIDR +metadata: + name: machine-a-tron +spec: + cidrs: + - 10.100.0.0/16 +``` ## Helm Deployment @@ -54,7 +95,6 @@ manages BMC Services instead of static Helm-generated ones. helm upgrade --install nico-machine-a-tron ./helm/charts/nico-machine-a-tron \ --namespace nico-system \ --create-namespace \ - --set machineATron.enableIpmiSimulation=true \ --set mat-k8s-controller.enabled=true \ --set mat-k8s-controller.image.pullPolicy=Never ``` @@ -92,6 +132,57 @@ go test ./... ./mat-k8s-controller --kubeconfig ~/.kube/config --mat-url https://localhost:1266 ``` +## Troubleshooting + +### ClusterIP already allocated + +**Error:** +``` +creating service mat-bmc-host-xxxxx: Service "mat-bmc-host-xxxxx" is invalid: +spec.clusterIP: Invalid value: "10.100.0.5": provided IP is already allocated +``` + +**Cause:** Another Service is using the same ClusterIP. This happens when the +BMC IP range overlaps with Kubernetes' auto-allocated ClusterIPs. + +**Solutions:** + +1. **Reserve a ServiceCIDR** (Kubernetes 1.29+, recommended for production): + ```yaml + apiVersion: networking.k8s.io/v1beta1 + kind: ServiceCIDR + metadata: + name: machine-a-tron + spec: + cidrs: + - 10.100.0.0/16 + ``` + +2. **Use a different CIDR range** in machine-a-tron `oobDhcpRelayAddress` that + doesn't overlap with existing Services. + +3. **Delete the conflicting Service** if it's no longer needed: + ```bash + kubectl get svc -A -o wide | grep 10.100.0.5 + kubectl delete svc -n + ``` + +**Impact:** The affected machine's BMC Service is not created. In NICo site-explorer, +the machine appears unhealthy until the conflict is resolved. + +### BMC IP outside ServiceCIDR + +**Error:** +``` +creating service mat-bmc-host-xxxxx: Service "mat-bmc-host-xxxxx" is invalid: +spec.clusterIP: Invalid value: "192.168.100.5": provided IP is not in the valid range +``` + +**Cause:** Machine-a-tron's `oobDhcpRelayAddress` is not within Kubernetes ServiceCIDR. + +**Solution:** Update `oobDhcpRelayAddress` to use IPs from K8s ServiceCIDR (default +`10.96.0.0/12` for kind/k3d clusters). + ## Architecture ```mermaid diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index 475f65e859..44152339b2 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -39,8 +39,6 @@ func main() { "Path to kubeconfig (uses in-cluster config if empty)") targetSelector := flag.String("target-selector", envOrDefault("TARGET_SELECTOR", "app.kubernetes.io/name=nico-machine-a-tron"), "Pod selector for Services (comma-separated key=value pairs)") - clusterIPPrefix := flag.String("cluster-ip-prefix", os.Getenv("CLUSTER_IP_PREFIX"), - "Prefix for static ClusterIP assignment (e.g., 10.96)") insecureSkipVerify := flag.Bool("insecure-skip-verify", envBoolOrDefault("INSECURE_SKIP_VERIFY", true), "Skip TLS certificate verification (for self-signed certs)") logLevel := flag.String("log-level", envOrDefault("LOG_LEVEL", "info"), @@ -65,7 +63,6 @@ func main() { Str("namespace", *namespace). Dur("sync_interval", *syncInterval). Str("target_selector", *targetSelector). - Str("cluster_ip_prefix", *clusterIPPrefix). Bool("insecure_skip_verify", *insecureSkipVerify). Msg("starting controller") @@ -101,9 +98,8 @@ func main() { // Create service builder builder := &controller.ServiceBuilder{ - Namespace: *namespace, - TargetSelector: selector, - ClusterIPPrefix: *clusterIPPrefix, + Namespace: *namespace, + TargetSelector: selector, } // Create reconciler diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go index 8df18301d8..a2b728a510 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -63,10 +63,6 @@ type ServiceBuilder struct { // TargetSelector is the pod selector that Services should target. // This should match the machine-a-tron pod labels. TargetSelector map[string]string - // ClusterIPPrefix is a prefix for static ClusterIP assignment. - // If set along with BMC IP, Services get predictable IPs. - // Format: "10.96.{last-two-octets-of-bmc-ip}" - ClusterIPPrefix string } // BuildServiceName generates a consistent service name for a machine. @@ -147,12 +143,10 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT }, } - // Set static ClusterIP if configured and BMC IP is known - if b.ClusterIPPrefix != "" && machine.BMC.IP != nil { - clusterIP := buildStaticClusterIP(b.ClusterIPPrefix, *machine.BMC.IP) - if clusterIP != "" { - svc.Spec.ClusterIP = clusterIP - } + // Use BMC IP directly as ClusterIP + // Requires machine-a-tron oobDhcpRelayAddress to be within K8s ServiceCIDR + if machine.BMC.IP != nil { + svc.Spec.ClusterIP = *machine.BMC.IP } return svc @@ -180,21 +174,6 @@ func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatu return services } -// buildStaticClusterIP constructs a ClusterIP from a prefix and BMC IP. -// For example, with prefix "10.96" and bmcIP "172.20.0.20", -// it produces "10.96.0.20" (uses last two octets of BMC IP). -func buildStaticClusterIP(prefix, bmcIP string) string { - // Parse last two octets from BMC IP - // This is a simple implementation; production might need more robust parsing - var a, b, c, d int - n, err := fmt.Sscanf(bmcIP, "%d.%d.%d.%d", &a, &b, &c, &d) - if err != nil || n != 4 { - return "" - } - - return fmt.Sprintf("%s.%d.%d", prefix, c, d) -} - // ServiceDiff represents changes between desired and existing services. type ServiceDiff struct { Create []*corev1.Service diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go index f3f8064aa2..1118e2a4e7 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go @@ -184,13 +184,12 @@ func TestServiceBuilder_BuildService_DPU(t *testing.T) { assert.Equal(t, "parent-host-uuid", svc.Labels[LabelParentMatID]) } -func TestServiceBuilder_BuildService_StaticClusterIP(t *testing.T) { +func TestServiceBuilder_BuildService_BMCIPAsClusterIP(t *testing.T) { builder := &ServiceBuilder{ Namespace: "test-ns", TargetSelector: map[string]string{ "app": "machine-a-tron", }, - ClusterIPPrefix: "10.96", } machine := &matclient.MachineStatus{ @@ -198,7 +197,7 @@ func TestServiceBuilder_BuildService_StaticClusterIP(t *testing.T) { APIState: "Ready", PowerState: "On", BMC: matclient.BMCStatus{ - IP: ptr("172.20.0.20"), + IP: ptr("10.100.0.20"), Redfish: matclient.EndpointStatus{ ReachablePort: 443, ListenPort: 8443, @@ -208,8 +207,8 @@ func TestServiceBuilder_BuildService_StaticClusterIP(t *testing.T) { svc := builder.BuildService(machine, MachineTypeHost, "") - // Check static ClusterIP was set - assert.Equal(t, "10.96.0.20", svc.Spec.ClusterIP) + // Check BMC IP is used directly as ClusterIP + assert.Equal(t, "10.100.0.20", svc.Spec.ClusterIP) } func TestServiceBuilder_BuildServicesFromStatus(t *testing.T) { diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml index e3dd5085f6..4887d96c37 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml @@ -28,9 +28,6 @@ spec: - --insecure-skip-verify={{ .Values.config.insecureSkipVerify }} - --log-level={{ .Values.config.logLevel }} - --target-selector={{ include "mat-k8s-controller.targetSelector" . }} - {{- if .Values.config.clusterIpPrefix }} - - --cluster-ip-prefix={{ .Values.config.clusterIpPrefix }} - {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} securityContext: diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml index 5ce419777f..e5c1e7fb32 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -22,11 +22,9 @@ resources: ## Controller configuration config: - ## Machine-a-tron URL (optional) ## Default: https://-bmc-mock..svc.cluster.local:1266 matUrl: "" - ## Target selector for Services (optional) ## Default: app.kubernetes.io/name=nico-machine-a-tron targetSelector: "" @@ -39,17 +37,11 @@ config: ## Log level (debug, info, warn, error) logLevel: "info" - ## Optional: Prefix for static ClusterIP assignment - ## Leave empty for Kubernetes auto-assigned ClusterIPs - clusterIpPrefix: "" - -## Service account configuration serviceAccount: create: true name: "" annotations: {} -## RBAC configuration rbac: create: true From 3e152216be8f3b3f4b8c3940eb83de93ffd5ec12 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 22 Jul 2026 22:01:42 -0600 Subject: [PATCH 06/17] fix: machine-a-tron controller pod discovery --- dev/k8s/machine-a-tron-controller/README.md | 208 ++++++------------ .../cmd/mat-k8s-controller/main.go | 75 +++---- .../pkg/controller/controller.go | 107 +++++---- .../pkg/controller/controller_test.go | 60 ++--- .../pkg/controller/discovery.go | 56 +++++ .../mat-k8s-controller/templates/_helpers.tpl | 10 +- .../templates/deployment.yaml | 2 +- .../charts/mat-k8s-controller/values.yaml | 12 +- .../templates/deployment.yaml | 6 +- .../templates/service.yaml | 1 + helm/charts/nico-machine-a-tron/values.yaml | 53 +---- 11 files changed, 259 insertions(+), 331 deletions(-) create mode 100644 dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md index 53f2d41079..eadb30b3e4 100644 --- a/dev/k8s/machine-a-tron-controller/README.md +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -1,36 +1,30 @@ # Machine-a-tron Kubernetes Controller -A Kubernetes controller that reconciles Services for machine-a-tron mock BMC endpoints. +A Kubernetes controller that auto-discovers machine-a-tron pods and creates Services for mock BMC endpoints. ## Overview -When machine-a-tron runs in Kubernetes, mock BMCs need stable network exposure so that -NICo components can reach Redfish and IPMI/SOL endpoints. -This controller polls machine-a-tron status and reconciles one Service per mock BMC. +In multi-pod machine-a-tron deployments, each pod simulates different machines with unique BMC endpoints. This controller: -Machine-a-tron itself remains Kubernetes-agnostic. This controller bridges the gap. +1. **Discovers** all machine-a-tron bmc-mock Services in the namespace +2. **Polls** each pod's `/machines/status` API +3. **Creates** one Kubernetes Service per BMC with: + - Redfish TCP port (443 → internal listen port) + - IPMI UDP port (623 → internal listen port) when enabled +4. **Updates** Services when machine status changes +5. **Deletes** stale Services when machines disappear ## Features -- Polls machine-a-tron `GET /machines/status` API -- Creates one Service per mock BMC (hosts and DPUs) -- Exposes Redfish TCP (port 443 → internal listen port) -- Exposes IPMI UDP when enabled (port 623 → internal listen port) -- Uses BMC IP directly as Service ClusterIP -- Labels and annotates Services with machine/BMC identity -- Updates Services when machine status changes -- Deletes stale Services when machines disappear +- **Auto-discovery**: Finds all machine-a-tron pods automatically +- **Multi-pod support**: Aggregates machines from all pods +- **Direct BMC IP as ClusterIP**: Services use BMC IP as ClusterIP (requires oobDhcpRelayAddress within K8s ServiceCIDR) +- **Automatic cleanup**: Removes Services for deleted machines ## Build ```bash -# Build binary -go build -o mat-k8s-controller ./cmd/mat-k8s-controller - -# Build container docker build -t mat-k8s-controller:latest . - -# Load to kind cluster kind load docker-image mat-k8s-controller:latest --name ``` @@ -38,172 +32,96 @@ kind load docker-image mat-k8s-controller:latest --name | Flag | Env Var | Default | Description | |------|---------|---------|-------------| -| `--mat-url` | `MAT_URL` | `https://nico-machine-a-tron-bmc-mock:1266` | Machine-a-tron base URL | -| `--namespace` | `NAMESPACE` | `nico-system` | Namespace for Services | +| `--namespace` | `NAMESPACE` | `nico-system` | Namespace for Services and discovery | | `--sync-interval` | `SYNC_INTERVAL` | `30s` | Reconciliation interval | -| `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector | +| `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector for Services | +| `--bmc-mock-port` | `BMC_MOCK_PORT` | `1266` | BMC mock service port | | `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `true` | Skip TLS verification | | `--log-level` | `LOG_LEVEL` | `info` | Log level | -## ClusterIP Assignment - -The controller uses BMC IP directly as Service ClusterIP. This requires machine-a-tron's -`oobDhcpRelayAddress` to be within the Kubernetes ServiceCIDR range. - -### Setup - -1. Configure machine-a-tron with `oobDhcpRelayAddress` from K8s ServiceCIDR: - ```yaml - pods: - pod-0: - machines: - compute: - oobDhcpRelayAddress: "10.100.0.1" # Must be in K8s ServiceCIDR - ``` - -2. Add the network to NICo configuration: - ```toml - [networks.MAT-BMC-SERVICES] - type = "underlay" - prefix = "10.100.0.0/20" - ``` - -3. Result: BMC IP `10.100.0.5` → Service ClusterIP `10.100.0.5` - -### CIDR Reservation (Recommended) - -To avoid conflicts with other Services, reserve a CIDR range for machine-a-tron. -Kubernetes 1.29+ supports multiple ServiceCIDRs with the `MultiCIDRServiceAllocator` -feature gate: - -```yaml -apiVersion: networking.k8s.io/v1beta1 -kind: ServiceCIDR -metadata: - name: machine-a-tron -spec: - cidrs: - - 10.100.0.0/16 -``` - ## Helm Deployment -The controller is a subchart of `nico-machine-a-tron`. When enabled, it dynamically -manages BMC Services instead of static Helm-generated ones. +Enable in your values: -```bash -helm upgrade --install nico-machine-a-tron ./helm/charts/nico-machine-a-tron \ - --namespace nico-system \ - --create-namespace \ - --set mat-k8s-controller.enabled=true \ - --set mat-k8s-controller.image.pullPolicy=Never +```yaml +nico-machine-a-tron: + mat-k8s-controller: + enabled: true + image: + repository: mat-k8s-controller + tag: latest + pullPolicy: Never + config: + logLevel: debug ``` -Configuration is inherited from the parent chart: -- `matUrl` → derived from release name -- `namespace` → release namespace -- `targetSelector` → matches machine-a-tron pods - ## Service Structure -Each Service has: +Each created Service has: **Labels:** - `app.kubernetes.io/managed-by: mat-k8s-controller` -- `machine-a-tron.nvidia.com/mat-id` -- `machine-a-tron.nvidia.com/machine-type` (host/dpu) +- `machine-a-tron.nvidia.com/mat-id: ` +- `machine-a-tron.nvidia.com/machine-type: host|dpu` **Annotations:** -- `machine-a-tron.nvidia.com/bmc-ip` -- `machine-a-tron.nvidia.com/api-state` -- `machine-a-tron.nvidia.com/power-state` +- `machine-a-tron.nvidia.com/bmc-ip: ` +- `machine-a-tron.nvidia.com/api-state: ` +- `machine-a-tron.nvidia.com/power-state: ` **Ports:** -- `redfish` TCP 443 → targetPort (always) -- `ipmi` UDP 623 → targetPort (when IPMI enabled) - -## Development - -```bash -# Run tests -go test ./... - -# Run locally -./mat-k8s-controller --kubeconfig ~/.kube/config --mat-url https://localhost:1266 -``` +- `redfish`: TCP 443 → targetPort (listen port) +- `ipmi`: UDP 623 → targetPort (listen port) - when IPMI enabled ## Troubleshooting -### ClusterIP already allocated +### No machine-a-tron instances discovered -**Error:** -``` -creating service mat-bmc-host-xxxxx: Service "mat-bmc-host-xxxxx" is invalid: -spec.clusterIP: Invalid value: "10.100.0.5": provided IP is already allocated -``` +Check that machine-a-tron pods have bmc-mock Services: -**Cause:** Another Service is using the same ClusterIP. This happens when the -BMC IP range overlaps with Kubernetes' auto-allocated ClusterIPs. +```bash +kubectl -n nico-system get svc -l app.kubernetes.io/name=nico-machine-a-tron +``` -**Solutions:** +### ClusterIP already allocated -1. **Reserve a ServiceCIDR** (Kubernetes 1.29+, recommended for production): - ```yaml - apiVersion: networking.k8s.io/v1beta1 - kind: ServiceCIDR - metadata: - name: machine-a-tron - spec: - cidrs: - - 10.100.0.0/16 - ``` +The BMC IP is already used by another Service. Options: -2. **Use a different CIDR range** in machine-a-tron `oobDhcpRelayAddress` that - doesn't overlap with existing Services. +1. Reserve a ServiceCIDR for machine-a-tron (K8s 1.29+) +2. Use a different oobDhcpRelayAddress range +3. Delete conflicting Services -3. **Delete the conflicting Service** if it's no longer needed: - ```bash - kubectl get svc -A -o wide | grep 10.100.0.5 - kubectl delete svc -n - ``` +## Development -**Impact:** The affected machine's BMC Service is not created. In NICo site-explorer, -the machine appears unhealthy until the conflict is resolved. +```bash +# Build +go build ./... -### BMC IP outside ServiceCIDR +# Test +go test ./... -**Error:** -``` -creating service mat-bmc-host-xxxxx: Service "mat-bmc-host-xxxxx" is invalid: -spec.clusterIP: Invalid value: "192.168.100.5": provided IP is not in the valid range +# Run locally +go run ./cmd/mat-k8s-controller --kubeconfig ~/.kube/config --log-level debug ``` -**Cause:** Machine-a-tron's `oobDhcpRelayAddress` is not within Kubernetes ServiceCIDR. - -**Solution:** Update `oobDhcpRelayAddress` to use IPs from K8s ServiceCIDR (default -`10.96.0.0/12` for kind/k3d clusters). - ## Architecture ```mermaid flowchart LR - subgraph MAT[machine-a-tron] - API["/machines/status"] - end - subgraph K8s[Kubernetes] Controller[mat-k8s-controller] - subgraph Services - HostSvc[mat-bmc-host-xxx] - DpuSvc[mat-bmc-dpu-yyy] + subgraph MAT[machine-a-tron pods] + MAT0[mat-0-bmc-mock] + MAT1[mat-1-bmc-mock] + end + subgraph Services[Created Services] + Svc1[mat-bmc-host-xxx] + Svc2[mat-bmc-dpu-yyy] end end - Controller -->|polls| API + Controller -->|discovers| MAT + Controller -->|polls /machines/status| MAT0 + Controller -->|polls /machines/status| MAT1 Controller -->|creates/updates/deletes| Services ``` - -## Related - -- [GitHub Issue #3380](https://github.com/NVIDIA/infra-controller/issues/3380) - Feature request -- [GitHub Issue #3379](https://github.com/NVIDIA/infra-controller/issues/3379) - Status API diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index 44152339b2..996132577f 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -4,9 +4,9 @@ // mat-k8s-controller is a Kubernetes controller that reconciles Services // for machine-a-tron mock BMC endpoints. // -// It polls the machine-a-tron /machines/status API and creates/updates/deletes -// Kubernetes Services to expose Redfish (and optionally IPMI) endpoints for -// each mock BMC. +// It discovers machine-a-tron pods via their bmc-mock Services, polls each +// pod's /machines/status API, and creates/updates/deletes Kubernetes Services +// to expose Redfish (and optionally IPMI) endpoints for each mock BMC. package main import ( @@ -15,6 +15,7 @@ import ( "os" "os/signal" "strconv" + "strings" "syscall" "time" @@ -29,10 +30,10 @@ import ( func main() { // Flags - matURL := flag.String("mat-url", envOrDefault("MAT_URL", "https://nico-machine-a-tron-bmc-mock:1266"), - "Machine-a-tron base URL") namespace := flag.String("namespace", envOrDefault("NAMESPACE", "nico-system"), - "Kubernetes namespace for Services") + "Kubernetes namespace for Services and machine-a-tron discovery") + discoverySelector := flag.String("discovery-selector", envOrDefault("DISCOVERY_SELECTOR", "machine-a-tron.nvidia.com/service=true"), + "Label selector for discovering machine-a-tron bmc-mock Services") syncInterval := flag.Duration("sync-interval", parseDurationOrDefault("SYNC_INTERVAL", 30*time.Second), "Interval between reconciliation passes") kubeconfig := flag.String("kubeconfig", os.Getenv("KUBECONFIG"), @@ -43,6 +44,8 @@ func main() { "Skip TLS certificate verification (for self-signed certs)") logLevel := flag.String("log-level", envOrDefault("LOG_LEVEL", "info"), "Log level (debug, info, warn, error)") + bmcMockPort := flag.Int("bmc-mock-port", envIntOrDefault("BMC_MOCK_PORT", 1266), + "Port number for bmc-mock service") flag.Parse() @@ -59,24 +62,14 @@ func main() { Logger() logger.Info(). - Str("mat_url", *matURL). Str("namespace", *namespace). + Str("discovery_selector", *discoverySelector). Dur("sync_interval", *syncInterval). Str("target_selector", *targetSelector). + Int("bmc_mock_port", *bmcMockPort). Bool("insecure_skip_verify", *insecureSkipVerify). Msg("starting controller") - // Create machine-a-tron client - clientOpts := []matclient.Option{matclient.WithLogger(logger)} - if *insecureSkipVerify { - clientOpts = append(clientOpts, matclient.WithInsecureSkipVerify()) - } - - matClient, err := matclient.NewClient(*matURL, clientOpts...) - if err != nil { - logger.Fatal().Err(err).Msg("failed to create machine-a-tron client") - } - // Create Kubernetes client var k8sConfig *rest.Config if *kubeconfig != "" { @@ -93,6 +86,12 @@ func main() { logger.Fatal().Err(err).Msg("failed to create Kubernetes clientset") } + // Create machine-a-tron client options + clientOpts := []matclient.Option{matclient.WithLogger(logger)} + if *insecureSkipVerify { + clientOpts = append(clientOpts, matclient.WithInsecureSkipVerify()) + } + // Parse target selector selector := parseSelector(*targetSelector) @@ -102,9 +101,10 @@ func main() { TargetSelector: selector, } - // Create reconciler + // Create discovery and reconciler + discovery := controller.NewMatPodDiscovery(clientset, *namespace, *bmcMockPort, *discoverySelector) k8sClient := controller.NewRealK8sServiceClient(clientset) - reconciler := controller.NewReconciler(matClient, builder, k8sClient) + reconciler := controller.NewReconciler(discovery, builder, k8sClient, clientOpts, logger) // Setup signal handling ctx, cancel := context.WithCancel(context.Background()) @@ -180,6 +180,15 @@ func envBoolOrDefault(key string, defaultValue bool) bool { return defaultValue } +func envIntOrDefault(key string, defaultValue int) int { + if v := os.Getenv(key); v != "" { + if i, err := strconv.Atoi(v); err == nil { + return i + } + } + return defaultValue +} + func parseDurationOrDefault(envKey string, defaultValue time.Duration) time.Duration { if v := os.Getenv(envKey); v != "" { if d, err := time.ParseDuration(v); err == nil { @@ -195,32 +204,10 @@ func parseSelector(s string) map[string]string { return result } - // Simple parser for key=value,key2=value2 format - pairs := splitPairs(s, ',') - for _, pair := range pairs { - kv := splitPairs(pair, '=') - if len(kv) == 2 { + for _, pair := range strings.Split(s, ",") { + if kv := strings.SplitN(pair, "=", 2); len(kv) == 2 { result[kv[0]] = kv[1] } } return result } - -func splitPairs(s string, sep rune) []string { - var result []string - var current string - for _, r := range s { - if r == sep { - if current != "" { - result = append(result, current) - } - current = "" - } else { - current += string(r) - } - } - if current != "" { - result = append(result, current) - } - return result -} diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go index a2b728a510..2dd300e44b 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -10,6 +10,7 @@ import ( "fmt" "strconv" + "github.com/rs/zerolog" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -61,13 +62,11 @@ type ServiceBuilder struct { // Namespace is the target namespace for Services. Namespace string // TargetSelector is the pod selector that Services should target. - // This should match the machine-a-tron pod labels. TargetSelector map[string]string } // BuildServiceName generates a consistent service name for a machine. func BuildServiceName(machineType, matID string) string { - // Use first 8 chars of mat-id for reasonable length shortID := matID if len(matID) > 8 { shortID = matID[:8] @@ -94,19 +93,15 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT if machine.MachineID != nil { labels[LabelMachineID] = *machine.MachineID } - if parentMatID != "" { labels[LabelParentMatID] = parentMatID } - if machine.BMC.IP != nil { annotations[AnnotationBMCIP] = *machine.BMC.IP } - if machine.HardwareType != nil { annotations[AnnotationHardwareType] = *machine.HardwareType } - if machine.BMC.IPMI != nil { annotations[AnnotationIPMIListenPort] = strconv.Itoa(int(machine.BMC.IPMI.ListenPort)) } @@ -144,7 +139,6 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT } // Use BMC IP directly as ClusterIP - // Requires machine-a-tron oobDhcpRelayAddress to be within K8s ServiceCIDR if machine.BMC.IP != nil { svc.Spec.ClusterIP = *machine.BMC.IP } @@ -158,12 +152,9 @@ func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatu for i := range status.Machines { machine := &status.Machines[i] - - // Build service for the host BMC svc := b.BuildService(machine, MachineTypeHost, "") services = append(services, svc) - // Build services for DPU BMCs for j := range machine.DPUs { dpu := &machine.DPUs[j] dpuSvc := b.BuildService(dpu, MachineTypeDPU, machine.MatID) @@ -178,7 +169,7 @@ func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatu type ServiceDiff struct { Create []*corev1.Service Update []*corev1.Service - Delete []string // service names to delete + Delete []string } // ComputeServiceDiff calculates what changes need to be made. @@ -201,9 +192,7 @@ func ComputeServiceDiff(desired []*corev1.Service, existing []*corev1.Service) S } if needsUpdate(svc, existingSvc) { - // Copy ResourceVersion for update svc.ResourceVersion = existingSvc.ResourceVersion - // Preserve ClusterIP if not explicitly set if svc.Spec.ClusterIP == "" { svc.Spec.ClusterIP = existingSvc.Spec.ClusterIP } @@ -211,7 +200,6 @@ func ComputeServiceDiff(desired []*corev1.Service, existing []*corev1.Service) S } } - // Find services to delete (exist but not in desired) for name, svc := range existingByName { if !desiredNames[name] && isManagedByController(svc) { diff.Delete = append(diff.Delete, name) @@ -221,9 +209,7 @@ func ComputeServiceDiff(desired []*corev1.Service, existing []*corev1.Service) S return diff } -// needsUpdate checks if a service needs to be updated. func needsUpdate(desired, existing *corev1.Service) bool { - // Check port changes if len(desired.Spec.Ports) != len(existing.Spec.Ports) { return true } @@ -243,14 +229,12 @@ func needsUpdate(desired, existing *corev1.Service) bool { } } - // Check label changes (except for managed-by which should always be set) for k, v := range desired.Labels { if existing.Labels[k] != v { return true } } - // Check annotation changes for k, v := range desired.Annotations { if existing.Annotations[k] != v { return true @@ -260,18 +244,10 @@ func needsUpdate(desired, existing *corev1.Service) bool { return false } -// isManagedByController checks if a service is managed by this controller. func isManagedByController(svc *corev1.Service) bool { return svc.Labels[LabelManagedBy] == LabelManagedByValue } -// Reconciler handles the reconciliation loop. -type Reconciler struct { - matClient *matclient.Client - serviceBuilder *ServiceBuilder - k8sClient K8sServiceClient -} - // K8sServiceClient is an interface for Kubernetes Service operations. type K8sServiceClient interface { List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) @@ -280,15 +256,6 @@ type K8sServiceClient interface { Delete(ctx context.Context, namespace, name string) error } -// NewReconciler creates a new Reconciler. -func NewReconciler(matClient *matclient.Client, builder *ServiceBuilder, k8sClient K8sServiceClient) *Reconciler { - return &Reconciler{ - matClient: matClient, - serviceBuilder: builder, - k8sClient: k8sClient, - } -} - // ReconcileResult contains the result of a reconciliation. type ReconcileResult struct { Created int @@ -297,19 +264,70 @@ type ReconcileResult struct { Errors []error } -// Reconcile performs a single reconciliation pass. +// Reconciler discovers machine-a-tron pods and reconciles Services. +type Reconciler struct { + discovery *MatPodDiscovery + serviceBuilder *ServiceBuilder + k8sClient K8sServiceClient + clientOpts []matclient.Option + logger zerolog.Logger +} + +// NewReconciler creates a new Reconciler. +func NewReconciler( + discovery *MatPodDiscovery, + builder *ServiceBuilder, + k8sClient K8sServiceClient, + clientOpts []matclient.Option, + logger zerolog.Logger, +) *Reconciler { + return &Reconciler{ + discovery: discovery, + serviceBuilder: builder, + k8sClient: k8sClient, + clientOpts: clientOpts, + logger: logger, + } +} + +// Reconcile performs a reconciliation pass across all discovered machine-a-tron instances. func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { result := ReconcileResult{} - // Fetch current machine status - status, err := r.matClient.GetMachinesStatus(ctx) + // Discover machine-a-tron URLs + urls, err := r.discovery.DiscoverURLs(ctx) if err != nil { - result.Errors = append(result.Errors, fmt.Errorf("fetching machine status: %w", err)) + result.Errors = append(result.Errors, fmt.Errorf("discovering machine-a-tron instances: %w", err)) + return result + } + + if len(urls) == 0 { + r.logger.Warn().Msg("no machine-a-tron instances discovered") return result } - // Build desired services - desired := r.serviceBuilder.BuildServicesFromStatus(status) + r.logger.Debug().Strs("urls", urls).Msg("discovered machine-a-tron instances") + + // Collect all machines from all instances + var allDesired []*corev1.Service + + for _, url := range urls { + client, err := matclient.NewClient(url, r.clientOpts...) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("creating client for %s: %w", url, err)) + continue + } + + status, err := client.GetMachinesStatus(ctx) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("fetching status from %s: %w", url, err)) + continue + } + + services := r.serviceBuilder.BuildServicesFromStatus(status) + r.logger.Debug().Str("url", url).Int("machines", len(services)).Msg("fetched machines from instance") + allDesired = append(allDesired, services...) + } // List existing managed services selector := fmt.Sprintf("%s=%s", LabelManagedBy, LabelManagedByValue) @@ -319,10 +337,9 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { return result } - // Compute diff - diff := ComputeServiceDiff(desired, existing) + // Compute and apply diff + diff := ComputeServiceDiff(allDesired, existing) - // Apply creates for _, svc := range diff.Create { if err := r.k8sClient.Create(ctx, svc); err != nil { result.Errors = append(result.Errors, fmt.Errorf("creating service %s: %w", svc.Name, err)) @@ -331,7 +348,6 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { } } - // Apply updates for _, svc := range diff.Update { if err := r.k8sClient.Update(ctx, svc); err != nil { result.Errors = append(result.Errors, fmt.Errorf("updating service %s: %w", svc.Name, err)) @@ -340,7 +356,6 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { } } - // Apply deletes for _, name := range diff.Delete { if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { result.Errors = append(result.Errors, fmt.Errorf("deleting service %s: %w", name, err)) diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go index 1118e2a4e7..a4de26c7f1 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go @@ -388,10 +388,10 @@ func TestComputeServiceDiff(t *testing.T) { } } -func TestReconciler_Reconcile(t *testing.T) { +func TestReconcileLogic(t *testing.T) { ctx := context.Background() - mockClient := &mockK8sClient{ + mockK8s := &mockK8sClient{ services: make(map[string]*corev1.Service), } @@ -402,58 +402,53 @@ func TestReconciler_Reconcile(t *testing.T) { }, } - matClient := &mockMatClient{ - status: &matclient.MachinesStatusResponse{ - Machines: []matclient.MachineStatus{ - { - MatID: "host-1234", - APIState: "Ready", - PowerState: "On", - BMC: matclient.BMCStatus{ - IP: ptr("192.168.1.100"), - Redfish: matclient.EndpointStatus{ - ReachablePort: 443, - ListenPort: 8443, - }, + status := &matclient.MachinesStatusResponse{ + Machines: []matclient.MachineStatus{ + { + MatID: "host-1234", + APIState: "Ready", + PowerState: "On", + BMC: matclient.BMCStatus{ + IP: ptr("192.168.1.100"), + Redfish: matclient.EndpointStatus{ + ReachablePort: 443, + ListenPort: 8443, }, }, }, }, } - reconciler := NewReconciler(nil, builder, mockClient) - reconciler.matClient = nil // We'll use the mock - // First reconcile: should create service - result := reconcileWithMockMatClient(ctx, reconciler, mockClient, matClient.status) + result := runTestReconcile(ctx, builder, mockK8s, status) assert.Equal(t, 1, result.Created) assert.Equal(t, 0, result.Updated) assert.Equal(t, 0, result.Deleted) assert.Empty(t, result.Errors) - assert.Len(t, mockClient.services, 1) + assert.Len(t, mockK8s.services, 1) // Second reconcile with same state: no changes - result = reconcileWithMockMatClient(ctx, reconciler, mockClient, matClient.status) + result = runTestReconcile(ctx, builder, mockK8s, status) assert.Equal(t, 0, result.Created) assert.Equal(t, 0, result.Updated) assert.Equal(t, 0, result.Deleted) // Third reconcile with empty status: should delete - result = reconcileWithMockMatClient(ctx, reconciler, mockClient, &matclient.MachinesStatusResponse{}) + result = runTestReconcile(ctx, builder, mockK8s, &matclient.MachinesStatusResponse{}) assert.Equal(t, 0, result.Created) assert.Equal(t, 0, result.Updated) assert.Equal(t, 1, result.Deleted) - assert.Empty(t, mockClient.services) + assert.Empty(t, mockK8s.services) } -// Helper function to run reconcile with mock mat client -func reconcileWithMockMatClient(ctx context.Context, r *Reconciler, k8s *mockK8sClient, status *matclient.MachinesStatusResponse) ReconcileResult { +// runTestReconcile tests the reconciliation logic without discovery. +func runTestReconcile(ctx context.Context, builder *ServiceBuilder, k8s *mockK8sClient, status *matclient.MachinesStatusResponse) ReconcileResult { result := ReconcileResult{} - desired := r.serviceBuilder.BuildServicesFromStatus(status) + desired := builder.BuildServicesFromStatus(status) selector := LabelManagedBy + "=" + LabelManagedByValue - existing, _ := k8s.List(ctx, r.serviceBuilder.Namespace, selector) + existing, _ := k8s.List(ctx, builder.Namespace, selector) diff := ComputeServiceDiff(desired, existing) @@ -474,7 +469,7 @@ func reconcileWithMockMatClient(ctx context.Context, r *Reconciler, k8s *mockK8s } for _, name := range diff.Delete { - if err := k8s.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { + if err := k8s.Delete(ctx, builder.Namespace, name); err != nil { result.Errors = append(result.Errors, err) } else { result.Deleted++ @@ -544,15 +539,6 @@ func (m *mockK8sClient) Delete(ctx context.Context, namespace, name string) erro return nil } -type mockMatClient struct { - status *matclient.MachinesStatusResponse - err error -} - -func (m *mockMatClient) GetMachinesStatus(ctx context.Context) (*matclient.MachinesStatusResponse, error) { - return m.status, m.err -} - func ptr[T any](v T) *T { return &v } diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go new file mode 100644 index 0000000000..d4584e71fe --- /dev/null +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// DefaultDiscoverySelector is the default label selector for discovering +// machine-a-tron bmc-mock Services. +const DefaultDiscoverySelector = "machine-a-tron.nvidia.com/service=true" + +// MatPodDiscovery discovers machine-a-tron pods via their bmc-mock Services. +type MatPodDiscovery struct { + clientset kubernetes.Interface + namespace string + port int + labelSelector string +} + +// NewMatPodDiscovery creates a new MatPodDiscovery. +// If labelSelector is empty, uses DefaultDiscoverySelector. +func NewMatPodDiscovery(clientset kubernetes.Interface, namespace string, port int, labelSelector string) *MatPodDiscovery { + if labelSelector == "" { + labelSelector = DefaultDiscoverySelector + } + return &MatPodDiscovery{ + clientset: clientset, + namespace: namespace, + port: port, + labelSelector: labelSelector, + } +} + +// DiscoverURLs finds all machine-a-tron bmc-mock service URLs. +func (d *MatPodDiscovery) DiscoverURLs(ctx context.Context) ([]string, error) { + services, err := d.clientset.CoreV1().Services(d.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: d.labelSelector, + }) + if err != nil { + return nil, fmt.Errorf("listing services with selector %q: %w", d.labelSelector, err) + } + + var urls []string + for _, svc := range services.Items { + url := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", svc.Name, d.namespace, d.port) + urls = append(urls, url) + } + + return urls, nil +} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl index 68fd9e9e65..929925b427 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl @@ -63,14 +63,10 @@ Namespace - inherited from parent release {{- end }} {{/* -Machine-a-tron URL - configurable, defaults to release-based naming +Discovery selector - finds machine-a-tron bmc-mock Services */}} -{{- define "mat-k8s-controller.matUrl" -}} -{{- if .Values.config.matUrl }} -{{- .Values.config.matUrl }} -{{- else }} -{{- printf "https://%s-bmc-mock.%s.svc.cluster.local:1266" .Release.Name .Release.Namespace }} -{{- end }} +{{- define "mat-k8s-controller.discoverySelector" -}} +{{- default "machine-a-tron.nvidia.com/service=true" .Values.config.discoverySelector }} {{- end }} {{/* diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml index 4887d96c37..5ece9c4514 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/deployment.yaml @@ -22,8 +22,8 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} args: - - --mat-url={{ include "mat-k8s-controller.matUrl" . }} - --namespace={{ include "mat-k8s-controller.namespace" . }} + - --discovery-selector={{ include "mat-k8s-controller.discoverySelector" . }} - --sync-interval={{ .Values.config.syncInterval }} - --insecure-skip-verify={{ .Values.config.insecureSkipVerify }} - --log-level={{ .Values.config.logLevel }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml index e5c1e7fb32..7f60e8c0a0 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -1,7 +1,7 @@ -## mat-k8s-controller - Kubernetes controller for machine-a-tron BMC services +## mat-k8s-controller - Kubernetes controller for machine-a-tron services ## -## This controller dynamically creates/updates/deletes Kubernetes Services -## based on machine-a-tron's /machines/status API. +## This controller auto-discovers machine-a-tron pods via their bmc-mock Services +## and creates/updates/deletes Kubernetes Services for each endpoint. ## Enable/disable the controller enabled: false @@ -22,9 +22,11 @@ resources: ## Controller configuration config: - ## Default: https://-bmc-mock..svc.cluster.local:1266 - matUrl: "" + ## Label selector for discovering machine-a-tron Services + ## Default: machine-a-tron.nvidia.com/service=true + discoverySelector: "" + ## Pod selector for created Services (routes traffic to mat pods) ## Default: app.kubernetes.io/name=nico-machine-a-tron targetSelector: "" diff --git a/helm/charts/nico-machine-a-tron/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/templates/deployment.yaml index 05074aa06f..b041a0fb63 100644 --- a/helm/charts/nico-machine-a-tron/templates/deployment.yaml +++ b/helm/charts/nico-machine-a-tron/templates/deployment.yaml @@ -14,7 +14,7 @@ Count pods that should be deployed: {{- $activePods = add $activePods 1 }} {{- end }} {{- else }} -{{- if $podConfig.machines }} +{{- if and $podConfig.machines (gt (len $podConfig.machines) 0) }} {{- $activePods = add $activePods 1 }} {{- end }} {{- end }} @@ -24,14 +24,14 @@ Count pods that should be deployed: {{/* Skip conditions: - When bmcServices.enabled: skip pods without CIDR -- Otherwise: skip pods without machines +- Otherwise: skip pods without machines (or empty machines) */}} {{- if $root.Values.bmcServices.enabled }} {{- if not $podConfig.cidr }} {{- continue }} {{- end }} {{- else }} -{{- if not $podConfig.machines }} +{{- if not (and $podConfig.machines (gt (len $podConfig.machines) 0)) }} {{- continue }} {{- end }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/service.yaml b/helm/charts/nico-machine-a-tron/templates/service.yaml index a8faab7fc6..9c15ea3f07 100644 --- a/helm/charts/nico-machine-a-tron/templates/service.yaml +++ b/helm/charts/nico-machine-a-tron/templates/service.yaml @@ -45,6 +45,7 @@ metadata: labels: {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} app: {{ $baseName }} + machine-a-tron.nvidia.com/service: "true" {{- if gt $activePods 1 }} nvidia-infra-controller/pod-name: {{ $podName | quote }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index 9ed30d95fc..b5b11a80f1 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -356,60 +356,27 @@ configFiles: matConfigs: {} ## ============================================================================= -## Machine-a-tron Kubernetes Controller (Subchart) +## Machine-a-tron Kubernetes Controller ## ============================================================================= -## When enabled, the controller dynamically creates BMC services based on -## machine-a-tron's /machines/status API instead of static Helm-generated services. +## Discovers machine-a-tron pods and creates K8s Services for each mock BMC. +## Enable for multi-pod deployments to dynamically manage BMC Services. ## -## Benefits: -## - Services reflect actual machine state (annotations show power/API state) -## - Automatic cleanup when machines are removed -## - No need to pre-calculate BMC counts and CIDRs +## Auto-discovered: +## - Finds bmc-mock Services via label: machine-a-tron.nvidia.com/service=true +## - Polls /machines/status from each discovered instance ## -## Requirements: -## - bmcServices.enabled must also be true (for ServiceCIDR allocation) -## - Controller image must be available in the cluster -## -## ============================================================================= -## Machine-a-tron Kubernetes Controller (Subchart) -## ============================================================================= -## When enabled, the controller dynamically creates BMC services based on -## machine-a-tron's /machines/status API instead of static Helm-generated services. -## -## Configuration is inherited from parent chart: -## - matUrl: derived from release name + bmc-mock service -## - namespace: inherited from release namespace -## - clusterIpPrefix: extracted from pods.*.cidr -## -## Requirements: -## - bmcServices.enabled must also be true (for ServiceCIDR allocation) -## - Controller image must be available in the cluster -## -## ============================================================================= -## Machine-a-tron Kubernetes Controller (Subchart) -## ============================================================================= -## When enabled, the controller dynamically creates BMC services based on -## machine-a-tron's /machines/status API instead of static Helm-generated services. -## -## Inherited automatically: -## - matUrl: https://-bmc-mock..svc.cluster.local:1266 -## - namespace: release namespace -## - targetSelector: app.kubernetes.io/name=nico-machine-a-tron -## -## Requirements: -## - Controller image must be available in the cluster +## Note: BMC IP must be within Kubernetes ServiceCIDR (default 10.96.0.0/12) +## for ClusterIP assignment. See README troubleshooting for ClusterIP conflicts. ## mat-k8s-controller: enabled: false - + image: repository: mat-k8s-controller tag: latest pullPolicy: IfNotPresent - + config: syncInterval: "30s" insecureSkipVerify: true logLevel: "info" - ## Optional: set for predictable ClusterIPs (e.g., "10.96") - clusterIpPrefix: "" From b3f6a4ff622726aa492623dd4fb0115b4d37a396 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 22 Jul 2026 22:44:34 -0600 Subject: [PATCH 07/17] fix: machine-a-tron controller pod selector --- .../cmd/mat-k8s-controller/main.go | 2 +- .../pkg/controller/controller.go | 62 ++++++++++++++----- .../pkg/controller/controller_test.go | 27 ++++---- .../pkg/controller/discovery.go | 20 ++++-- 4 files changed, 75 insertions(+), 36 deletions(-) diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index 996132577f..6d247ff515 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -98,7 +98,7 @@ func main() { // Create service builder builder := &controller.ServiceBuilder{ Namespace: *namespace, - TargetSelector: selector, + BaseSelector: selector, } // Create discovery and reconciler diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go index 2dd300e44b..7ecee08873 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -24,6 +24,9 @@ const ( // LabelManagedByValue is the value for the managed-by label. LabelManagedByValue = "mat-k8s-controller" + // LabelPodName is the label that identifies which machine-a-tron pod owns this service. + LabelPodName = "nvidia-infra-controller/pod-name" + // LabelMatID is the machine-a-tron ID label. LabelMatID = "machine-a-tron.nvidia.com/mat-id" // LabelMachineID is the NICo machine ID label. @@ -61,8 +64,8 @@ const ( type ServiceBuilder struct { // Namespace is the target namespace for Services. Namespace string - // TargetSelector is the pod selector that Services should target. - TargetSelector map[string]string + // BaseSelector is the base pod selector (e.g., app.kubernetes.io/name=nico-machine-a-tron). + BaseSelector map[string]string } // BuildServiceName generates a consistent service name for a machine. @@ -75,7 +78,8 @@ func BuildServiceName(machineType, matID string) string { } // BuildService creates a Kubernetes Service for a machine's BMC. -func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineType, parentMatID string) *corev1.Service { +// podName is used to create a pod-specific selector for multi-pod deployments. +func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineType, parentMatID, podName string) *corev1.Service { name := BuildServiceName(machineType, machine.MatID) labels := map[string]string{ @@ -96,6 +100,9 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT if parentMatID != "" { labels[LabelParentMatID] = parentMatID } + if podName != "" { + labels[LabelPodName] = podName + } if machine.BMC.IP != nil { annotations[AnnotationBMCIP] = *machine.BMC.IP } @@ -124,6 +131,15 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT }) } + // Build selector: base selector + pod-specific selector for multi-pod + selector := make(map[string]string) + for k, v := range b.BaseSelector { + selector[k] = v + } + if podName != "" { + selector[LabelPodName] = podName + } + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -133,7 +149,7 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, - Selector: b.TargetSelector, + Selector: selector, Ports: ports, }, } @@ -147,17 +163,18 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT } // BuildServicesFromStatus generates all Services from a machines status response. -func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatusResponse) []*corev1.Service { +// podName identifies which machine-a-tron pod these machines belong to. +func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatusResponse, podName string) []*corev1.Service { var services []*corev1.Service for i := range status.Machines { machine := &status.Machines[i] - svc := b.BuildService(machine, MachineTypeHost, "") + svc := b.BuildService(machine, MachineTypeHost, "", podName) services = append(services, svc) for j := range machine.DPUs { dpu := &machine.DPUs[j] - dpuSvc := b.BuildService(dpu, MachineTypeDPU, machine.MatID) + dpuSvc := b.BuildService(dpu, MachineTypeDPU, machine.MatID, podName) services = append(services, dpuSvc) } } @@ -241,6 +258,13 @@ func needsUpdate(desired, existing *corev1.Service) bool { } } + // Check selector changes (important for multi-pod) + for k, v := range desired.Spec.Selector { + if existing.Spec.Selector[k] != v { + return true + } + } + return false } @@ -294,38 +318,42 @@ func NewReconciler( func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { result := ReconcileResult{} - // Discover machine-a-tron URLs - urls, err := r.discovery.DiscoverURLs(ctx) + // Discover machine-a-tron instances + instances, err := r.discovery.Discover(ctx) if err != nil { result.Errors = append(result.Errors, fmt.Errorf("discovering machine-a-tron instances: %w", err)) return result } - if len(urls) == 0 { + if len(instances) == 0 { r.logger.Warn().Msg("no machine-a-tron instances discovered") return result } - r.logger.Debug().Strs("urls", urls).Msg("discovered machine-a-tron instances") + r.logger.Debug().Int("count", len(instances)).Msg("discovered machine-a-tron instances") // Collect all machines from all instances var allDesired []*corev1.Service - for _, url := range urls { - client, err := matclient.NewClient(url, r.clientOpts...) + for _, instance := range instances { + client, err := matclient.NewClient(instance.URL, r.clientOpts...) if err != nil { - result.Errors = append(result.Errors, fmt.Errorf("creating client for %s: %w", url, err)) + result.Errors = append(result.Errors, fmt.Errorf("creating client for %s: %w", instance.URL, err)) continue } status, err := client.GetMachinesStatus(ctx) if err != nil { - result.Errors = append(result.Errors, fmt.Errorf("fetching status from %s: %w", url, err)) + result.Errors = append(result.Errors, fmt.Errorf("fetching status from %s: %w", instance.URL, err)) continue } - services := r.serviceBuilder.BuildServicesFromStatus(status) - r.logger.Debug().Str("url", url).Int("machines", len(services)).Msg("fetched machines from instance") + services := r.serviceBuilder.BuildServicesFromStatus(status, instance.PodName) + r.logger.Debug(). + Str("url", instance.URL). + Str("pod", instance.PodName). + Int("machines", len(services)). + Msg("fetched machines from instance") allDesired = append(allDesired, services...) } diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go index a4de26c7f1..ad2274f722 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go @@ -50,7 +50,7 @@ func TestBuildServiceName(t *testing.T) { func TestServiceBuilder_BuildService(t *testing.T) { builder := &ServiceBuilder{ Namespace: "test-ns", - TargetSelector: map[string]string{ + BaseSelector: map[string]string{ "app": "machine-a-tron", }, } @@ -70,7 +70,7 @@ func TestServiceBuilder_BuildService(t *testing.T) { }, } - svc := builder.BuildService(machine, MachineTypeHost, "") + svc := builder.BuildService(machine, MachineTypeHost, "", "") // Check basic metadata assert.Equal(t, "mat-bmc-host-host-uui", svc.Name) @@ -97,13 +97,13 @@ func TestServiceBuilder_BuildService(t *testing.T) { assert.Equal(t, intstr.FromInt32(8443), svc.Spec.Ports[0].TargetPort) // Check selector - assert.Equal(t, builder.TargetSelector, svc.Spec.Selector) + assert.Equal(t, builder.BaseSelector, svc.Spec.Selector) } func TestServiceBuilder_BuildService_WithIPMI(t *testing.T) { builder := &ServiceBuilder{ Namespace: "test-ns", - TargetSelector: map[string]string{ + BaseSelector: map[string]string{ "app": "machine-a-tron", }, } @@ -125,7 +125,7 @@ func TestServiceBuilder_BuildService_WithIPMI(t *testing.T) { }, } - svc := builder.BuildService(machine, MachineTypeHost, "") + svc := builder.BuildService(machine, MachineTypeHost, "", "") // Check we have both ports require.Len(t, svc.Spec.Ports, 2) @@ -158,7 +158,7 @@ func TestServiceBuilder_BuildService_WithIPMI(t *testing.T) { func TestServiceBuilder_BuildService_DPU(t *testing.T) { builder := &ServiceBuilder{ Namespace: "test-ns", - TargetSelector: map[string]string{ + BaseSelector: map[string]string{ "app": "machine-a-tron", }, } @@ -177,7 +177,7 @@ func TestServiceBuilder_BuildService_DPU(t *testing.T) { }, } - svc := builder.BuildService(dpu, MachineTypeDPU, "parent-host-uuid") + svc := builder.BuildService(dpu, MachineTypeDPU, "parent-host-uuid", "") // Check DPU-specific labels assert.Equal(t, MachineTypeDPU, svc.Labels[LabelMachineType]) @@ -187,7 +187,7 @@ func TestServiceBuilder_BuildService_DPU(t *testing.T) { func TestServiceBuilder_BuildService_BMCIPAsClusterIP(t *testing.T) { builder := &ServiceBuilder{ Namespace: "test-ns", - TargetSelector: map[string]string{ + BaseSelector: map[string]string{ "app": "machine-a-tron", }, } @@ -205,7 +205,7 @@ func TestServiceBuilder_BuildService_BMCIPAsClusterIP(t *testing.T) { }, } - svc := builder.BuildService(machine, MachineTypeHost, "") + svc := builder.BuildService(machine, MachineTypeHost, "", "") // Check BMC IP is used directly as ClusterIP assert.Equal(t, "10.100.0.20", svc.Spec.ClusterIP) @@ -214,7 +214,7 @@ func TestServiceBuilder_BuildService_BMCIPAsClusterIP(t *testing.T) { func TestServiceBuilder_BuildServicesFromStatus(t *testing.T) { builder := &ServiceBuilder{ Namespace: "test-ns", - TargetSelector: map[string]string{ + BaseSelector: map[string]string{ "app": "machine-a-tron", }, } @@ -258,7 +258,7 @@ func TestServiceBuilder_BuildServicesFromStatus(t *testing.T) { }, } - services := builder.BuildServicesFromStatus(status) + services := builder.BuildServicesFromStatus(status, "") // Should have 4 services: 2 hosts + 2 DPUs assert.Len(t, services, 4) @@ -397,7 +397,7 @@ func TestReconcileLogic(t *testing.T) { builder := &ServiceBuilder{ Namespace: "test-ns", - TargetSelector: map[string]string{ + BaseSelector: map[string]string{ "app": "machine-a-tron", }, } @@ -445,7 +445,7 @@ func TestReconcileLogic(t *testing.T) { func runTestReconcile(ctx context.Context, builder *ServiceBuilder, k8s *mockK8sClient, status *matclient.MachinesStatusResponse) ReconcileResult { result := ReconcileResult{} - desired := builder.BuildServicesFromStatus(status) + desired := builder.BuildServicesFromStatus(status, "") selector := LabelManagedBy + "=" + LabelManagedByValue existing, _ := k8s.List(ctx, builder.Namespace, selector) @@ -542,3 +542,4 @@ func (m *mockK8sClient) Delete(ctx context.Context, namespace, name string) erro func ptr[T any](v T) *T { return &v } + diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go index d4584e71fe..f47e931ca2 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go @@ -15,6 +15,12 @@ import ( // machine-a-tron bmc-mock Services. const DefaultDiscoverySelector = "machine-a-tron.nvidia.com/service=true" +// DiscoveredInstance represents a discovered machine-a-tron instance. +type DiscoveredInstance struct { + URL string + PodName string +} + // MatPodDiscovery discovers machine-a-tron pods via their bmc-mock Services. type MatPodDiscovery struct { clientset kubernetes.Interface @@ -37,8 +43,8 @@ func NewMatPodDiscovery(clientset kubernetes.Interface, namespace string, port i } } -// DiscoverURLs finds all machine-a-tron bmc-mock service URLs. -func (d *MatPodDiscovery) DiscoverURLs(ctx context.Context) ([]string, error) { +// Discover finds all machine-a-tron bmc-mock services and returns their URLs with pod names. +func (d *MatPodDiscovery) Discover(ctx context.Context) ([]DiscoveredInstance, error) { services, err := d.clientset.CoreV1().Services(d.namespace).List(ctx, metav1.ListOptions{ LabelSelector: d.labelSelector, }) @@ -46,11 +52,15 @@ func (d *MatPodDiscovery) DiscoverURLs(ctx context.Context) ([]string, error) { return nil, fmt.Errorf("listing services with selector %q: %w", d.labelSelector, err) } - var urls []string + var instances []DiscoveredInstance for _, svc := range services.Items { url := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", svc.Name, d.namespace, d.port) - urls = append(urls, url) + podName := svc.Labels[LabelPodName] + instances = append(instances, DiscoveredInstance{ + URL: url, + PodName: podName, + }) } - return urls, nil + return instances, nil } From 417d1c23b79a51d1fb85c09d4487b9024282929f Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 00:05:05 -0600 Subject: [PATCH 08/17] fix(machine-a-tron-ctrl): pr review issues --- dev/k8s/machine-a-tron-controller/Dockerfile | 15 +-- dev/k8s/machine-a-tron-controller/Makefile | 43 ++---- dev/k8s/machine-a-tron-controller/README.md | 123 ++++++++---------- .../cmd/mat-k8s-controller/main.go | 10 +- dev/k8s/machine-a-tron-controller/go.mod | 2 +- dev/k8s/machine-a-tron-controller/go.sum | 4 +- .../pkg/controller/controller.go | 82 +++++++++--- .../pkg/matclient/client.go | 86 ++++++++++-- .../mat-k8s-controller/templates/_helpers.tpl | 16 ++- .../charts/mat-k8s-controller/values.yaml | 9 +- helm/charts/nico-machine-a-tron/values.yaml | 2 +- 11 files changed, 236 insertions(+), 156 deletions(-) diff --git a/dev/k8s/machine-a-tron-controller/Dockerfile b/dev/k8s/machine-a-tron-controller/Dockerfile index f62797f4a8..b85f782687 100644 --- a/dev/k8s/machine-a-tron-controller/Dockerfile +++ b/dev/k8s/machine-a-tron-controller/Dockerfile @@ -1,24 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.26.4-alpine AS builder +# Build stage +FROM golang:1.26.4-alpine@sha256:4cfbe7d6f07723e1efe21b4251ccf17d92a8c2a9e6f9e88d3f3f5026e5db8f7d AS builder -WORKDIR /build +WORKDIR /app -# Copy go mod files first for caching COPY go.mod go.sum ./ RUN go mod download -# Copy source COPY . . - -# Build static binary RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o mat-k8s-controller ./cmd/mat-k8s-controller -# Runtime image -FROM gcr.io/distroless/static:nonroot +# Runtime stage +FROM gcr.io/distroless/static:nonroot@sha256:9ecc53c269509f63c69a266168b81a2f4c9d10c2f03e0d53dce6b8a2f9e1c1f1 -COPY --from=builder /build/mat-k8s-controller /mat-k8s-controller +COPY --from=builder /app/mat-k8s-controller /mat-k8s-controller USER nonroot:nonroot diff --git a/dev/k8s/machine-a-tron-controller/Makefile b/dev/k8s/machine-a-tron-controller/Makefile index 7f9f3dd5b2..07a7eb5e79 100644 --- a/dev/k8s/machine-a-tron-controller/Makefile +++ b/dev/k8s/machine-a-tron-controller/Makefile @@ -1,54 +1,37 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -.PHONY: build test lint docker-build clean +.PHONY: build test test-coverage lint fmt docker-build docker-build-latest clean run verify -BINARY := mat-k8s-controller -DOCKER_IMAGE := mat-k8s-controller -VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") +TAG ?= dev -# Build the binary build: - CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/mat-k8s-controller + @mkdir -p bin + go build -o bin/mat-k8s-controller ./cmd/mat-k8s-controller -# Run all tests test: - go test -v -race ./... + go test -v ./... -# Run tests with coverage test-coverage: - go test -v -race -coverprofile=coverage.out ./... + go test -coverprofile=coverage.out ./... go tool cover -html=coverage.out -o coverage.html -# Run linter lint: golangci-lint run ./... -# Format code fmt: go fmt ./... -# Build Docker image docker-build: - docker build -t $(DOCKER_IMAGE):$(VERSION) . + docker build -t mat-k8s-controller:$(TAG) . -# Build Docker image with latest tag -docker-build-latest: docker-build - docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest +docker-build-latest: + docker build -t mat-k8s-controller:latest . -# Clean build artifacts clean: - rm -f $(BINARY) coverage.out coverage.html + rm -rf bin/ coverage.out coverage.html -# Run locally (requires kubeconfig and machine-a-tron running) run: - go run ./cmd/mat-k8s-controller --log-level debug - -# Verify dependencies -verify: - go mod verify - go mod tidy - @if [ -n "$$(git status --porcelain go.mod go.sum)" ]; then \ - echo "go.mod or go.sum changed after tidy"; \ - exit 1; \ - fi + go run ./cmd/mat-k8s-controller --kubeconfig=$(KUBECONFIG) --log-level=debug + +verify: lint test diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md index eadb30b3e4..a6cce0e135 100644 --- a/dev/k8s/machine-a-tron-controller/README.md +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -1,25 +1,14 @@ # Machine-a-tron Kubernetes Controller -A Kubernetes controller that auto-discovers machine-a-tron pods and creates Services for mock BMC endpoints. - -## Overview - -In multi-pod machine-a-tron deployments, each pod simulates different machines with unique BMC endpoints. This controller: - -1. **Discovers** all machine-a-tron bmc-mock Services in the namespace -2. **Polls** each pod's `/machines/status` API -3. **Creates** one Kubernetes Service per BMC with: - - Redfish TCP port (443 → internal listen port) - - IPMI UDP port (623 → internal listen port) when enabled -4. **Updates** Services when machine status changes -5. **Deletes** stale Services when machines disappear +Kubernetes controller that auto-discovers machine-a-tron pods and creates Services for mock BMC endpoints. ## Features -- **Auto-discovery**: Finds all machine-a-tron pods automatically -- **Multi-pod support**: Aggregates machines from all pods -- **Direct BMC IP as ClusterIP**: Services use BMC IP as ClusterIP (requires oobDhcpRelayAddress within K8s ServiceCIDR) -- **Automatic cleanup**: Removes Services for deleted machines +- Auto-discovers machine-a-tron pods via `machine-a-tron.nvidia.com/service=true` label +- Creates ClusterIP Services with BMC IP for each mock BMC +- Supports Redfish (TCP 443) ports +- Multi-pod deployments with pod-specific routing +- Automatic cleanup of stale Services ## Build @@ -32,96 +21,88 @@ kind load docker-image mat-k8s-controller:latest --name | Flag | Env Var | Default | Description | |------|---------|---------|-------------| -| `--namespace` | `NAMESPACE` | `nico-system` | Namespace for Services and discovery | +| `--namespace` | `NAMESPACE` | `nico-system` | Kubernetes namespace | +| `--discovery-selector` | `DISCOVERY_SELECTOR` | `machine-a-tron.nvidia.com/service=true` | Label selector for discovery | | `--sync-interval` | `SYNC_INTERVAL` | `30s` | Reconciliation interval | | `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector for Services | -| `--bmc-mock-port` | `BMC_MOCK_PORT` | `1266` | BMC mock service port | -| `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `true` | Skip TLS verification | +| `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `false` | Skip TLS verification (dev only) | | `--log-level` | `LOG_LEVEL` | `info` | Log level | +| `--bmc-mock-port` | `BMC_MOCK_PORT` | `1266` | BMC mock service port | +| `--kubeconfig` | `KUBECONFIG` | (empty) | Path to kubeconfig (dev only, uses in-cluster config if empty) | ## Helm Deployment -Enable in your values: +Enable in parent chart: ```yaml -nico-machine-a-tron: - mat-k8s-controller: - enabled: true - image: - repository: mat-k8s-controller - tag: latest - pullPolicy: Never - config: - logLevel: debug +mat-k8s-controller: + enabled: true + image: + pullPolicy: Never # For local images + config: + insecureSkipVerify: true # Only for dev with self-signed certs ``` ## Service Structure -Each created Service has: +Created Services have: **Labels:** - `app.kubernetes.io/managed-by: mat-k8s-controller` - `machine-a-tron.nvidia.com/mat-id: ` - `machine-a-tron.nvidia.com/machine-type: host|dpu` +- `nvidia-infra-controller/pod-name: ` (multi-pod) **Annotations:** -- `machine-a-tron.nvidia.com/bmc-ip: ` -- `machine-a-tron.nvidia.com/api-state: ` -- `machine-a-tron.nvidia.com/power-state: ` - -**Ports:** -- `redfish`: TCP 443 → targetPort (listen port) -- `ipmi`: UDP 623 → targetPort (listen port) - when IPMI enabled +- `machine-a-tron.nvidia.com/bmc-ip` +- `machine-a-tron.nvidia.com/api-state` +- `machine-a-tron.nvidia.com/power-state` +- `machine-a-tron.nvidia.com/hardware-type` -## Troubleshooting - -### No machine-a-tron instances discovered - -Check that machine-a-tron pods have bmc-mock Services: +## Development ```bash -kubectl -n nico-system get svc -l app.kubernetes.io/name=nico-machine-a-tron +make build +make test +make run KUBECONFIG=~/.kube/config ``` +## Troubleshooting + ### ClusterIP already allocated -The BMC IP is already used by another Service. Options: +BMC IP is outside ServiceCIDR or already in use. +**Solutions:** 1. Reserve a ServiceCIDR for machine-a-tron (K8s 1.29+) -2. Use a different oobDhcpRelayAddress range +2. Use a CIDR within the cluster's ServiceCIDR 3. Delete conflicting Services -## Development - -```bash -# Build -go build ./... - -# Test -go test ./... +### ClusterIP change detected -# Run locally -go run ./cmd/mat-k8s-controller --kubeconfig ~/.kube/config --log-level debug -``` +BMC IP changed but ClusterIP is immutable. Controller will delete and recreate the Service. ## Architecture ```mermaid flowchart LR - subgraph K8s[Kubernetes] - Controller[mat-k8s-controller] - subgraph MAT[machine-a-tron pods] - MAT0[mat-0-bmc-mock] - MAT1[mat-1-bmc-mock] - end - subgraph Services[Created Services] - Svc1[mat-bmc-host-xxx] - Svc2[mat-bmc-dpu-yyy] - end + subgraph MAT[machine-a-tron pods] + MAT0[mat-0/bmc-mock] + MAT1[mat-1/bmc-mock] + end + + subgraph Controller + Discovery[Service Discovery] + Reconciler[Reconciler] + end + + subgraph K8s[Kubernetes Services] + SVC1[mat-bmc-host-xxx] + SVC2[mat-bmc-dpu-yyy] end - Controller -->|discovers| MAT - Controller -->|polls /machines/status| MAT0 - Controller -->|polls /machines/status| MAT1 - Controller -->|creates/updates/deletes| Services + Discovery -->|discovers| MAT0 + Discovery -->|discovers| MAT1 + Reconciler -->|polls /machines/status| MAT + Reconciler -->|creates/updates/deletes| K8s ``` diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index 6d247ff515..606b21f7dd 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -37,11 +37,11 @@ func main() { syncInterval := flag.Duration("sync-interval", parseDurationOrDefault("SYNC_INTERVAL", 30*time.Second), "Interval between reconciliation passes") kubeconfig := flag.String("kubeconfig", os.Getenv("KUBECONFIG"), - "Path to kubeconfig (uses in-cluster config if empty)") + "Path to kubeconfig file (uses in-cluster config if empty, development only)") targetSelector := flag.String("target-selector", envOrDefault("TARGET_SELECTOR", "app.kubernetes.io/name=nico-machine-a-tron"), "Pod selector for Services (comma-separated key=value pairs)") - insecureSkipVerify := flag.Bool("insecure-skip-verify", envBoolOrDefault("INSECURE_SKIP_VERIFY", true), - "Skip TLS certificate verification (for self-signed certs)") + insecureSkipVerify := flag.Bool("insecure-skip-verify", envBoolOrDefault("INSECURE_SKIP_VERIFY", false), + "Skip TLS certificate verification (use only for development with self-signed certs)") logLevel := flag.String("log-level", envOrDefault("LOG_LEVEL", "info"), "Log level (debug, info, warn, error)") bmcMockPort := flag.Int("bmc-mock-port", envIntOrDefault("BMC_MOCK_PORT", 1266), @@ -97,7 +97,7 @@ func main() { // Create service builder builder := &controller.ServiceBuilder{ - Namespace: *namespace, + Namespace: *namespace, BaseSelector: selector, } @@ -206,7 +206,7 @@ func parseSelector(s string) map[string]string { for _, pair := range strings.Split(s, ",") { if kv := strings.SplitN(pair, "=", 2); len(kv) == 2 { - result[kv[0]] = kv[1] + result[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) } } return result diff --git a/dev/k8s/machine-a-tron-controller/go.mod b/dev/k8s/machine-a-tron-controller/go.mod index ccf7d728a1..4bba495f13 100644 --- a/dev/k8s/machine-a-tron-controller/go.mod +++ b/dev/k8s/machine-a-tron-controller/go.mod @@ -40,7 +40,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect golang.org/x/net v0.30.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect golang.org/x/text v0.19.0 // indirect diff --git a/dev/k8s/machine-a-tron-controller/go.sum b/dev/k8s/machine-a-tron-controller/go.sum index 922bf09481..476d95621c 100644 --- a/dev/k8s/machine-a-tron-controller/go.sum +++ b/dev/k8s/machine-a-tron-controller/go.sum @@ -104,8 +104,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go index 7ecee08873..17f756422d 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -184,9 +184,10 @@ func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatu // ServiceDiff represents changes between desired and existing services. type ServiceDiff struct { - Create []*corev1.Service - Update []*corev1.Service - Delete []string + Create []*corev1.Service + Update []*corev1.Service + Delete []string + Recreate []*corev1.Service // Services that need delete+create due to immutable field changes } // ComputeServiceDiff calculates what changes need to be made. @@ -208,6 +209,13 @@ func ComputeServiceDiff(desired []*corev1.Service, existing []*corev1.Service) S continue } + // Check if ClusterIP changed - this requires recreate since ClusterIP is immutable + if svc.Spec.ClusterIP != "" && existingSvc.Spec.ClusterIP != "" && + svc.Spec.ClusterIP != existingSvc.Spec.ClusterIP { + diff.Recreate = append(diff.Recreate, svc) + continue + } + if needsUpdate(svc, existingSvc) { svc.ResourceVersion = existingSvc.ResourceVersion if svc.Spec.ClusterIP == "" { @@ -246,17 +254,27 @@ func needsUpdate(desired, existing *corev1.Service) bool { } } + // Check for added or changed labels for k, v := range desired.Labels { if existing.Labels[k] != v { return true } } + // Check for removed labels + if len(desired.Labels) != len(existing.Labels) { + return true + } + // Check for added or changed annotations for k, v := range desired.Annotations { if existing.Annotations[k] != v { return true } } + // Check for removed annotations + if len(desired.Annotations) != len(existing.Annotations) { + return true + } // Check selector changes (important for multi-pod) for k, v := range desired.Spec.Selector { @@ -264,6 +282,9 @@ func needsUpdate(desired, existing *corev1.Service) bool { return true } } + if len(desired.Spec.Selector) != len(existing.Spec.Selector) { + return true + } return false } @@ -282,10 +303,11 @@ type K8sServiceClient interface { // ReconcileResult contains the result of a reconciliation. type ReconcileResult struct { - Created int - Updated int - Deleted int - Errors []error + Created int + Updated int + Deleted int + Recreated int + Errors []error } // Reconciler discovers machine-a-tron pods and reconciles Services. @@ -334,17 +356,20 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { // Collect all machines from all instances var allDesired []*corev1.Service + fetchFailed := false for _, instance := range instances { client, err := matclient.NewClient(instance.URL, r.clientOpts...) if err != nil { result.Errors = append(result.Errors, fmt.Errorf("creating client for %s: %w", instance.URL, err)) + fetchFailed = true continue } status, err := client.GetMachinesStatus(ctx) if err != nil { result.Errors = append(result.Errors, fmt.Errorf("fetching status from %s: %w", instance.URL, err)) + fetchFailed = true continue } @@ -368,6 +393,40 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { // Compute and apply diff diff := ComputeServiceDiff(allDesired, existing) + // Process deletes first (needed for recreate to work) + // Skip deletions if any fetch failed to prevent spurious Service removal + if fetchFailed { + r.logger.Warn().Msg("skipping deletions due to partial status-fetch failures") + } + for _, name := range diff.Delete { + if fetchFailed { + continue + } + if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { + result.Errors = append(result.Errors, fmt.Errorf("deleting service %s: %w", name, err)) + } else { + result.Deleted++ + } + } + + // Process recreates (delete then create for immutable field changes like ClusterIP) + // Skip recreates if any fetch failed to prevent spurious Service removal + for _, svc := range diff.Recreate { + if fetchFailed { + continue + } + if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, svc.Name); err != nil { + result.Errors = append(result.Errors, fmt.Errorf("deleting service %s for recreate: %w", svc.Name, err)) + continue + } + if err := r.k8sClient.Create(ctx, svc); err != nil { + result.Errors = append(result.Errors, fmt.Errorf("recreating service %s: %w", svc.Name, err)) + } else { + result.Recreated++ + } + } + + // Process creates for _, svc := range diff.Create { if err := r.k8sClient.Create(ctx, svc); err != nil { result.Errors = append(result.Errors, fmt.Errorf("creating service %s: %w", svc.Name, err)) @@ -376,6 +435,7 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { } } + // Process updates for _, svc := range diff.Update { if err := r.k8sClient.Update(ctx, svc); err != nil { result.Errors = append(result.Errors, fmt.Errorf("updating service %s: %w", svc.Name, err)) @@ -384,13 +444,5 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { } } - for _, name := range diff.Delete { - if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { - result.Errors = append(result.Errors, fmt.Errorf("deleting service %s: %w", name, err)) - } else { - result.Deleted++ - } - } - return result } diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go index b25ec6aa63..b9b9aac089 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go @@ -12,16 +12,25 @@ import ( "io" "net/http" "net/url" + "strings" "time" "github.com/rs/zerolog" ) +const ( + // maxResponseSize is the maximum allowed response body size (10 MB). + maxResponseSize = 10 * 1024 * 1024 + // maxErrorBodySize is the maximum error body size to include in error messages. + maxErrorBodySize = 1024 +) + // Client is an HTTP client for the machine-a-tron API. type Client struct { - baseURL string - httpClient *http.Client - logger zerolog.Logger + baseURL string + httpClient *http.Client + logger zerolog.Logger + insecureSkipVerify bool } // Option configures a Client. @@ -45,23 +54,38 @@ func WithLogger(logger zerolog.Logger) Option { // Use only for development with self-signed certificates. func WithInsecureSkipVerify() Option { return func(client *Client) { - transport := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // Intentional for dev/test with self-signed certs - }, - } - client.httpClient.Transport = transport + client.insecureSkipVerify = true } } // NewClient creates a new machine-a-tron API client. +// baseURL must be an absolute HTTP(S) URL with a host. func NewClient(baseURL string, opts ...Option) (*Client, error) { - if _, err := url.Parse(baseURL); err != nil { + parsed, err := url.Parse(baseURL) + if err != nil { return nil, fmt.Errorf("invalid base URL: %w", err) } + // Validate URL is absolute with proper scheme and host + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return nil, fmt.Errorf("invalid base URL: scheme must be http or https, got %q", parsed.Scheme) + } + if parsed.Host == "" { + return nil, fmt.Errorf("invalid base URL: missing host") + } + if parsed.RawQuery != "" { + return nil, fmt.Errorf("invalid base URL: query string not allowed") + } + if parsed.Fragment != "" { + return nil, fmt.Errorf("invalid base URL: fragment not allowed") + } + + // Normalize: rebuild from parsed components, trim trailing slash + parsed.Path = strings.TrimSuffix(parsed.Path, "/") + normalizedURL := parsed.String() + c := &Client{ - baseURL: baseURL, + baseURL: normalizedURL, httpClient: &http.Client{ Timeout: 30 * time.Second, }, @@ -72,9 +96,32 @@ func NewClient(baseURL string, opts ...Option) (*Client, error) { opt(c) } + // Apply insecure TLS after all options to preserve existing transport settings + if c.insecureSkipVerify { + c.applyInsecureTLS() + } + return c, nil } +// applyInsecureTLS configures the client to skip TLS verification while preserving +// existing transport settings. +func (c *Client) applyInsecureTLS() { + transport, ok := c.httpClient.Transport.(*http.Transport) + if !ok || transport == nil { + transport = http.DefaultTransport.(*http.Transport).Clone() + } else { + transport = transport.Clone() + } + + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.InsecureSkipVerify = true //nolint:gosec // Intentional for dev/test with self-signed certs + + c.httpClient.Transport = transport +} + // GetMachinesStatus fetches the current machine status from machine-a-tron. func (c *Client) GetMachinesStatus(ctx context.Context) (*MachinesStatusResponse, error) { reqURL := c.baseURL + "/machines/status" @@ -94,13 +141,16 @@ func (c *Client) GetMachinesStatus(ctx context.Context) (*MachinesStatusResponse } defer resp.Body.Close() + // Limit response body size to prevent memory exhaustion + limitedReader := io.LimitReader(resp.Body, maxResponseSize) + if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodySize)) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, truncateString(string(body), maxErrorBodySize)) } var result MachinesStatusResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + if err := json.NewDecoder(limitedReader).Decode(&result); err != nil { return nil, fmt.Errorf("decoding response: %w", err) } @@ -108,3 +158,11 @@ func (c *Client) GetMachinesStatus(ctx context.Context) (*MachinesStatusResponse return &result, nil } + +// truncateString truncates a string to maxLen and adds "..." if truncated. +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen-3] + "..." +} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl index 929925b427..eb448e8415 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl @@ -63,15 +63,23 @@ Namespace - inherited from parent release {{- end }} {{/* -Discovery selector - finds machine-a-tron bmc-mock Services +Discovery selector - finds machine-a-tron bmc-mock Services scoped to this release */}} {{- define "mat-k8s-controller.discoverySelector" -}} -{{- default "machine-a-tron.nvidia.com/service=true" .Values.config.discoverySelector }} +{{- if .Values.config.discoverySelector }} +{{- .Values.config.discoverySelector }} +{{- else }} +{{- printf "machine-a-tron.nvidia.com/service=true,app.kubernetes.io/instance=%s" .Release.Name }} +{{- end }} {{- end }} {{/* -Target selector - matches parent chart's machine-a-tron pods +Target selector - matches this release's machine-a-tron pods */}} {{- define "mat-k8s-controller.targetSelector" -}} -{{- default "app.kubernetes.io/name=nico-machine-a-tron" .Values.config.targetSelector }} +{{- if .Values.config.targetSelector }} +{{- .Values.config.targetSelector }} +{{- else }} +{{- printf "app.kubernetes.io/name=nico-machine-a-tron,app.kubernetes.io/instance=%s" .Release.Name }} +{{- end }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml index 7f60e8c0a0..4cb31cefc7 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -23,18 +23,19 @@ resources: ## Controller configuration config: ## Label selector for discovering machine-a-tron Services - ## Default: machine-a-tron.nvidia.com/service=true + ## Default: machine-a-tron.nvidia.com/service=true,app.kubernetes.io/instance= discoverySelector: "" ## Pod selector for created Services (routes traffic to mat pods) - ## Default: app.kubernetes.io/name=nico-machine-a-tron + ## Default: app.kubernetes.io/name=nico-machine-a-tron,app.kubernetes.io/instance= targetSelector: "" ## Reconciliation interval syncInterval: "30s" - ## Skip TLS certificate verification (for self-signed certs) - insecureSkipVerify: true + ## Skip TLS certificate verification + ## WARNING: Only enable for development with self-signed certificates + insecureSkipVerify: false ## Log level (debug, info, warn, error) logLevel: "info" diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index b5b11a80f1..ca7401233c 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -378,5 +378,5 @@ mat-k8s-controller: config: syncInterval: "30s" - insecureSkipVerify: true + insecureSkipVerify: false logLevel: "info" From 0c7d2e69dcf28084f669c3aa721778cd94a1a76a Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 00:13:48 -0600 Subject: [PATCH 09/17] feat(machine-a-tron-ctrl): image build --- .github/workflows/ci.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ff4df17e85..b4826942b1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -816,6 +816,32 @@ jobs: timeout_minutes: 60 secrets: inherit + build-mat-k8s-controller: + if: >- + ${{ + always() + && github.event_name != 'schedule' + && needs.prepare.result == 'success' + && needs.prepare.outputs.source_files_changed == 'true' + }} + needs: + - prepare + uses: ./.github/workflows/docker-build.yml + with: + dockerfile_path: dev/k8s/machine-a-tron-controller/Dockerfile + context_path: dev/k8s/machine-a-tron-controller + image_name: ${{ needs.prepare.outputs.image_registry }}/mat-k8s-controller + image_tag: ${{ needs.prepare.outputs.version }} + additional_tags: ${{ needs.prepare.outputs.image_registry }}/mat-k8s-controller:${{ needs.prepare.outputs.major_minor_version }}-latest + platforms: linux/amd64,linux/arm64 + runner: linux-amd64-cpu4 + push: ${{ needs.prepare.outputs.publish_images == 'true' }} + load: false + scan: true + tag_latest: false + timeout_minutes: 30 + secrets: inherit + test-release-container-services: if: >- ${{ @@ -1834,6 +1860,7 @@ jobs: - build-release-container-x86_64 - build-release-container-aarch64 - build-machine-a-tron + - build-mat-k8s-controller - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb @@ -1938,6 +1965,7 @@ jobs: - build-release-container-x86_64 - build-release-container-aarch64 - build-machine-a-tron + - build-mat-k8s-controller - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb From f460d8c88014dec794b33bdc05481400247095c3 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 00:21:02 -0600 Subject: [PATCH 10/17] fix(machine-a-tron-ctrl): dockerfile --- dev/k8s/machine-a-tron-controller/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/k8s/machine-a-tron-controller/Dockerfile b/dev/k8s/machine-a-tron-controller/Dockerfile index b85f782687..08a1497907 100644 --- a/dev/k8s/machine-a-tron-controller/Dockerfile +++ b/dev/k8s/machine-a-tron-controller/Dockerfile @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # Build stage -FROM golang:1.26.4-alpine@sha256:4cfbe7d6f07723e1efe21b4251ccf17d92a8c2a9e6f9e88d3f3f5026e5db8f7d AS builder +FROM golang:1.26.4-alpine@sha256:3ad57304ad93bbec8548a0437ad9e06a455660655d9af011d58b993f6f615648 AS builder WORKDIR /app @@ -13,7 +13,7 @@ COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o mat-k8s-controller ./cmd/mat-k8s-controller # Runtime stage -FROM gcr.io/distroless/static:nonroot@sha256:9ecc53c269509f63c69a266168b81a2f4c9d10c2f03e0d53dce6b8a2f9e1c1f1 +FROM gcr.io/distroless/static:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6 COPY --from=builder /app/mat-k8s-controller /mat-k8s-controller From 95f5118a64775dd506ed715a367187ac9db73f2e Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 00:36:46 -0600 Subject: [PATCH 11/17] fix(machine-a-tron-ctrl): pr review --- crates/machine-a-tron/README.md | 4 +-- dev/k8s/machine-a-tron-controller/README.md | 26 ++++++++++++------- .../cmd/mat-k8s-controller/main.go | 2 +- .../pkg/controller/controller.go | 20 +++++++------- .../pkg/controller/discovery.go | 2 +- helm/charts/nico-machine-a-tron/README.md | 2 +- .../mat-k8s-controller/templates/_helpers.tpl | 2 +- .../charts/mat-k8s-controller/values.yaml | 2 +- .../templates/bmc-services.yaml | 2 +- .../templates/service.yaml | 2 +- helm/charts/nico-machine-a-tron/values.yaml | 2 +- 11 files changed, 36 insertions(+), 30 deletions(-) diff --git a/crates/machine-a-tron/README.md b/crates/machine-a-tron/README.md index ace4a2bceb..2ed87de295 100644 --- a/crates/machine-a-tron/README.md +++ b/crates/machine-a-tron/README.md @@ -9,7 +9,7 @@ periodically report health network observations. ## Usage -``` +```text target/debug/machine-a-tron -h Usage: machine-a-tron [OPTIONS] --relay-address [NICO_API] @@ -120,4 +120,4 @@ setup with the appropriate overrides for redfish so that it will send all reques > for all libredfish calls. If you want to go back to running the TUI locally, you'll want to manually edit the > generated nico-api-site-config.toml and drop the `override_target_host` line. You may also want to edit the > `mat.toml` in the same directory and set the host_count to 0 so that the in-cluster machine-a-tron doesn't run any -> mock machines. \ No newline at end of file +> mock machines. diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md index a6cce0e135..13e8e001f7 100644 --- a/dev/k8s/machine-a-tron-controller/README.md +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -1,10 +1,12 @@ # Machine-a-tron Kubernetes Controller -Kubernetes controller that auto-discovers machine-a-tron pods and creates Services for mock BMC endpoints. +Kubernetes controller that auto-discovers machine-a-tron pods and creates +Services for mock BMC endpoints. ## Features -- Auto-discovers machine-a-tron pods via `machine-a-tron.nvidia.com/service=true` label +- Auto-discovers machine-a-tron pods via `nvidia-infra-controller/mat-service=true` + label - Creates ClusterIP Services with BMC IP for each mock BMC - Supports Redfish (TCP 443) ports - Multi-pod deployments with pod-specific routing @@ -22,7 +24,7 @@ kind load docker-image mat-k8s-controller:latest --name | Flag | Env Var | Default | Description | |------|---------|---------|-------------| | `--namespace` | `NAMESPACE` | `nico-system` | Kubernetes namespace | -| `--discovery-selector` | `DISCOVERY_SELECTOR` | `machine-a-tron.nvidia.com/service=true` | Label selector for discovery | +| `--discovery-selector` | `DISCOVERY_SELECTOR` | `nvidia-infra-controller/mat-service=true` | Label selector for discovery | | `--sync-interval` | `SYNC_INTERVAL` | `30s` | Reconciliation interval | | `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector for Services | | `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `false` | Skip TLS verification (dev only) | @@ -48,16 +50,18 @@ mat-k8s-controller: Created Services have: **Labels:** + - `app.kubernetes.io/managed-by: mat-k8s-controller` -- `machine-a-tron.nvidia.com/mat-id: ` -- `machine-a-tron.nvidia.com/machine-type: host|dpu` +- `nvidia-infra-controller/mat-id: ` +- `nvidia-infra-controller/mat-machine-type: host|dpu` - `nvidia-infra-controller/pod-name: ` (multi-pod) **Annotations:** -- `machine-a-tron.nvidia.com/bmc-ip` -- `machine-a-tron.nvidia.com/api-state` -- `machine-a-tron.nvidia.com/power-state` -- `machine-a-tron.nvidia.com/hardware-type` + +- `nvidia-infra-controller/mat-bmc-ip` +- `nvidia-infra-controller/mat-api-state` +- `nvidia-infra-controller/mat-power-state` +- `nvidia-infra-controller/mat-hardware-type` ## Development @@ -74,13 +78,15 @@ make run KUBECONFIG=~/.kube/config BMC IP is outside ServiceCIDR or already in use. **Solutions:** + 1. Reserve a ServiceCIDR for machine-a-tron (K8s 1.29+) 2. Use a CIDR within the cluster's ServiceCIDR 3. Delete conflicting Services ### ClusterIP change detected -BMC IP changed but ClusterIP is immutable. Controller will delete and recreate the Service. +BMC IP changed but ClusterIP is immutable. Controller will delete and recreate +the Service. ## Architecture diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index 606b21f7dd..096dff2299 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -32,7 +32,7 @@ func main() { // Flags namespace := flag.String("namespace", envOrDefault("NAMESPACE", "nico-system"), "Kubernetes namespace for Services and machine-a-tron discovery") - discoverySelector := flag.String("discovery-selector", envOrDefault("DISCOVERY_SELECTOR", "machine-a-tron.nvidia.com/service=true"), + discoverySelector := flag.String("discovery-selector", envOrDefault("DISCOVERY_SELECTOR", "nvidia-infra-controller/mat-service=true"), "Label selector for discovering machine-a-tron bmc-mock Services") syncInterval := flag.Duration("sync-interval", parseDurationOrDefault("SYNC_INTERVAL", 30*time.Second), "Interval between reconciliation passes") diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go index 17f756422d..c6bbf296b0 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -28,26 +28,26 @@ const ( LabelPodName = "nvidia-infra-controller/pod-name" // LabelMatID is the machine-a-tron ID label. - LabelMatID = "machine-a-tron.nvidia.com/mat-id" + LabelMatID = "nvidia-infra-controller/mat-id" // LabelMachineID is the NICo machine ID label. - LabelMachineID = "machine-a-tron.nvidia.com/machine-id" + LabelMachineID = "nvidia-infra-controller/mat-machine-id" // LabelMachineType indicates if this is a host or DPU. - LabelMachineType = "machine-a-tron.nvidia.com/machine-type" + LabelMachineType = "nvidia-infra-controller/mat-machine-type" // LabelParentMatID is the parent host's mat-id for DPU services. - LabelParentMatID = "machine-a-tron.nvidia.com/parent-mat-id" + LabelParentMatID = "nvidia-infra-controller/mat-parent-id" // AnnotationBMCIP stores the BMC IP address. - AnnotationBMCIP = "machine-a-tron.nvidia.com/bmc-ip" + AnnotationBMCIP = "nvidia-infra-controller/mat-bmc-ip" // AnnotationAPIState stores the machine's API state. - AnnotationAPIState = "machine-a-tron.nvidia.com/api-state" + AnnotationAPIState = "nvidia-infra-controller/mat-api-state" // AnnotationPowerState stores the machine's power state. - AnnotationPowerState = "machine-a-tron.nvidia.com/power-state" + AnnotationPowerState = "nvidia-infra-controller/mat-power-state" // AnnotationHardwareType stores the hardware type. - AnnotationHardwareType = "machine-a-tron.nvidia.com/hardware-type" + AnnotationHardwareType = "nvidia-infra-controller/mat-hardware-type" // AnnotationRedfishListenPort stores the internal Redfish listen port. - AnnotationRedfishListenPort = "machine-a-tron.nvidia.com/redfish-listen-port" + AnnotationRedfishListenPort = "nvidia-infra-controller/mat-redfish-listen-port" // AnnotationIPMIListenPort stores the internal IPMI listen port. - AnnotationIPMIListenPort = "machine-a-tron.nvidia.com/ipmi-listen-port" + AnnotationIPMIListenPort = "nvidia-infra-controller/mat-ipmi-listen-port" // MachineTypeHost indicates a host machine. MachineTypeHost = "host" diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go index f47e931ca2..152e0502b4 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/discovery.go @@ -13,7 +13,7 @@ import ( // DefaultDiscoverySelector is the default label selector for discovering // machine-a-tron bmc-mock Services. -const DefaultDiscoverySelector = "machine-a-tron.nvidia.com/service=true" +const DefaultDiscoverySelector = "nvidia-infra-controller/mat-service=true" // DiscoveredInstance represents a discovered machine-a-tron instance. type DiscoveredInstance struct { diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 933d1c4042..4fb8cf0b70 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -98,7 +98,7 @@ pods: compute: hwType: wiwynn_gb200_nvl hostCount: 252 # 18 trays × 14 racks - dpuPerHostCount: 2 # 2 BF3 per tray → 504 DPU BMCs + dpuPerHostCount: 2 # 2 BF3 per tray -> 504 DPU BMCs oobDhcpRelayAddress: "10.100.0.1" switches: hwType: nvidia_switch_nd5200_ld diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl index eb448e8415..8ffc14cc89 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl @@ -69,7 +69,7 @@ Discovery selector - finds machine-a-tron bmc-mock Services scoped to this relea {{- if .Values.config.discoverySelector }} {{- .Values.config.discoverySelector }} {{- else }} -{{- printf "machine-a-tron.nvidia.com/service=true,app.kubernetes.io/instance=%s" .Release.Name }} +{{- printf "nvidia-infra-controller/mat-service=true,app.kubernetes.io/instance=%s" .Release.Name }} {{- end }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml index 4cb31cefc7..8b40ccf085 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -23,7 +23,7 @@ resources: ## Controller configuration config: ## Label selector for discovering machine-a-tron Services - ## Default: machine-a-tron.nvidia.com/service=true,app.kubernetes.io/instance= + ## Default: nvidia-infra-controller/mat-service=true,app.kubernetes.io/instance= discoverySelector: "" ## Pod selector for created Services (routes traffic to mat pods) diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml index 369359d347..6bbc29db89 100644 --- a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml +++ b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml @@ -107,7 +107,7 @@ metadata: labels: {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} app.kubernetes.io/component: bmc-service - nvidia-infra-controller/bmc-ip: "{{ $currentIP }}" + nvidia-infra-controller/mat-bmc-ip: "{{ $currentIP }}" nvidia-infra-controller/bmc-index: "{{ $bmcIndex }}" nvidia-infra-controller/pod-name: {{ $podName | quote }} annotations: diff --git a/helm/charts/nico-machine-a-tron/templates/service.yaml b/helm/charts/nico-machine-a-tron/templates/service.yaml index 9c15ea3f07..ffbf9bb2b0 100644 --- a/helm/charts/nico-machine-a-tron/templates/service.yaml +++ b/helm/charts/nico-machine-a-tron/templates/service.yaml @@ -45,7 +45,7 @@ metadata: labels: {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} app: {{ $baseName }} - machine-a-tron.nvidia.com/service: "true" + nvidia-infra-controller/mat-service: "true" {{- if gt $activePods 1 }} nvidia-infra-controller/pod-name: {{ $podName | quote }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index ca7401233c..ae24b88d63 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -362,7 +362,7 @@ configFiles: ## Enable for multi-pod deployments to dynamically manage BMC Services. ## ## Auto-discovered: -## - Finds bmc-mock Services via label: machine-a-tron.nvidia.com/service=true +## - Finds bmc-mock Services via label: nvidia-infra-controller/mat-service=true ## - Polls /machines/status from each discovered instance ## ## Note: BMC IP must be within Kubernetes ServiceCIDR (default 10.96.0.0/12) From 02fbec4bc83496c2ac980a20db8463f047ef8090 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 10:58:16 -0600 Subject: [PATCH 12/17] fix(machine-a-tron-ctrl): pr review --- .github/workflows/ci.yaml | 2 ++ dev/k8s/machine-a-tron-controller/Makefile | 4 +++- dev/k8s/machine-a-tron-controller/README.md | 2 +- .../cmd/mat-k8s-controller/main.go | 7 +++++++ dev/k8s/machine-a-tron-controller/pkg/matclient/client.go | 1 + 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b4826942b1..5c7adc0648 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2045,6 +2045,8 @@ jobs: - security-secret-scan - lint-police - check-rest-core-proto-sync + - build-machine-a-tron + - build-mat-k8s-controller steps: - name: Decide pass/fail env: diff --git a/dev/k8s/machine-a-tron-controller/Makefile b/dev/k8s/machine-a-tron-controller/Makefile index 07a7eb5e79..818dc02b43 100644 --- a/dev/k8s/machine-a-tron-controller/Makefile +++ b/dev/k8s/machine-a-tron-controller/Makefile @@ -1,10 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -.PHONY: build test test-coverage lint fmt docker-build docker-build-latest clean run verify +.PHONY: all build test test-coverage lint fmt docker-build docker-build-latest clean run verify TAG ?= dev +all: build + build: @mkdir -p bin go build -o bin/mat-k8s-controller ./cmd/mat-k8s-controller diff --git a/dev/k8s/machine-a-tron-controller/README.md b/dev/k8s/machine-a-tron-controller/README.md index 13e8e001f7..9d6eabf62c 100644 --- a/dev/k8s/machine-a-tron-controller/README.md +++ b/dev/k8s/machine-a-tron-controller/README.md @@ -68,7 +68,7 @@ Created Services have: ```bash make build make test -make run KUBECONFIG=~/.kube/config +make run KUBECONFIG="$HOME/.kube/config" ``` ## Troubleshooting diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index 096dff2299..b75aa3563d 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -12,6 +12,7 @@ package main import ( "context" "flag" + "fmt" "os" "os/signal" "strconv" @@ -49,6 +50,12 @@ func main() { flag.Parse() + // Validate sync interval + if *syncInterval <= 0 { + fmt.Fprintf(os.Stderr, "error: sync-interval must be positive, got %v\n", *syncInterval) + os.Exit(1) + } + // Setup logger level, err := zerolog.ParseLevel(*logLevel) if err != nil { diff --git a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go index b9b9aac089..8aec987a47 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go +++ b/dev/k8s/machine-a-tron-controller/pkg/matclient/client.go @@ -118,6 +118,7 @@ func (c *Client) applyInsecureTLS() { transport.TLSClientConfig = &tls.Config{} } transport.TLSClientConfig.InsecureSkipVerify = true //nolint:gosec // Intentional for dev/test with self-signed certs + transport.TLSClientConfig.MinVersion = tls.VersionTLS12 c.httpClient.Transport = transport } From 5068f1bed4e540c0b51aa248da710df1d8119756 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 11:13:44 -0600 Subject: [PATCH 13/17] fix(machine-a-tron-ctrl): pr review --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5c7adc0648..a2e8646484 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -171,6 +171,7 @@ jobs: - 'Cargo.lock' - 'Makefile.toml' - 'rust-toolchain.toml' + - 'dev/k8s/machine-a-tron-controller/**' - name: Calculate version id: version run: | From c6f79b9d982a57c0920136e6846058ece100a987 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 11:41:26 -0600 Subject: [PATCH 14/17] fix(machine-a-tron-ctrl): services and target selectore --- .../mat-k8s-controller/templates/_helpers.tpl | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl index 8ffc14cc89..b23e8eea21 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/templates/_helpers.tpl @@ -63,23 +63,15 @@ Namespace - inherited from parent release {{- end }} {{/* -Discovery selector - finds machine-a-tron bmc-mock Services scoped to this release +Discovery selector - finds machine-a-tron bmc-mock Services */}} {{- define "mat-k8s-controller.discoverySelector" -}} -{{- if .Values.config.discoverySelector }} -{{- .Values.config.discoverySelector }} -{{- else }} -{{- printf "nvidia-infra-controller/mat-service=true,app.kubernetes.io/instance=%s" .Release.Name }} -{{- end }} +{{- default "nvidia-infra-controller/mat-service=true" .Values.config.discoverySelector }} {{- end }} {{/* -Target selector - matches this release's machine-a-tron pods +Target selector - matches machine-a-tron pods */}} {{- define "mat-k8s-controller.targetSelector" -}} -{{- if .Values.config.targetSelector }} -{{- .Values.config.targetSelector }} -{{- else }} -{{- printf "app.kubernetes.io/name=nico-machine-a-tron,app.kubernetes.io/instance=%s" .Release.Name }} -{{- end }} +{{- default "app.kubernetes.io/name=nico-machine-a-tron" .Values.config.targetSelector }} {{- end }} From 188faa684ba9e00296e8f81a931bbe6180fd396f Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 11:51:32 -0600 Subject: [PATCH 15/17] fix(machine-a-tron-ctrl): service short name --- .../machine-a-tron-controller/pkg/controller/controller.go | 4 ++-- .../pkg/controller/controller_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go index c6bbf296b0..d60f76192a 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -71,8 +71,8 @@ type ServiceBuilder struct { // BuildServiceName generates a consistent service name for a machine. func BuildServiceName(machineType, matID string) string { shortID := matID - if len(matID) > 8 { - shortID = matID[:8] + if len(matID) > 12 { + shortID = matID[:12] } return fmt.Sprintf("mat-bmc-%s-%s", machineType, shortID) } diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go index ad2274f722..fca963f730 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller_test.go @@ -25,12 +25,12 @@ func TestBuildServiceName(t *testing.T) { { machineType: MachineTypeHost, matID: "12345678-1234-1234-1234-123456789abc", - want: "mat-bmc-host-12345678", + want: "mat-bmc-host-12345678-123", }, { machineType: MachineTypeDPU, matID: "abcdefgh-1234-1234-1234-123456789abc", - want: "mat-bmc-dpu-abcdefgh", + want: "mat-bmc-dpu-abcdefgh-123", }, { machineType: MachineTypeHost, @@ -73,7 +73,7 @@ func TestServiceBuilder_BuildService(t *testing.T) { svc := builder.BuildService(machine, MachineTypeHost, "", "") // Check basic metadata - assert.Equal(t, "mat-bmc-host-host-uui", svc.Name) + assert.Equal(t, "mat-bmc-host-host-uuid-12", svc.Name) assert.Equal(t, "test-ns", svc.Namespace) // Check labels From 31e512620966cd8610a2d7e2a31a7263b1b1fef4 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 13:45:30 -0600 Subject: [PATCH 16/17] feat(machine-a-tron-ctrl): add concurrency and k8s limits --- .../pkg/controller/controller.go | 437 ++++++++++++------ .../charts/mat-k8s-controller/values.yaml | 8 +- 2 files changed, 300 insertions(+), 145 deletions(-) diff --git a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go index d60f76192a..3bfcbd9139 100644 --- a/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go +++ b/dev/k8s/machine-a-tron-controller/pkg/controller/controller.go @@ -9,6 +9,8 @@ import ( "context" "fmt" "strconv" + "sync" + "sync/atomic" "github.com/rs/zerolog" corev1 "k8s.io/api/core/v1" @@ -31,40 +33,41 @@ const ( LabelMatID = "nvidia-infra-controller/mat-id" // LabelMachineID is the NICo machine ID label. LabelMachineID = "nvidia-infra-controller/mat-machine-id" - // LabelMachineType indicates if this is a host or DPU. + // LabelMachineType distinguishes host vs dpu. LabelMachineType = "nvidia-infra-controller/mat-machine-type" - // LabelParentMatID is the parent host's mat-id for DPU services. + // LabelParentMatID links DPUs to their parent host. LabelParentMatID = "nvidia-infra-controller/mat-parent-id" - // AnnotationBMCIP stores the BMC IP address. + // AnnotationBMCIP is the BMC IP address annotation. AnnotationBMCIP = "nvidia-infra-controller/mat-bmc-ip" - // AnnotationAPIState stores the machine's API state. + // AnnotationAPIState is the API state annotation. AnnotationAPIState = "nvidia-infra-controller/mat-api-state" - // AnnotationPowerState stores the machine's power state. + // AnnotationPowerState is the power state annotation. AnnotationPowerState = "nvidia-infra-controller/mat-power-state" - // AnnotationHardwareType stores the hardware type. + // AnnotationHardwareType is the hardware type annotation. AnnotationHardwareType = "nvidia-infra-controller/mat-hardware-type" - // AnnotationRedfishListenPort stores the internal Redfish listen port. + // AnnotationRedfishListenPort is the Redfish listen port annotation. AnnotationRedfishListenPort = "nvidia-infra-controller/mat-redfish-listen-port" - // AnnotationIPMIListenPort stores the internal IPMI listen port. + // AnnotationIPMIListenPort is the IPMI listen port annotation. AnnotationIPMIListenPort = "nvidia-infra-controller/mat-ipmi-listen-port" - // MachineTypeHost indicates a host machine. + // MachineTypeHost is the machine type for hosts. MachineTypeHost = "host" - // MachineTypeDPU indicates a DPU. + // MachineTypeDPU is the machine type for DPUs. MachineTypeDPU = "dpu" - // PortNameRedfish is the name of the Redfish port in the Service. + // PortNameRedfish is the name of the Redfish port. PortNameRedfish = "redfish" - // PortNameIPMI is the name of the IPMI port in the Service. + // PortNameIPMI is the name of the IPMI port. PortNameIPMI = "ipmi" + + // DefaultConcurrency is the default number of concurrent workers for K8s API calls. + DefaultConcurrency = 50 ) // ServiceBuilder builds Kubernetes Services from machine status. type ServiceBuilder struct { - // Namespace is the target namespace for Services. - Namespace string - // BaseSelector is the base pod selector (e.g., app.kubernetes.io/name=nico-machine-a-tron). + Namespace string BaseSelector map[string]string } @@ -87,21 +90,17 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT LabelMatID: machine.MatID, LabelMachineType: machineType, } - - annotations := map[string]string{ - AnnotationAPIState: machine.APIState, - AnnotationPowerState: machine.PowerState, - AnnotationRedfishListenPort: strconv.Itoa(int(machine.BMC.Redfish.ListenPort)), - } - if machine.MachineID != nil { labels[LabelMachineID] = *machine.MachineID } if parentMatID != "" { labels[LabelParentMatID] = parentMatID } - if podName != "" { - labels[LabelPodName] = podName + + annotations := map[string]string{ + AnnotationAPIState: machine.APIState, + AnnotationPowerState: machine.PowerState, + AnnotationRedfishListenPort: strconv.Itoa(int(machine.BMC.Redfish.ListenPort)), } if machine.BMC.IP != nil { annotations[AnnotationBMCIP] = *machine.BMC.IP @@ -109,9 +108,6 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT if machine.HardwareType != nil { annotations[AnnotationHardwareType] = *machine.HardwareType } - if machine.BMC.IPMI != nil { - annotations[AnnotationIPMIListenPort] = strconv.Itoa(int(machine.BMC.IPMI.ListenPort)) - } ports := []corev1.ServicePort{ { @@ -122,6 +118,7 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT }, } + // Add IPMI port if available if machine.BMC.IPMI != nil { ports = append(ports, corev1.ServicePort{ Name: PortNameIPMI, @@ -129,9 +126,10 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT Port: int32(machine.BMC.IPMI.ReachablePort), TargetPort: intstr.FromInt32(int32(machine.BMC.IPMI.ListenPort)), }) + annotations[AnnotationIPMIListenPort] = strconv.Itoa(int(machine.BMC.IPMI.ListenPort)) } - // Build selector: base selector + pod-specific selector for multi-pod + // Build selector - include pod name for multi-pod deployments selector := make(map[string]string) for k, v := range b.BaseSelector { selector[k] = v @@ -154,7 +152,7 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT }, } - // Use BMC IP directly as ClusterIP + // Set ClusterIP to BMC IP for direct addressing if machine.BMC.IP != nil { svc.Spec.ClusterIP = *machine.BMC.IP } @@ -162,19 +160,19 @@ func (b *ServiceBuilder) BuildService(machine *matclient.MachineStatus, machineT return svc } -// BuildServicesFromStatus generates all Services from a machines status response. -// podName identifies which machine-a-tron pod these machines belong to. +// BuildServicesFromStatus builds Services for all machines in the status response. +// podName is used to create pod-specific selectors for multi-pod deployments. func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatusResponse, podName string) []*corev1.Service { var services []*corev1.Service - for i := range status.Machines { - machine := &status.Machines[i] - svc := b.BuildService(machine, MachineTypeHost, "", podName) + for _, machine := range status.Machines { + // Build service for the host + svc := b.BuildService(&machine, MachineTypeHost, "", podName) services = append(services, svc) - for j := range machine.DPUs { - dpu := &machine.DPUs[j] - dpuSvc := b.BuildService(dpu, MachineTypeDPU, machine.MatID, podName) + // Build services for DPUs + for _, dpu := range machine.DPUs { + dpuSvc := b.BuildService(&dpu, MachineTypeDPU, machine.MatID, podName) services = append(services, dpuSvc) } } @@ -182,118 +180,122 @@ func (b *ServiceBuilder) BuildServicesFromStatus(status *matclient.MachinesStatu return services } -// ServiceDiff represents changes between desired and existing services. +// ServiceDiff represents the differences between desired and existing services. type ServiceDiff struct { - Create []*corev1.Service - Update []*corev1.Service - Delete []string + Create []*corev1.Service + Update []*corev1.Service Recreate []*corev1.Service // Services that need delete+create due to immutable field changes + Delete []string } -// ComputeServiceDiff calculates what changes need to be made. +// ComputeServiceDiff calculates the differences between desired and existing services. func ComputeServiceDiff(desired []*corev1.Service, existing []*corev1.Service) ServiceDiff { - var diff ServiceDiff + diff := ServiceDiff{} - existingByName := make(map[string]*corev1.Service) + existingMap := make(map[string]*corev1.Service) for _, svc := range existing { - existingByName[svc.Name] = svc + existingMap[svc.Name] = svc } - desiredNames := make(map[string]bool) + desiredMap := make(map[string]*corev1.Service) for _, svc := range desired { - desiredNames[svc.Name] = true + desiredMap[svc.Name] = svc + } - existingSvc, exists := existingByName[svc.Name] + // Find services to create or update + for _, svc := range desired { + existingSvc, exists := existingMap[svc.Name] if !exists { diff.Create = append(diff.Create, svc) - continue - } - - // Check if ClusterIP changed - this requires recreate since ClusterIP is immutable - if svc.Spec.ClusterIP != "" && existingSvc.Spec.ClusterIP != "" && - svc.Spec.ClusterIP != existingSvc.Spec.ClusterIP { - diff.Recreate = append(diff.Recreate, svc) - continue - } - - if needsUpdate(svc, existingSvc) { + } else if needsUpdate(svc, existingSvc) { svc.ResourceVersion = existingSvc.ResourceVersion - if svc.Spec.ClusterIP == "" { - svc.Spec.ClusterIP = existingSvc.Spec.ClusterIP + // Check if ClusterIP is changing (immutable field) + if svc.Spec.ClusterIP != "" && existingSvc.Spec.ClusterIP != "" && + svc.Spec.ClusterIP != existingSvc.Spec.ClusterIP { + // ClusterIP changed - need to delete and recreate + diff.Recreate = append(diff.Recreate, svc) + } else { + // Preserve existing ClusterIP if not explicitly set + if svc.Spec.ClusterIP == "" { + svc.Spec.ClusterIP = existingSvc.Spec.ClusterIP + } + diff.Update = append(diff.Update, svc) } - diff.Update = append(diff.Update, svc) } } - for name, svc := range existingByName { - if !desiredNames[name] && isManagedByController(svc) { - diff.Delete = append(diff.Delete, name) + // Find services to delete (managed by us but no longer desired) + for _, existing := range existing { + if _, wanted := desiredMap[existing.Name]; !wanted { + // Only delete if we manage this service + if existing.Labels[LabelManagedBy] == LabelManagedByValue { + diff.Delete = append(diff.Delete, existing.Name) + } } } return diff } +// needsUpdate checks if a service needs to be updated. func needsUpdate(desired, existing *corev1.Service) bool { + // Check ports if len(desired.Spec.Ports) != len(existing.Spec.Ports) { return true } - - existingPorts := make(map[string]corev1.ServicePort) - for _, p := range existing.Spec.Ports { - existingPorts[p.Name] = p - } - - for _, dp := range desired.Spec.Ports { - ep, ok := existingPorts[dp.Name] - if !ok { + for i, port := range desired.Spec.Ports { + if i >= len(existing.Spec.Ports) { return true } - if dp.Port != ep.Port || dp.TargetPort != ep.TargetPort || dp.Protocol != ep.Protocol { + existingPort := existing.Spec.Ports[i] + if port.Name != existingPort.Name || + port.Port != existingPort.Port || + port.Protocol != existingPort.Protocol || + port.TargetPort.IntValue() != existingPort.TargetPort.IntValue() { return true } } - // Check for added or changed labels - for k, v := range desired.Labels { - if existing.Labels[k] != v { + // Check selector + if len(desired.Spec.Selector) != len(existing.Spec.Selector) { + return true + } + for k, v := range desired.Spec.Selector { + if existing.Spec.Selector[k] != v { return true } } - // Check for removed labels + + // Check labels if len(desired.Labels) != len(existing.Labels) { return true } - - // Check for added or changed annotations - for k, v := range desired.Annotations { - if existing.Annotations[k] != v { + for k, v := range desired.Labels { + if existing.Labels[k] != v { return true } } - // Check for removed annotations + + // Check annotations if len(desired.Annotations) != len(existing.Annotations) { return true } - - // Check selector changes (important for multi-pod) - for k, v := range desired.Spec.Selector { - if existing.Spec.Selector[k] != v { + for k, v := range desired.Annotations { + if existing.Annotations[k] != v { return true } } - if len(desired.Spec.Selector) != len(existing.Spec.Selector) { + + // Check ClusterIP change + if desired.Spec.ClusterIP != "" && existing.Spec.ClusterIP != "" && + desired.Spec.ClusterIP != existing.Spec.ClusterIP { return true } return false } -func isManagedByController(svc *corev1.Service) bool { - return svc.Labels[LabelManagedBy] == LabelManagedByValue -} - -// K8sServiceClient is an interface for Kubernetes Service operations. +// K8sServiceClient defines the interface for Kubernetes service operations. type K8sServiceClient interface { List(ctx context.Context, namespace string, labelSelector string) ([]*corev1.Service, error) Create(ctx context.Context, svc *corev1.Service) error @@ -301,7 +303,7 @@ type K8sServiceClient interface { Delete(ctx context.Context, namespace, name string) error } -// ReconcileResult contains the result of a reconciliation. +// ReconcileResult holds the results of a reconciliation cycle. type ReconcileResult struct { Created int Updated int @@ -310,40 +312,49 @@ type ReconcileResult struct { Errors []error } -// Reconciler discovers machine-a-tron pods and reconciles Services. +// Reconciler reconciles Kubernetes Services with machine-a-tron machine status. type Reconciler struct { discovery *MatPodDiscovery serviceBuilder *ServiceBuilder k8sClient K8sServiceClient clientOpts []matclient.Option logger zerolog.Logger + concurrency int } // NewReconciler creates a new Reconciler. func NewReconciler( discovery *MatPodDiscovery, - builder *ServiceBuilder, + serviceBuilder *ServiceBuilder, k8sClient K8sServiceClient, clientOpts []matclient.Option, logger zerolog.Logger, ) *Reconciler { return &Reconciler{ discovery: discovery, - serviceBuilder: builder, + serviceBuilder: serviceBuilder, k8sClient: k8sClient, clientOpts: clientOpts, logger: logger, + concurrency: DefaultConcurrency, } } -// Reconcile performs a reconciliation pass across all discovered machine-a-tron instances. +// SetConcurrency sets the number of concurrent workers for K8s API calls. +func (r *Reconciler) SetConcurrency(n int) { + if n > 0 { + r.concurrency = n + } +} + +// Reconcile performs a full reconciliation cycle. func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { result := ReconcileResult{} // Discover machine-a-tron instances instances, err := r.discovery.Discover(ctx) if err != nil { - result.Errors = append(result.Errors, fmt.Errorf("discovering machine-a-tron instances: %w", err)) + result.Errors = append(result.Errors, fmt.Errorf("discovering instances: %w", err)) return result } @@ -352,9 +363,11 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { return result } - r.logger.Debug().Int("count", len(instances)).Msg("discovered machine-a-tron instances") + r.logger.Debug(). + Int("count", len(instances)). + Msg("discovered machine-a-tron instances") - // Collect all machines from all instances + // Collect all desired services from all instances var allDesired []*corev1.Service fetchFailed := false @@ -366,6 +379,10 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { continue } + r.logger.Debug(). + Str("url", instance.URL). + Msg("fetching machine status") + status, err := client.GetMachinesStatus(ctx) if err != nil { result.Errors = append(result.Errors, fmt.Errorf("fetching status from %s: %w", instance.URL, err)) @@ -373,18 +390,23 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { continue } - services := r.serviceBuilder.BuildServicesFromStatus(status, instance.PodName) r.logger.Debug(). Str("url", instance.URL). Str("pod", instance.PodName). - Int("machines", len(services)). - Msg("fetched machines from instance") + Int("machines", len(status.Machines)). + Msg("fetched machine status") + + services := r.serviceBuilder.BuildServicesFromStatus(status, instance.PodName) allDesired = append(allDesired, services...) } - // List existing managed services - selector := fmt.Sprintf("%s=%s", LabelManagedBy, LabelManagedByValue) - existing, err := r.k8sClient.List(ctx, r.serviceBuilder.Namespace, selector) + r.logger.Info(). + Int("total_services", len(allDesired)). + Msg("built desired services from all instances") + + // List existing services + existing, err := r.k8sClient.List(ctx, r.serviceBuilder.Namespace, + fmt.Sprintf("%s=%s", LabelManagedBy, LabelManagedByValue)) if err != nil { result.Errors = append(result.Errors, fmt.Errorf("listing existing services: %w", err)) return result @@ -393,56 +415,189 @@ func (r *Reconciler) Reconcile(ctx context.Context) ReconcileResult { // Compute and apply diff diff := ComputeServiceDiff(allDesired, existing) + r.logger.Info(). + Int("create", len(diff.Create)). + Int("update", len(diff.Update)). + Int("delete", len(diff.Delete)). + Int("recreate", len(diff.Recreate)). + Msg("computed service diff") + // Process deletes first (needed for recreate to work) // Skip deletions if any fetch failed to prevent spurious Service removal if fetchFailed { r.logger.Warn().Msg("skipping deletions due to partial status-fetch failures") } - for _, name := range diff.Delete { - if fetchFailed { - continue - } - if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { - result.Errors = append(result.Errors, fmt.Errorf("deleting service %s: %w", name, err)) - } else { - result.Deleted++ - } + + // Process deletes concurrently + if !fetchFailed && len(diff.Delete) > 0 { + deleted := r.processDeletesConcurrently(ctx, diff.Delete) + result.Deleted = deleted } // Process recreates (delete then create for immutable field changes like ClusterIP) // Skip recreates if any fetch failed to prevent spurious Service removal - for _, svc := range diff.Recreate { - if fetchFailed { - continue - } - if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, svc.Name); err != nil { - result.Errors = append(result.Errors, fmt.Errorf("deleting service %s for recreate: %w", svc.Name, err)) - continue + if !fetchFailed && len(diff.Recreate) > 0 { + recreated := r.processRecreatesConcurrently(ctx, diff.Recreate, &result) + result.Recreated = recreated + } + + // Process creates concurrently + if len(diff.Create) > 0 { + created := r.processCreatesConcurrently(ctx, diff.Create, &result) + result.Created = created + } + + // Process updates concurrently + if len(diff.Update) > 0 { + updated := r.processUpdatesConcurrently(ctx, diff.Update, &result) + result.Updated = updated + } + + return result +} + +// processDeletesConcurrently deletes services using a worker pool. +func (r *Reconciler) processDeletesConcurrently(ctx context.Context, names []string) int { + var deleted int64 + var wg sync.WaitGroup + sem := make(chan struct{}, r.concurrency) + + for i, name := range names { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(names)). + Msg("delete progress") } - if err := r.k8sClient.Create(ctx, svc); err != nil { - result.Errors = append(result.Errors, fmt.Errorf("recreating service %s: %w", svc.Name, err)) - } else { - result.Recreated++ + + wg.Add(1) + sem <- struct{}{} + + go func(name string) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, name); err != nil { + r.logger.Error().Err(err).Str("service", name).Msg("failed to delete service") + } else { + atomic.AddInt64(&deleted, 1) + } + }(name) + } + + wg.Wait() + return int(deleted) +} + +// processRecreatesConcurrently handles services that need delete+create. +func (r *Reconciler) processRecreatesConcurrently(ctx context.Context, services []*corev1.Service, result *ReconcileResult) int { + var recreated int64 + var wg sync.WaitGroup + sem := make(chan struct{}, r.concurrency) + + for i, svc := range services { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(services)). + Msg("recreate progress") } + + wg.Add(1) + sem <- struct{}{} + + go func(svc *corev1.Service) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Delete(ctx, r.serviceBuilder.Namespace, svc.Name); err != nil { + r.logger.Error().Err(err).Str("service", svc.Name).Msg("failed to delete service for recreate") + return + } + // Clear ResourceVersion for create + svc.ResourceVersion = "" + if err := r.k8sClient.Create(ctx, svc); err != nil { + r.logger.Error().Err(err).Str("service", svc.Name).Msg("failed to create service after delete") + } else { + atomic.AddInt64(&recreated, 1) + } + }(svc) } - // Process creates - for _, svc := range diff.Create { - if err := r.k8sClient.Create(ctx, svc); err != nil { - result.Errors = append(result.Errors, fmt.Errorf("creating service %s: %w", svc.Name, err)) - } else { - result.Created++ + wg.Wait() + return int(recreated) +} + +// processCreatesConcurrently creates services using a worker pool. +func (r *Reconciler) processCreatesConcurrently(ctx context.Context, services []*corev1.Service, result *ReconcileResult) int { + var created int64 + var wg sync.WaitGroup + var errMu sync.Mutex + sem := make(chan struct{}, r.concurrency) + + for i, svc := range services { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(services)). + Int64("created", atomic.LoadInt64(&created)). + Msg("create progress") } + + wg.Add(1) + sem <- struct{}{} + + go func(svc *corev1.Service) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Create(ctx, svc); err != nil { + errMu.Lock() + result.Errors = append(result.Errors, fmt.Errorf("creating service %s: %w", svc.Name, err)) + errMu.Unlock() + } else { + atomic.AddInt64(&created, 1) + } + }(svc) } - // Process updates - for _, svc := range diff.Update { - if err := r.k8sClient.Update(ctx, svc); err != nil { - result.Errors = append(result.Errors, fmt.Errorf("updating service %s: %w", svc.Name, err)) - } else { - result.Updated++ + wg.Wait() + return int(created) +} + +// processUpdatesConcurrently updates services using a worker pool. +func (r *Reconciler) processUpdatesConcurrently(ctx context.Context, services []*corev1.Service, result *ReconcileResult) int { + var updated int64 + var wg sync.WaitGroup + var errMu sync.Mutex + sem := make(chan struct{}, r.concurrency) + + for i, svc := range services { + if i > 0 && i%100 == 0 { + r.logger.Info(). + Int("progress", i). + Int("total", len(services)). + Int64("updated", atomic.LoadInt64(&updated)). + Msg("update progress") } + + wg.Add(1) + sem <- struct{}{} + + go func(svc *corev1.Service) { + defer wg.Done() + defer func() { <-sem }() + + if err := r.k8sClient.Update(ctx, svc); err != nil { + errMu.Lock() + result.Errors = append(result.Errors, fmt.Errorf("updating service %s: %w", svc.Name, err)) + errMu.Unlock() + } else { + atomic.AddInt64(&updated, 1) + } + }(svc) } - return result + wg.Wait() + return int(updated) } diff --git a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml index 8b40ccf085..7e3f186477 100644 --- a/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml +++ b/helm/charts/nico-machine-a-tron/charts/mat-k8s-controller/values.yaml @@ -14,11 +14,11 @@ image: ## Resource limits resources: limits: - cpu: 200m - memory: 128Mi + cpu: "1" + memory: 512Mi requests: - cpu: 50m - memory: 64Mi + cpu: 100m + memory: 128Mi ## Controller configuration config: From 40751f516d2eef30dbc2be5c5394571aea417074 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 23 Jul 2026 14:04:24 -0600 Subject: [PATCH 17/17] fix(machine-a-tron-ctrl): k8s client bulk limits --- .../machine-a-tron-controller/cmd/mat-k8s-controller/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go index b75aa3563d..7c5977a186 100644 --- a/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go +++ b/dev/k8s/machine-a-tron-controller/cmd/mat-k8s-controller/main.go @@ -88,6 +88,10 @@ func main() { logger.Fatal().Err(err).Msg("failed to create Kubernetes config") } + // Increase rate limits for bulk operations + k8sConfig.QPS = 100 + k8sConfig.Burst = 200 + clientset, err := kubernetes.NewForConfig(k8sConfig) if err != nil { logger.Fatal().Err(err).Msg("failed to create Kubernetes clientset")