diff --git a/helm/kagent/templates/_helpers.tpl b/helm/kagent/templates/_helpers.tpl index 2c517ba21..36c5c3a54 100644 --- a/helm/kagent/templates/_helpers.tpl +++ b/helm/kagent/templates/_helpers.tpl @@ -132,6 +132,33 @@ Check if leader election should be enabled (more than 1 replica) {{- gt (.Values.controller.replicas | int) 1 -}} {{- end -}} +{{/* +Extract the TCP port from controller.metrics.bindAddress. + +Anchors the digit run to the end of the string so every Go-style +address form the controller binary accepts is handled correctly: bare +":port", host-qualified "host:port", and bracketed IPv6 "[::1]:port" +all yield the trailing port. Returns "0" or "" when the binary's +disable sentinel is in use; callers must consult +`kagent.controller.metricsEnabled` before rendering manifests. +*/}} +{{- define "kagent.controller.metricsPort" -}} +{{- regexFind "[0-9]+$" (.Values.controller.metrics.bindAddress | toString) -}} +{{- end -}} + +{{/* +Returns "1" when the controller metrics resources (Service, RBAC, +container port, env vars) should render, empty otherwise. Honours both +disable signals: `controller.metrics.enabled=false` and the binary's +own `--metrics-bind-address=0` sentinel reached through `bindAddress`. +The two are equivalent so the field name keeps faith with the binary's +documented contract (see go/core/pkg/app/app.go). +*/}} +{{- define "kagent.controller.metricsEnabled" -}} +{{- $port := include "kagent.controller.metricsPort" . -}} +{{- if and .Values.controller.metrics.enabled $port (ne $port "0") -}}1{{- end -}} +{{- end -}} + {{/* PostgreSQL service name for the bundled postgres instance */}} diff --git a/helm/kagent/templates/controller-deployment.yaml b/helm/kagent/templates/controller-deployment.yaml index ee7119b8e..28cfa39a1 100644 --- a/helm/kagent/templates/controller-deployment.yaml +++ b/helm/kagent/templates/controller-deployment.yaml @@ -84,6 +84,12 @@ spec: {{- else }} {{ fail "No database connection configured. Set database.postgres.url, database.postgres.urlFile, or enable database.postgres.bundled." }} {{- end }} + {{- if include "kagent.controller.metricsEnabled" . }} + - name: METRICS_BIND_ADDRESS + value: {{ .Values.controller.metrics.bindAddress | quote }} + - name: METRICS_SECURE + value: {{ .Values.controller.metrics.secureServing | quote }} + {{- end }} {{- with .Values.controller.env }} {{- toYaml . | nindent 12 }} {{- end }} @@ -97,6 +103,11 @@ spec: - name: http containerPort: {{ .Values.controller.service.ports.targetPort }} protocol: TCP + {{- if .Values.controller.metrics.enabled }} + - name: metrics + containerPort: {{ include "kagent.controller.metricsPort" . | int }} + protocol: TCP + {{- end }} resources: {{- toYaml .Values.controller.resources | nindent 12 }} {{- with (.Values.controller.securityContext | default .Values.securityContext) }} diff --git a/helm/kagent/templates/controller-metrics-service.yaml b/helm/kagent/templates/controller-metrics-service.yaml new file mode 100644 index 000000000..973645cfb --- /dev/null +++ b/helm/kagent/templates/controller-metrics-service.yaml @@ -0,0 +1,18 @@ +{{- if include "kagent.controller.metricsEnabled" . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kagent.fullname" . }}-controller-metrics + namespace: {{ include "kagent.namespace" . }} + labels: + {{- include "kagent.controller.labels" . | nindent 4 }} +spec: + type: {{ .Values.controller.metrics.service.type }} + ports: + - name: {{ ternary "https" "http-metrics" .Values.controller.metrics.secureServing }} + port: {{ .Values.controller.metrics.service.port }} + targetPort: {{ include "kagent.controller.metricsPort" . | int }} + protocol: TCP + selector: + {{- include "kagent.controller.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/kagent/templates/rbac/metrics-auth-clusterrole.yaml b/helm/kagent/templates/rbac/metrics-auth-clusterrole.yaml new file mode 100644 index 000000000..a97d5425f --- /dev/null +++ b/helm/kagent/templates/rbac/metrics-auth-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if and (include "kagent.controller.metricsEnabled" .) .Values.controller.metrics.secureServing }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kagent.fullname" . }}-metrics-auth-role + labels: + {{- include "kagent.controller.labels" . | nindent 4 }} +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +{{- end }} diff --git a/helm/kagent/templates/rbac/metrics-auth-clusterrolebinding.yaml b/helm/kagent/templates/rbac/metrics-auth-clusterrolebinding.yaml new file mode 100644 index 000000000..0f9d655dc --- /dev/null +++ b/helm/kagent/templates/rbac/metrics-auth-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (include "kagent.controller.metricsEnabled" .) .Values.controller.metrics.secureServing }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kagent.fullname" . }}-metrics-auth-rolebinding + labels: + {{- include "kagent.controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kagent.fullname" . }}-metrics-auth-role +subjects: + - kind: ServiceAccount + name: {{ include "kagent.fullname" . }}-controller + namespace: {{ include "kagent.namespace" . }} +{{- end }} diff --git a/helm/kagent/templates/rbac/metrics-reader-clusterrole.yaml b/helm/kagent/templates/rbac/metrics-reader-clusterrole.yaml new file mode 100644 index 000000000..434500923 --- /dev/null +++ b/helm/kagent/templates/rbac/metrics-reader-clusterrole.yaml @@ -0,0 +1,13 @@ +{{- if include "kagent.controller.metricsEnabled" . }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kagent.fullname" . }}-metrics-reader + labels: + {{- include "kagent.controller.labels" . | nindent 4 }} +rules: + - nonResourceURLs: + - "/metrics" + verbs: + - get +{{- end }} diff --git a/helm/kagent/tests/controller-deployment_test.yaml b/helm/kagent/tests/controller-deployment_test.yaml index a35a9c227..a26c8e937 100644 --- a/helm/kagent/tests/controller-deployment_test.yaml +++ b/helm/kagent/tests/controller-deployment_test.yaml @@ -496,3 +496,152 @@ tests: path: spec.template.spec.containers[0].env content: name: POSTGRES_PASSWORD + + - it: should not expose metrics container port by default + template: controller-deployment.yaml + asserts: + - lengthEqual: + path: spec.template.spec.containers[0].ports + count: 1 + - notContains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + + - it: should not set metrics env vars by default + template: controller-deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_BIND_ADDRESS + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_SECURE + any: true + + - it: should expose metrics container port when enabled + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + containerPort: 8443 + protocol: TCP + + - it: should set metrics env vars when enabled + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_BIND_ADDRESS + value: ":8443" + - contains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_SECURE + value: "true" + + - it: should reflect insecure serving in env + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + controller.metrics.secureServing: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_SECURE + value: "false" + + - it: should derive metrics container port from bindAddress + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: ":9443" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_BIND_ADDRESS + value: ":9443" + - contains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + containerPort: 9443 + protocol: TCP + + - it: should derive metrics container port from a host-qualified bindAddress + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "127.0.0.1:9443" + asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + containerPort: 9443 + protocol: TCP + + - it: should derive metrics container port from a bracketed IPv6 bindAddress + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "[::1]:9443" + asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + containerPort: 9443 + protocol: TCP + + - it: should not gain metrics wiring when bindAddress disables metrics + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "0" + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_BIND_ADDRESS + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_SECURE + any: true + - notContains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + + - it: should let controller.env override the chart-supplied metrics env + template: controller-deployment.yaml + set: + controller.metrics.enabled: true + controller.env: + - name: METRICS_BIND_ADDRESS + value: "0" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_BIND_ADDRESS + value: ":8443" + - contains: + path: spec.template.spec.containers[0].env + content: + name: METRICS_BIND_ADDRESS + value: "0" diff --git a/helm/kagent/tests/controller-metrics-rbac_test.yaml b/helm/kagent/tests/controller-metrics-rbac_test.yaml new file mode 100644 index 000000000..87a3fce36 --- /dev/null +++ b/helm/kagent/tests/controller-metrics-rbac_test.yaml @@ -0,0 +1,128 @@ +suite: test controller metrics rbac +templates: + - rbac/metrics-reader-clusterrole.yaml + - rbac/metrics-auth-clusterrole.yaml + - rbac/metrics-auth-clusterrolebinding.yaml +tests: + - it: should not render any metrics rbac by default + asserts: + - hasDocuments: + count: 0 + template: rbac/metrics-reader-clusterrole.yaml + - hasDocuments: + count: 0 + template: rbac/metrics-auth-clusterrole.yaml + - hasDocuments: + count: 0 + template: rbac/metrics-auth-clusterrolebinding.yaml + + - it: should render metrics-reader when metrics enabled + set: + controller.metrics.enabled: true + template: rbac/metrics-reader-clusterrole.yaml + asserts: + - isKind: + of: ClusterRole + - equal: + path: metadata.name + value: RELEASE-NAME-metrics-reader + - equal: + path: rules[0].nonResourceURLs[0] + value: /metrics + - equal: + path: rules[0].verbs[0] + value: get + + - it: should render auth role and binding when secure serving enabled + set: + controller.metrics.enabled: true + asserts: + - isKind: + of: ClusterRole + template: rbac/metrics-auth-clusterrole.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-metrics-auth-role + template: rbac/metrics-auth-clusterrole.yaml + - isKind: + of: ClusterRoleBinding + template: rbac/metrics-auth-clusterrolebinding.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-metrics-auth-rolebinding + template: rbac/metrics-auth-clusterrolebinding.yaml + + - it: auth role should grant token and subject access reviews + set: + controller.metrics.enabled: true + template: rbac/metrics-auth-clusterrole.yaml + asserts: + - equal: + path: rules[0].apiGroups[0] + value: authentication.k8s.io + - equal: + path: rules[0].resources[0] + value: tokenreviews + - equal: + path: rules[0].verbs[0] + value: create + - equal: + path: rules[1].apiGroups[0] + value: authorization.k8s.io + - equal: + path: rules[1].resources[0] + value: subjectaccessreviews + - equal: + path: rules[1].verbs[0] + value: create + + - it: auth rolebinding should reference controller serviceaccount + set: + controller.metrics.enabled: true + template: rbac/metrics-auth-clusterrolebinding.yaml + asserts: + - equal: + path: roleRef.kind + value: ClusterRole + - equal: + path: roleRef.name + value: RELEASE-NAME-metrics-auth-role + - equal: + path: subjects[0].kind + value: ServiceAccount + - equal: + path: subjects[0].name + value: RELEASE-NAME-controller + - equal: + path: subjects[0].namespace + value: NAMESPACE + + - it: should skip auth role when secure serving disabled + set: + controller.metrics.enabled: true + controller.metrics.secureServing: false + asserts: + - hasDocuments: + count: 1 + template: rbac/metrics-reader-clusterrole.yaml + - hasDocuments: + count: 0 + template: rbac/metrics-auth-clusterrole.yaml + - hasDocuments: + count: 0 + template: rbac/metrics-auth-clusterrolebinding.yaml + + - it: should skip all metrics rbac when bindAddress disables metrics + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "0" + asserts: + - hasDocuments: + count: 0 + template: rbac/metrics-reader-clusterrole.yaml + - hasDocuments: + count: 0 + template: rbac/metrics-auth-clusterrole.yaml + - hasDocuments: + count: 0 + template: rbac/metrics-auth-clusterrolebinding.yaml diff --git a/helm/kagent/tests/controller-metrics-service_test.yaml b/helm/kagent/tests/controller-metrics-service_test.yaml new file mode 100644 index 000000000..751f8ee7e --- /dev/null +++ b/helm/kagent/tests/controller-metrics-service_test.yaml @@ -0,0 +1,129 @@ +suite: test controller metrics service +templates: + - controller-metrics-service.yaml +tests: + - it: should not render by default + asserts: + - hasDocuments: + count: 0 + + - it: should render when metrics are enabled + set: + controller.metrics.enabled: true + asserts: + - isKind: + of: Service + - equal: + path: metadata.name + value: RELEASE-NAME-controller-metrics + - equal: + path: spec.type + value: ClusterIP + - hasDocuments: + count: 1 + + - it: should expose https port from values + set: + controller.metrics.enabled: true + asserts: + - equal: + path: spec.ports[0].name + value: https + - equal: + path: spec.ports[0].port + value: 8443 + - equal: + path: spec.ports[0].targetPort + value: 8443 + - equal: + path: spec.ports[0].protocol + value: TCP + + - it: should derive targetPort from bindAddress + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: ":9443" + controller.metrics.service.port: 9443 + asserts: + - equal: + path: spec.ports[0].port + value: 9443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + + - it: should derive targetPort from a host-qualified bindAddress + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "127.0.0.1:9443" + controller.metrics.service.port: 9443 + asserts: + - equal: + path: spec.ports[0].targetPort + value: 9443 + + - it: should derive targetPort from a bracketed IPv6 bindAddress + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "[::1]:9443" + controller.metrics.service.port: 9443 + asserts: + - equal: + path: spec.ports[0].targetPort + value: 9443 + + - it: should not render when bindAddress disables metrics + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "0" + asserts: + - hasDocuments: + count: 0 + + - it: should not render when bindAddress is empty + set: + controller.metrics.enabled: true + controller.metrics.bindAddress: "" + asserts: + - hasDocuments: + count: 0 + + - it: should rename port when secure serving disabled + set: + controller.metrics.enabled: true + controller.metrics.secureServing: false + asserts: + - equal: + path: spec.ports[0].name + value: http-metrics + + - it: should select controller pods + set: + controller.metrics.enabled: true + asserts: + - equal: + path: spec.selector["app.kubernetes.io/name"] + value: kagent + - equal: + path: spec.selector["app.kubernetes.io/instance"] + value: RELEASE-NAME + - equal: + path: spec.selector["app.kubernetes.io/component"] + value: controller + + - it: should be in correct namespace + set: + controller.metrics.enabled: true + asserts: + - equal: + path: metadata.namespace + value: NAMESPACE + + - it: should use custom namespace when overridden + set: + controller.metrics.enabled: true + namespaceOverride: "custom-namespace" + asserts: + - equal: + path: metadata.namespace + value: custom-namespace diff --git a/helm/kagent/values.yaml b/helm/kagent/values.yaml index 446cc54e7..07c1978a9 100644 --- a/helm/kagent/values.yaml +++ b/helm/kagent/values.yaml @@ -222,6 +222,26 @@ controller: ports: port: 8083 targetPort: 8083 + # -- Prometheus-style /metrics endpoint for the controller manager. + # When enabled, provisions a dedicated metrics Service plus the + # ClusterRoles required for authenticated scrapes. Bind + # `-metrics-reader` to your Prometheus ServiceAccount to grant + # scrape access. Use `bindAddress` for any port change: the Service + # `targetPort` and the pod `containerPort` are derived from it at + # template time, so overriding `METRICS_BIND_ADDRESS` via + # `controller.env` shifts only the runtime listener and leaves the + # rendered Service pointing at the chart-time port. Setting + # `bindAddress: "0"` (or empty) is treated as a disable signal — + # equivalent to `enabled: false` — to keep faith with the controller + # binary's documented contract for `--metrics-bind-address`. + # @default -- disabled + metrics: + enabled: false + bindAddress: ":8443" + secureServing: true + service: + type: ClusterIP + port: 8443 env: [] envFrom: []