diff --git a/charts/truvami-monitoring/Chart.yaml b/charts/truvami-monitoring/Chart.yaml index 10302db..48129b7 100644 --- a/charts/truvami-monitoring/Chart.yaml +++ b/charts/truvami-monitoring/Chart.yaml @@ -15,4 +15,4 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.0 +version: 0.5.0 diff --git a/charts/truvami-monitoring/README.md b/charts/truvami-monitoring/README.md index dec4341..3ed37d6 100644 --- a/charts/truvami-monitoring/README.md +++ b/charts/truvami-monitoring/README.md @@ -1,6 +1,6 @@ # truvami-monitoring -![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.0](https://img.shields.io/badge/Version-0.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) This chart contains all CRD's for the truvami-stack monitoring and alerting. @@ -17,7 +17,8 @@ This chart contains all CRD's for the truvami-stack monitoring and alerting. | alertLabels.altravis_test | string | `"true"` | | | alertLabels.code | string | `"SBCC"` | | | alertLabels.customer | string | `"truvami"` | | -| alertmanager.enabled | bool | `true` | | +| alertmanager.configSelectorLabels | string | `nil` | | +| alertmanager.enabled | bool | `false` | | | alertmanager.global.http_config.enable_http2 | bool | `true` | | | alertmanager.global.http_config.follow_redirects | bool | `true` | | | alertmanager.global.resolve_timeout | string | `"5m"` | | @@ -32,6 +33,7 @@ This chart contains all CRD's for the truvami-stack monitoring and alerting. | alertmanager.inhibit_rules[1].equal[0] | string | `"service"` | | | alertmanager.inhibit_rules[1].source_matchers[0] | string | `"maintenance=\"true\""` | | | alertmanager.inhibit_rules[1].target_matchers[0] | string | `"service=~\".*\""` | | +| alertmanager.portName | string | `"web"` | | | alertmanager.receivers.critical.enabled | bool | `true` | | | alertmanager.receivers.critical.slack.channel | string | `"#critical-alerts"` | | | alertmanager.receivers.critical.slack.enabled | bool | `false` | | @@ -91,10 +93,12 @@ This chart contains all CRD's for the truvami-stack monitoring and alerting. | alertmanager.receivers.teams.summary | string | `"{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}"` | | | alertmanager.receivers.teams.title | string | `"{{ .GroupLabels.alertname }}"` | | | alertmanager.receivers.teams.webhook_url | string | `""` | | +| alertmanager.replicas | int | `2` | | | alertmanager.resources.limits.cpu | string | `"500m"` | | | alertmanager.resources.limits.memory | string | `"512Mi"` | | | alertmanager.resources.requests.cpu | string | `"50m"` | | | alertmanager.resources.requests.memory | string | `"128Mi"` | | +| alertmanager.retention | string | `"120h"` | | | alertmanager.route.group_by[0] | string | `"alertname"` | | | alertmanager.route.group_by[1] | string | `"cluster"` | | | alertmanager.route.group_by[2] | string | `"service"` | | diff --git a/charts/truvami-monitoring/templates/alertmanager-config.yaml b/charts/truvami-monitoring/templates/alertmanager-config.yaml index fe71e4d..2fac3c3 100644 --- a/charts/truvami-monitoring/templates/alertmanager-config.yaml +++ b/charts/truvami-monitoring/templates/alertmanager-config.yaml @@ -33,10 +33,15 @@ spec: repeatInterval: '1h' # Watchdog alerts route + # NOTE: this must match the alert name emitted by templates/rule-watchdog.yaml, + # which is "TruvamiWatchdog". It previously read "Watchdog", and because the + # matcher is an exact match it never fired - so the dead-man's-switch + # heartbeat never reached watchdog.webhookUrl even when everything else in + # the alerting path was healthy. - matchers: - matchType: "=" name: "alertname" - value: "Watchdog" + value: "TruvamiWatchdog" receiver: 'watchdog-heartbeat' groupWait: '0s' groupInterval: '1m' diff --git a/charts/truvami-monitoring/templates/alertmanager-service.yaml b/charts/truvami-monitoring/templates/alertmanager-service.yaml new file mode 100644 index 0000000..198a0e5 --- /dev/null +++ b/charts/truvami-monitoring/templates/alertmanager-service.yaml @@ -0,0 +1,39 @@ +{{- if .Values.alertmanager.enabled }} +{{/* +Service fronting this release's Alertmanager pods. + +The Prometheus CR's spec.alerting.alertmanagers[] entry addresses Alertmanager +by *Service* name - prometheus-operator resolves that Service's Endpoints to +build the alerting targets. Without this object the reference dangles and +Prometheus reports zero activeAlertmanagers, so every firing alert is silently +dropped. The operator's own `alertmanager-operated` Service is not a substitute: +it selects `app.kubernetes.io/name: alertmanager` across the whole namespace, so +where several Alertmanager CRs coexist it would fan alerts out to all of them. + +The `alertmanager` label is set by prometheus-operator on the pods it creates +for an Alertmanager CR, and carries that CR's name - so this selector matches +only the Alertmanager owned by this release. +*/}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "truvami-monitoring.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "truvami-monitoring.labels" . | nindent 4 }} + app.kubernetes.io/component: alertmanager +spec: + type: ClusterIP + # Alertmanager gossips over the mesh ports; peers must be discoverable before + # they are Ready, and Prometheus should keep addressing a peer that is still + # settling rather than lose its only alerting target during a rollout. + publishNotReadyAddresses: true + selector: + alertmanager: {{ include "truvami-monitoring.fullname" . }} + app.kubernetes.io/name: alertmanager + ports: + - name: {{ .Values.alertmanager.portName | default "web" }} + port: 9093 + targetPort: {{ .Values.alertmanager.portName | default "web" }} + protocol: TCP +{{- end }} diff --git a/charts/truvami-monitoring/templates/prometheus.yaml b/charts/truvami-monitoring/templates/prometheus.yaml index 173157d..76f866a 100644 --- a/charts/truvami-monitoring/templates/prometheus.yaml +++ b/charts/truvami-monitoring/templates/prometheus.yaml @@ -84,11 +84,17 @@ spec: serviceMonitorNamespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ .Release.Namespace }} + {{- if .Values.alertmanager.enabled }} alerting: alertmanagers: + # Addresses the Service created by templates/alertmanager-service.yaml. + # Only emitted when the Alertmanager is enabled - pointing Prometheus at a + # Service that is not rendered leaves it with zero activeAlertmanagers and + # silently discards every firing alert. - namespace: {{ .Release.Namespace }} name: "{{ include "truvami-monitoring.fullname" . }}" - port: web + port: {{ .Values.alertmanager.portName | default "web" }} + {{- end }} enableRemoteWriteReceiver: {{ .Values.prometheus.enableRemoteWriteReceiver | default "false" }} {{- if .Values.thanos.enabled }} thanos: @@ -106,12 +112,48 @@ spec: {{- end }} {{- end }} +{{- if .Values.alertmanager.enabled }} --- apiVersion: monitoring.coreos.com/v1 kind: Alertmanager metadata: name: "{{ include "truvami-monitoring.fullname" . }}" + labels: + {{- include "truvami-monitoring.labels" . | nindent 4 }} spec: - replicas: 2 - configSecret: "{{ include "truvami-monitoring.fullname" . }}-alertmanager-config" + replicas: {{ .Values.alertmanager.replicas | default 2 }} + portName: {{ .Values.alertmanager.portName | default "web" }} + {{- with .Values.alertmanager.retention }} + retention: {{ . | quote }} + {{- end }} + # Use the AlertmanagerConfig rendered by templates/alertmanager-config.yaml as + # this Alertmanager's top-level configuration. + # + # This replaces the previous `configSecret` reference, which named a Secret + # ("-alertmanager-config") that no template in this chart has ever + # created. With that Secret absent, prometheus-operator fell back to its + # built-in default configuration, whose root route sends everything to the + # "null" receiver - so alerts were accepted and then discarded. + alertmanagerConfiguration: + name: {{ include "truvami-monitoring.fullname" . }}-config + # Additionally select any AlertmanagerConfig in this namespace that opts in, + # so teams can attach extra routes/receivers without editing this chart. + # Previously unset, which meant zero AlertmanagerConfigs were ever selected. + # + # An explicitly configured empty map is a meaningful value here - + # matchLabels: {} selects *every* AlertmanagerConfig in the namespace - so it + # must be distinguishable from "not configured". + # + # `default` cannot do that (it treats {} as empty), and neither can `hasKey`: + # the key is always present because values.yaml declares it. So the declared + # default is null, and only null means "not configured" - Helm lets a user + # map replace a null base wholesale, whereas a non-empty default map would be + # merged with the user's value and could never be reduced back to empty. + {{- $configSelector := .Values.alertmanager.configSelectorLabels }} + {{- if kindIs "invalid" $configSelector }} + {{- $configSelector = dict "truvami.com/alertmanager-config" "true" }} + {{- end }} + alertmanagerConfigSelector: + matchLabels: {{- toYaml $configSelector | nindent 6 }} resources: {{ toYaml .Values.alertmanager.resources | default "{}" | nindent 4 }} +{{- end }} diff --git a/charts/truvami-monitoring/values.yaml b/charts/truvami-monitoring/values.yaml index 2ec440b..5910109 100644 --- a/charts/truvami-monitoring/values.yaml +++ b/charts/truvami-monitoring/values.yaml @@ -373,7 +373,52 @@ watchdog: # AlertManager configuration alertmanager: - enabled: true + # Alerting is handled by Grafana-managed alert rules, not by Alertmanager, so + # this defaults to off. + # + # When false, no Alertmanager CR, AlertmanagerConfig, Alertmanager secrets or + # Alertmanager Service are rendered, AND the Prometheus CR omits its + # spec.alerting block - so Prometheus is not left pointed at a Service that + # was never created. + # + # Prior to chart 0.5.0 the Alertmanager CR ignored this flag and was always + # created, so environments that set `enabled: false` still ran Alertmanager + # pods. + # + # From 0.5.0 the chart stops *rendering* those objects - it does not delete + # what is already deployed. Where ArgoCD pruning is disabled (it is on both + # the dev and prod truvami-monitoring apps), the existing Alertmanager CR and + # the pods it owns survive the upgrade and must be removed by hand: + # kubectl -n delete alertmanager + enabled: false + + # Everything below applies only when `enabled: true`. + + # Number of Alertmanager replicas. Keep >1 so a single node event cannot take + # the alerting path down. + replicas: 2 + + # Name of the Alertmanager container port, and the port name referenced by + # both the Service and the Prometheus spec.alerting entry. Changing this + # keeps all three in sync. + portName: web + + # How long Alertmanager retains resolved alert state. + retention: 120h + + # Labels an AlertmanagerConfig in this namespace must carry to be merged into + # this Alertmanager, via spec.alertmanagerConfigSelector. The chart's own + # config is attached separately as the top-level configuration and does not + # need this label. + # + # Left null, the chart selects on `truvami.com/alertmanager-config: "true"`. + # Set a map to select on your own labels, or `{}` to select EVERY + # AlertmanagerConfig in the namespace (rarely what you want). + # + # The default is expressed in the template rather than here on purpose: Helm + # merges map values, so a non-empty default here could be added to but never + # reduced back to `{}`. + configSelectorLabels: # Resource configuration for Alertmanager resources: diff --git a/diagrams/truvami-monitoring.png b/diagrams/truvami-monitoring.png index 7c7132e..5cfa368 100644 Binary files a/diagrams/truvami-monitoring.png and b/diagrams/truvami-monitoring.png differ diff --git a/diagrams/truvami-monitoring.previous.png b/diagrams/truvami-monitoring.previous.png new file mode 100644 index 0000000..7c7132e Binary files /dev/null and b/diagrams/truvami-monitoring.previous.png differ