diff --git a/argocd/apps/12-vault-monitoring.yaml b/argocd/apps/12-vault-monitoring.yaml new file mode 100644 index 0000000..89e13e7 --- /dev/null +++ b/argocd/apps/12-vault-monitoring.yaml @@ -0,0 +1,33 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: vault-monitoring + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "12" +spec: + project: kernelcafe + source: + repoURL: https://github.com/systemdbrew/kernelcafe.git + targetRevision: main + path: infrastructure/vault + directory: + include: monitoring.yaml + destination: + server: https://kubernetes.default.svc + namespace: vault + syncPolicy: + automated: + prune: true + selfHeal: true + retry: + limit: -1 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - PruneLast=true + revisionHistoryLimit: 10 diff --git a/argocd/apps/35-monitoring.yaml b/argocd/apps/35-monitoring.yaml index 3c0c0c3..4859e0a 100644 --- a/argocd/apps/35-monitoring.yaml +++ b/argocd/apps/35-monitoring.yaml @@ -58,48 +58,6 @@ spec: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out on cluster {{ $labels.cluster }}. - - alert: VaultManualRolloutPending - expr: |- - ( - max by (namespace, statefulset, job, cluster) ( - kube_statefulset_status_current_revision{ - job="kube-state-metrics", - namespace="vault", - statefulset="vault" - } - unless - kube_statefulset_status_update_revision{ - job="kube-state-metrics", - namespace="vault", - statefulset="vault" - } - ) - * on (namespace, statefulset, job, cluster) - ( - kube_statefulset_replicas{ - job="kube-state-metrics", - namespace="vault", - statefulset="vault" - } - != - kube_statefulset_status_replicas_updated{ - job="kube-state-metrics", - namespace="vault", - statefulset="vault" - } - ) - ) - for: 30m - labels: - severity: info - annotations: - summary: Vault manual rollout is pending. - description: >- - Vault has a new StatefulSet revision waiting for its - controlled OnDelete rollout. Upgrade standby members one - at a time, unseal and verify Raft catch-up, then replace - the active member. - grafana: sidecar: dashboards: diff --git a/docs/vault-operations.md b/docs/vault-operations.md index 1c67f8a..ef6f921 100644 --- a/docs/vault-operations.md +++ b/docs/vault-operations.md @@ -46,22 +46,30 @@ unsealed, and caught up. ## Monitoring -The Vault Helm release creates a Prometheus `ServiceMonitor` that targets the -`vault-internal` service so Prometheus can observe every Vault server. Vault's -listener allows unauthenticated access only to the metrics endpoint. +Vault monitoring is managed as standalone GitOps resources in +`infrastructure/vault/monitoring.yaml` and applied by the `vault-monitoring` +Argo CD application. The `ServiceMonitor` selects the headless `vault-internal` +service, allowing Prometheus to scrape each of the three Vault members +individually on the named `http` port. Vault's listener permits unauthenticated +access to the metrics endpoint. Vault-specific alerts cover: - a server remaining sealed; - no active HA server; - more than one server reporting active; -- fewer than three Vault telemetry targets. +- fewer than three healthy Vault telemetry targets; +- a pending manual `OnDelete` rollout. The upstream `KubeStatefulSetUpdateNotRolledOut` rule is disabled and replaced -with a KernelCafe version that excludes only `vault/vault`. Vault gets a -separate informational `VaultManualRolloutPending` alert after 30 minutes so a -pending controlled upgrade remains visible without being treated as a failed -rolling deployment. +with a KernelCafe version that excludes only `vault/vault` so Vault's deliberate +upgrade workflow is not reported as a generic failed rollout. The separate +`VaultManualRolloutPending` warning remains visible after 30 minutes and points +to the controlled standby-first procedure above. + +The monitoring design was verified against a three-member cluster: Prometheus +reported all three `vault-internal` targets UP, all three members unsealed, +exactly one active member, and all five Vault alert rules loaded and healthy. ## Auto-unseal diff --git a/infrastructure/vault/monitoring.yaml b/infrastructure/vault/monitoring.yaml new file mode 100644 index 0000000..9cc5c94 --- /dev/null +++ b/infrastructure/vault/monitoring.yaml @@ -0,0 +1,78 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: vault + namespace: vault + labels: + app.kubernetes.io/name: vault +spec: + namespaceSelector: + matchNames: + - vault + selector: + matchLabels: + app.kubernetes.io/name: vault + vault-internal: "true" + endpoints: + - port: http + path: /v1/sys/metrics + params: + format: + - prometheus + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: vault-health + namespace: vault + labels: + app.kubernetes.io/name: vault +spec: + groups: + - name: vault.health + rules: + - alert: VaultSealed + expr: vault_core_unsealed == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Vault instance {{ $labels.instance }} is sealed" + description: "A Vault HA member has remained sealed for more than 2 minutes." + - alert: VaultNoActiveNode + expr: sum(vault_core_active) < 1 + for: 2m + labels: + severity: critical + annotations: + summary: "Vault has no active node" + description: "No Vault HA member is reporting itself active. Check Raft quorum and pod health immediately." + - alert: VaultMultipleActiveNodes + expr: sum(vault_core_active) > 1 + for: 2m + labels: + severity: critical + annotations: + summary: "Vault reports multiple active nodes" + description: "More than one Vault HA member is reporting itself active. Investigate cluster/Raft state immediately." + - alert: VaultTelemetryTargetMissing + expr: count(up{namespace="vault", job=~".*vault.*"} == 1) < 3 + for: 10m + labels: + severity: warning + annotations: + summary: "Vault telemetry is missing one or more HA members" + description: "Prometheus has fewer than three healthy Vault scrape targets for 10 minutes." + - alert: VaultManualRolloutPending + expr: | + kube_statefulset_status_current_revision{namespace="vault", statefulset="vault"} + unless on (namespace, statefulset) + kube_statefulset_status_update_revision{namespace="vault", statefulset="vault"} + for: 30m + labels: + severity: warning + annotations: + summary: "Vault has a manual rollout pending" + description: "Vault's desired StatefulSet revision differs from the running revision. Vault intentionally uses OnDelete with Shamir sealing; perform the documented standby-first rollout and unseal procedure." diff --git a/infrastructure/vault/values.yaml b/infrastructure/vault/values.yaml index 6199ad2..858b88e 100644 --- a/infrastructure/vault/values.yaml +++ b/infrastructure/vault/values.yaml @@ -66,62 +66,3 @@ ui: csi: enabled: false - -# Scrape every Vault server. The listener explicitly permits unauthenticated -# metrics, so standbys can expose their own seal/HA state as well as the active. -serverTelemetry: - serviceMonitor: - enabled: true - selectors: - release: monitoring - matchLabels: - vault-internal: "true" - - prometheusRules: - enabled: true - selectors: - release: monitoring - rules: - - alert: VaultSealed - expr: vault_core_unsealed == 0 - for: 2m - labels: - severity: critical - annotations: - summary: Vault server is sealed. - description: >- - Vault target {{ $labels.instance }} has remained sealed for more than - 2 minutes. Unseal it before taking another Raft member offline. - - - alert: VaultNoActiveNode - expr: sum(vault_core_active) < 1 - for: 2m - labels: - severity: critical - annotations: - summary: Vault has no active HA node. - description: >- - No scraped Vault server has reported itself active for more than - 2 minutes. Check Raft quorum, seal state, and pod health. - - - alert: VaultMultipleActiveNodes - expr: sum(vault_core_active) > 1 - for: 2m - labels: - severity: critical - annotations: - summary: Multiple Vault servers report active. - description: >- - More than one Vault server reports HA active state. Investigate - cluster connectivity and Raft leadership immediately. - - - alert: VaultTelemetryTargetsMissing - expr: count(vault_core_unsealed) < 3 - for: 5m - labels: - severity: warning - annotations: - summary: Vault telemetry is missing one or more servers. - description: >- - Prometheus is receiving Vault core telemetry from fewer than the - expected three servers. Check the ServiceMonitor and Vault pods.