From 987e0439f025b8316105d558e9da50b192c7863e Mon Sep 17 00:00:00 2001 From: Jonathan Bohbot <25457950+systemdbrew@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:46:32 -0400 Subject: [PATCH 1/3] feat(vault): enable Prometheus telemetry and alerts --- infrastructure/vault/values.yaml | 59 ++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/infrastructure/vault/values.yaml b/infrastructure/vault/values.yaml index 858b88e..6199ad2 100644 --- a/infrastructure/vault/values.yaml +++ b/infrastructure/vault/values.yaml @@ -66,3 +66,62 @@ ui: csi: enabled: false + +# Scrape every Vault server. The listener explicitly permits unauthenticated +# metrics, so standbys can expose their own seal/HA state as well as the active. +serverTelemetry: + serviceMonitor: + enabled: true + selectors: + release: monitoring + matchLabels: + vault-internal: "true" + + prometheusRules: + enabled: true + selectors: + release: monitoring + rules: + - alert: VaultSealed + expr: vault_core_unsealed == 0 + for: 2m + labels: + severity: critical + annotations: + summary: Vault server is sealed. + description: >- + Vault target {{ $labels.instance }} has remained sealed for more than + 2 minutes. Unseal it before taking another Raft member offline. + + - alert: VaultNoActiveNode + expr: sum(vault_core_active) < 1 + for: 2m + labels: + severity: critical + annotations: + summary: Vault has no active HA node. + description: >- + No scraped Vault server has reported itself active for more than + 2 minutes. Check Raft quorum, seal state, and pod health. + + - alert: VaultMultipleActiveNodes + expr: sum(vault_core_active) > 1 + for: 2m + labels: + severity: critical + annotations: + summary: Multiple Vault servers report active. + description: >- + More than one Vault server reports HA active state. Investigate + cluster connectivity and Raft leadership immediately. + + - alert: VaultTelemetryTargetsMissing + expr: count(vault_core_unsealed) < 3 + for: 5m + labels: + severity: warning + annotations: + summary: Vault telemetry is missing one or more servers. + description: >- + Prometheus is receiving Vault core telemetry from fewer than the + expected three servers. Check the ServiceMonitor and Vault pods. From 49667dc81c41b65cdc6642fe2e05cac9b70a88eb Mon Sep 17 00:00:00 2001 From: Jonathan Bohbot <25457950+systemdbrew@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:46:43 -0400 Subject: [PATCH 2/3] fix(monitoring): replace StatefulSet rollout alert --- argocd/apps/35-monitoring.yaml | 87 ++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/argocd/apps/35-monitoring.yaml b/argocd/apps/35-monitoring.yaml index 9ba310c..3c0c0c3 100644 --- a/argocd/apps/35-monitoring.yaml +++ b/argocd/apps/35-monitoring.yaml @@ -13,6 +13,93 @@ spec: targetRevision: 90.1.1 helm: values: | + # The upstream rule treats OnDelete StatefulSets as failed rollouts. + # Vault intentionally uses OnDelete so upgrades can be rolled and + # unsealed one Raft member at a time. A replacement rule below keeps + # normal StatefulSet coverage while excluding vault/vault. + defaultRules: + disabled: + KubeStatefulSetUpdateNotRolledOut: true + + additionalPrometheusRulesMap: + kernelcafe-statefulset-rollouts: + groups: + - name: kernelcafe-statefulset-rollouts + rules: + - alert: KubeStatefulSetUpdateNotRolledOut + expr: |- + ( + max by (namespace, statefulset, job, cluster) ( + kube_statefulset_status_current_revision{job="kube-state-metrics"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics"} + ) + * on (namespace, statefulset, job, cluster) + ( + kube_statefulset_replicas{job="kube-state-metrics"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics"} + ) + ) + and on (namespace, statefulset, job, cluster) + ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m]) == 0 + ) + unless on (namespace, statefulset) + ( + kube_statefulset_replicas{namespace="vault", statefulset="vault"} + ) + for: 15m + labels: + severity: warning + annotations: + summary: StatefulSet update has not been rolled out. + description: >- + StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} + update has not been rolled out on cluster {{ $labels.cluster }}. + + - alert: VaultManualRolloutPending + expr: |- + ( + max by (namespace, statefulset, job, cluster) ( + kube_statefulset_status_current_revision{ + job="kube-state-metrics", + namespace="vault", + statefulset="vault" + } + unless + kube_statefulset_status_update_revision{ + job="kube-state-metrics", + namespace="vault", + statefulset="vault" + } + ) + * on (namespace, statefulset, job, cluster) + ( + kube_statefulset_replicas{ + job="kube-state-metrics", + namespace="vault", + statefulset="vault" + } + != + kube_statefulset_status_replicas_updated{ + job="kube-state-metrics", + namespace="vault", + statefulset="vault" + } + ) + ) + for: 30m + labels: + severity: info + annotations: + summary: Vault manual rollout is pending. + description: >- + Vault has a new StatefulSet revision waiting for its + controlled OnDelete rollout. Upgrade standby members one + at a time, unseal and verify Raft catch-up, then replace + the active member. + grafana: sidecar: dashboards: From 959d6c962efda8b732a80890f592f5e15c03ff10 Mon Sep 17 00:00:00 2001 From: Jonathan Bohbot <25457950+systemdbrew@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:47:00 -0400 Subject: [PATCH 3/3] docs(vault): add rollout and auto-unseal runbook --- docs/vault-operations.md | 93 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 docs/vault-operations.md diff --git a/docs/vault-operations.md b/docs/vault-operations.md new file mode 100644 index 0000000..1c67f8a --- /dev/null +++ b/docs/vault-operations.md @@ -0,0 +1,93 @@ +# Vault operations + +KernelCafe runs a three-member HashiCorp Vault HA cluster backed by integrated +Raft storage and Longhorn persistent volumes. + +## Upgrade model + +Vault intentionally uses the Helm chart's `OnDelete` StatefulSet strategy. +Argo CD may update the StatefulSet pod template, but Kubernetes must not replace +all Vault members automatically while Shamir sealing is in use. + +For an image or pod-template update: + +1. Confirm all three pods are Ready and unsealed. +2. Confirm one node is active, two are standby, and Raft committed/applied + indexes are caught up. +3. Delete one standby pod. +4. Wait for it to return on the new StatefulSet revision. +5. If Shamir sealing is still configured, unseal it with the required key + shares. Never place unseal shares in Git, shell history, Kubernetes + manifests, or chat/log output. +6. Confirm the upgraded member is Ready, standby, and caught up with Raft. +7. Repeat for the second standby. +8. Delete the old active member. The upgraded members should elect a new + active node. +9. Unseal the recreated final member and verify it rejoins as Ready. +10. Confirm all pods use the same image/revision, exactly one member is active, + all are unsealed, and Raft indexes are caught up. + +Useful checks: + +```bash +kubectl -n vault get sts vault +kubectl -n vault get pods -l app.kubernetes.io/name=vault \ + -o custom-columns='NAME:.metadata.name,READY:.status.containerStatuses[*].ready,REVISION:.metadata.labels.controller-revision-hash,IMAGE:.spec.containers[*].image' + +for pod in vault-0 vault-1 vault-2; do + echo "===== $pod =====" + kubectl -n vault exec "$pod" -- vault status + echo +done +``` + +Do not delete a second member until the previously replaced member is Ready, +unsealed, and caught up. + +## Monitoring + +The Vault Helm release creates a Prometheus `ServiceMonitor` that targets the +`vault-internal` service so Prometheus can observe every Vault server. Vault's +listener allows unauthenticated access only to the metrics endpoint. + +Vault-specific alerts cover: + +- a server remaining sealed; +- no active HA server; +- more than one server reporting active; +- fewer than three Vault telemetry targets. + +The upstream `KubeStatefulSetUpdateNotRolledOut` rule is disabled and replaced +with a KernelCafe version that excludes only `vault/vault`. Vault gets a +separate informational `VaultManualRolloutPending` alert after 30 minutes so a +pending controlled upgrade remains visible without being treated as a failed +rolling deployment. + +## Auto-unseal + +Auto-unseal is the preferred future state, but it must not be configured until +KernelCafe has an independent trusted seal provider. Integrated Raft storage is +not itself an auto-unseal provider, and using this same Vault cluster as its own +Transit seal creates a circular dependency. + +Suitable providers include a supported cloud KMS/HSM or a separate, +independently operated Vault Transit service. The seal provider's credentials +must be supplied through an appropriate workload identity or Kubernetes Secret +that is not committed to this repository. + +Migration procedure once a provider is selected: + +1. Back up Vault Raft storage and verify the snapshot is recoverable. +2. Configure the new seal stanza and provider identity/credentials. +3. Follow HashiCorp's seal-migration procedure; do not simply replace the + Shamir stanza and restart every node. +4. Migrate one member at a time and preserve Raft quorum. +5. Restart a migrated member and verify it unseals without operator key entry. +6. Verify all three members, HA leadership, Raft replication, telemetry, and + External Secrets consumers. +7. Only after successful migration consider changing the StatefulSet rollout + strategy. Keeping `OnDelete` remains acceptable even with auto-unseal when + deliberate Vault upgrades are preferred. + +Until an independent seal provider is chosen and provisioned, Shamir remains +the intentional seal mechanism.