diff --git a/docs/examples/cassandra/monitoring/cas-alert-demo.yaml b/docs/examples/cassandra/monitoring/cas-alert-demo.yaml new file mode 100644 index 0000000000..063470e505 --- /dev/null +++ b/docs/examples/cassandra/monitoring/cas-alert-demo.yaml @@ -0,0 +1,26 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Cassandra +metadata: + name: cas-alert-demo + namespace: demo +spec: + version: 5.0.3 + topology: + rack: + - name: r0 + replicas: 2 + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageType: Durable + deletionPolicy: WipeOut + monitor: + agent: "prometheus.io/operator" + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/clickhouse/monitoring/clickhouse-alert-demo.yaml b/docs/examples/clickhouse/monitoring/clickhouse-alert-demo.yaml new file mode 100644 index 0000000000..6f4a672269 --- /dev/null +++ b/docs/examples/clickhouse/monitoring/clickhouse-alert-demo.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: ClickHouse +metadata: + name: clickhouse-alert-demo + namespace: alert-clickhouse +spec: + version: "24.4.1" + replicas: 1 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/elasticsearch/monitoring/es-alert.yaml b/docs/examples/elasticsearch/monitoring/es-alert.yaml new file mode 100644 index 0000000000..8fe4218400 --- /dev/null +++ b/docs/examples/elasticsearch/monitoring/es-alert.yaml @@ -0,0 +1,43 @@ +apiVersion: kubedb.com/v1 +kind: Elasticsearch +metadata: + name: es-alert + namespace: alert-elasticsearch +spec: + version: xpack-9.2.3 + deletionPolicy: WipeOut + topology: + master: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + data: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + ingest: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/hazelcast/monitoring/hazelcast-alert-demo.yaml b/docs/examples/hazelcast/monitoring/hazelcast-alert-demo.yaml new file mode 100644 index 0000000000..4126b36799 --- /dev/null +++ b/docs/examples/hazelcast/monitoring/hazelcast-alert-demo.yaml @@ -0,0 +1,25 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Hazelcast +metadata: + name: hazelcast-alert-demo + namespace: alert-hazelcast +spec: + version: "5.5.2" + replicas: 1 + licenseSecret: + name: hz-license-key + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/ignite/monitoring/ignite-alert-demo.yaml b/docs/examples/ignite/monitoring/ignite-alert-demo.yaml new file mode 100644 index 0000000000..5bd4c02286 --- /dev/null +++ b/docs/examples/ignite/monitoring/ignite-alert-demo.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Ignite +metadata: + name: ignite-alert-demo + namespace: alert-ignite +spec: + replicas: 1 + version: "2.17.0" + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/kafka/monitoring/kafka-alert-demo.yaml b/docs/examples/kafka/monitoring/kafka-alert-demo.yaml new file mode 100644 index 0000000000..f231b22cc6 --- /dev/null +++ b/docs/examples/kafka/monitoring/kafka-alert-demo.yaml @@ -0,0 +1,24 @@ +apiVersion: kubedb.com/v1 +kind: Kafka +metadata: + name: kafka-alert-demo + namespace: alert-kafka +spec: + replicas: 1 + version: "3.9.0" + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/mariadb/monitoring/mariadb-alert-demo.yaml b/docs/examples/mariadb/monitoring/mariadb-alert-demo.yaml new file mode 100644 index 0000000000..298c8f5ea0 --- /dev/null +++ b/docs/examples/mariadb/monitoring/mariadb-alert-demo.yaml @@ -0,0 +1,24 @@ +apiVersion: kubedb.com/v1 +kind: MariaDB +metadata: + name: mariadb-alert-demo + namespace: alert-mariadb +spec: + version: "11.5.2" + replicas: 1 + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/mongodb/monitoring/mongodb-alert-demo.yaml b/docs/examples/mongodb/monitoring/mongodb-alert-demo.yaml new file mode 100644 index 0000000000..96fbd15dbb --- /dev/null +++ b/docs/examples/mongodb/monitoring/mongodb-alert-demo.yaml @@ -0,0 +1,26 @@ +apiVersion: kubedb.com/v1 +kind: MongoDB +metadata: + name: mongodb-alert-demo + namespace: alert-mongodb +spec: + version: "8.0.17" + replicaSet: + name: "rs1" + replicas: 3 + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/mssqlserver/monitoring/mssqlserver-alert-demo.yaml b/docs/examples/mssqlserver/monitoring/mssqlserver-alert-demo.yaml new file mode 100644 index 0000000000..167b221826 --- /dev/null +++ b/docs/examples/mssqlserver/monitoring/mssqlserver-alert-demo.yaml @@ -0,0 +1,39 @@ +apiVersion: kubedb.com/v1alpha2 +kind: MSSQLServer +metadata: + name: mssqlserver-alert-demo + namespace: alert-mssqlserver +spec: + version: "2025-cu0" + replicas: 1 + storageType: Durable + tls: + issuerRef: + name: mssqlserver-ca-issuer + kind: Issuer + apiGroup: "cert-manager.io" + clientTLS: false + podTemplate: + spec: + containers: + - name: mssql + env: + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_PID + value: Evaluation + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/mssqlserver/monitoring/mssqlserver-ca-issuer.yaml b/docs/examples/mssqlserver/monitoring/mssqlserver-ca-issuer.yaml new file mode 100644 index 0000000000..5886814450 --- /dev/null +++ b/docs/examples/mssqlserver/monitoring/mssqlserver-ca-issuer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: mssqlserver-ca-issuer + namespace: alert-mssqlserver +spec: + ca: + secretName: mssqlserver-ca diff --git a/docs/examples/mysql/monitoring/mysql-alert-alert-mysql.yaml b/docs/examples/mysql/monitoring/mysql-alert-alert-mysql.yaml new file mode 100644 index 0000000000..0ec35a5f0f --- /dev/null +++ b/docs/examples/mysql/monitoring/mysql-alert-alert-mysql.yaml @@ -0,0 +1,22 @@ +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: mysql-alert-alert-mysql + namespace: alert-mysql +spec: + version: "8.4.8" + deletionPolicy: WipeOut + storage: + storageClassName: "longhorn" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus diff --git a/docs/examples/neo4j/monitoring/neo4j-alert-demo.yaml b/docs/examples/neo4j/monitoring/neo4j-alert-demo.yaml new file mode 100644 index 0000000000..51c56ebf94 --- /dev/null +++ b/docs/examples/neo4j/monitoring/neo4j-alert-demo.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-alert-demo + namespace: alert-neo4j +spec: + replicas: 3 + version: "2025.12.1" + deletionPolicy: WipeOut + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus diff --git a/docs/examples/percona-xtradb/monitoring/perconaxtradb-alert-demo.yaml b/docs/examples/percona-xtradb/monitoring/perconaxtradb-alert-demo.yaml new file mode 100644 index 0000000000..eff58cb3ae --- /dev/null +++ b/docs/examples/percona-xtradb/monitoring/perconaxtradb-alert-demo.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1 +kind: PerconaXtraDB +metadata: + name: perconaxtradb-alert-demo + namespace: alert-perconaxtradb +spec: + version: "8.0.40" + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/pgbouncer/monitoring/pg-backend-alert.yaml b/docs/examples/pgbouncer/monitoring/pg-backend-alert.yaml new file mode 100644 index 0000000000..a7eed078e0 --- /dev/null +++ b/docs/examples/pgbouncer/monitoring/pg-backend-alert.yaml @@ -0,0 +1,17 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-backend-alert + namespace: alert-pgbouncer +spec: + version: "16.13" + replicas: 1 + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut diff --git a/docs/examples/pgbouncer/monitoring/pgbouncer-alert.yaml b/docs/examples/pgbouncer/monitoring/pgbouncer-alert.yaml new file mode 100644 index 0000000000..1cef89a5a4 --- /dev/null +++ b/docs/examples/pgbouncer/monitoring/pgbouncer-alert.yaml @@ -0,0 +1,25 @@ +apiVersion: kubedb.com/v1 +kind: PgBouncer +metadata: + name: pgbouncer-alert + namespace: alert-pgbouncer +spec: + version: "1.23.1" + replicas: 1 + database: + syncUsers: true + databaseName: "postgres" + databaseRef: + name: "pg-backend-alert" + namespace: alert-pgbouncer + connectionPool: + maxClientConnections: 20 + reservePoolSize: 5 + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/pgpool/monitoring/pg-backend-alert.yaml b/docs/examples/pgpool/monitoring/pg-backend-alert.yaml new file mode 100644 index 0000000000..a4d9d949c2 --- /dev/null +++ b/docs/examples/pgpool/monitoring/pg-backend-alert.yaml @@ -0,0 +1,21 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-backend-alert + namespace: alert-pgpool +spec: + version: "16.13" + replicas: 1 + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + configuration: + inline: + user.conf: | + max_connections=200 + deletionPolicy: WipeOut diff --git a/docs/examples/pgpool/monitoring/pgpool-alert.yaml b/docs/examples/pgpool/monitoring/pgpool-alert.yaml new file mode 100644 index 0000000000..1040b05bfe --- /dev/null +++ b/docs/examples/pgpool/monitoring/pgpool-alert.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Pgpool +metadata: + name: pgpool-alert + namespace: alert-pgpool +spec: + version: "4.5.3" + postgresRef: + name: pg-backend-alert + namespace: alert-pgpool + configuration: + inline: + pgpool.conf: | + num_init_children=5 + max_pool=15 + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/proxysql/monitoring/my-group-alert.yaml b/docs/examples/proxysql/monitoring/my-group-alert.yaml new file mode 100644 index 0000000000..116353ba60 --- /dev/null +++ b/docs/examples/proxysql/monitoring/my-group-alert.yaml @@ -0,0 +1,21 @@ +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: my-group-alert + namespace: alert-proxysql +spec: + version: "9.1.0" + replicas: 3 + topology: + mode: GroupReplication + group: + name: "dc002fc3-c412-4d18-b1d4-66c1fbfbbc9b" + storageType: Durable + storage: + storageClassName: "longhorn" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut diff --git a/docs/examples/proxysql/monitoring/proxysql-alert.yaml b/docs/examples/proxysql/monitoring/proxysql-alert.yaml new file mode 100644 index 0000000000..9898a72ec6 --- /dev/null +++ b/docs/examples/proxysql/monitoring/proxysql-alert.yaml @@ -0,0 +1,20 @@ +apiVersion: kubedb.com/v1 +kind: ProxySQL +metadata: + name: proxysql-alert + namespace: alert-proxysql +spec: + version: "2.3.2-debian" + replicas: 1 + backend: + name: my-group-alert + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + exporter: + port: 42004 + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/rabbitmq/monitoring/rmq-alert-demo.yaml b/docs/examples/rabbitmq/monitoring/rmq-alert-demo.yaml new file mode 100644 index 0000000000..52befe9740 --- /dev/null +++ b/docs/examples/rabbitmq/monitoring/rmq-alert-demo.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: RabbitMQ +metadata: + name: rmq-alert-demo + namespace: alert-rabbitmq +spec: + version: "4.2.4" + replicas: 3 + deletionPolicy: WipeOut + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/redis/monitoring/rd-alert-demo.yaml b/docs/examples/redis/monitoring/rd-alert-demo.yaml new file mode 100644 index 0000000000..2f43358780 --- /dev/null +++ b/docs/examples/redis/monitoring/rd-alert-demo.yaml @@ -0,0 +1,22 @@ +apiVersion: kubedb.com/v1 +kind: Redis +metadata: + name: rd-alert-demo + namespace: alert-redis +spec: + version: "6.0.20" + deletionPolicy: WipeOut + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus diff --git a/docs/examples/singlestore/monitoring/singlestore-alert-demo.yaml b/docs/examples/singlestore/monitoring/singlestore-alert-demo.yaml new file mode 100644 index 0000000000..4924979dda --- /dev/null +++ b/docs/examples/singlestore/monitoring/singlestore-alert-demo.yaml @@ -0,0 +1,37 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: singlestore-alert-demo + namespace: alert-singlestore +spec: + version: "8.9.3" + storageType: Durable + topology: + aggregator: + replicas: 1 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + leaf: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s diff --git a/docs/examples/solr/monitoring/solr-alert-demo.yaml b/docs/examples/solr/monitoring/solr-alert-demo.yaml new file mode 100644 index 0000000000..b495d750a1 --- /dev/null +++ b/docs/examples/solr/monitoring/solr-alert-demo.yaml @@ -0,0 +1,30 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Solr +metadata: + name: solr-alert + namespace: alert-solr +spec: + version: "9.8.0" + replicas: 3 + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s + solrModules: + - s3-repository + - gcs-repository + - prometheus-exporter + zookeeperRef: + name: zoo + namespace: alert-solr + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: local-path + deletionPolicy: WipeOut diff --git a/docs/examples/solr/monitoring/zookeeper-alert-demo.yaml b/docs/examples/solr/monitoring/zookeeper-alert-demo.yaml new file mode 100644 index 0000000000..234911a762 --- /dev/null +++ b/docs/examples/solr/monitoring/zookeeper-alert-demo.yaml @@ -0,0 +1,17 @@ +apiVersion: kubedb.com/v1alpha2 +kind: ZooKeeper +metadata: + name: zoo + namespace: alert-solr +spec: + version: "3.8.3" + replicas: 3 + deletionPolicy: WipeOut + adminServerPort: 8080 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "100Mi" diff --git a/docs/examples/zookeeper/monitoring/zk-alert-demo.yaml b/docs/examples/zookeeper/monitoring/zk-alert-demo.yaml new file mode 100644 index 0000000000..a5ace0b079 --- /dev/null +++ b/docs/examples/zookeeper/monitoring/zk-alert-demo.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: ZooKeeper +metadata: + name: zk-alert-demo + namespace: demo +spec: + version: "3.8.3" + replicas: 3 + storage: + resources: + requests: + storage: "1Gi" + storageClassName: "standard" + accessModes: + - ReadWriteOnce + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus diff --git a/docs/guides/cassandra/monitoring/alerting.md b/docs/guides/cassandra/monitoring/alerting.md new file mode 100644 index 0000000000..44d8abc6a3 --- /dev/null +++ b/docs/guides/cassandra/monitoring/alerting.md @@ -0,0 +1,498 @@ +--- +title: Cassandra Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: cas-monitoring-alerting + name: Alerting + parent: cas-monitoring-cassandra + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Cassandra Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Cassandra cluster using the `cassandra-alerts` Helm chart. Like `neo4j-alerts`, this chart also bundles a Grafana dashboard that it imports automatically through a post-install Job — no separate dashboard chart is required. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-cas` namespace: + + ```bash + $ kubectl create ns alert-cas + namespace/alert-cas created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/cassandra/monitoring/overview.md). + +* You will also need a Grafana API key / token with **Editor** permission so the chart's dashboard-import Job can push the dashboard. See [Step 1](#step-1--create-a-grafana-api-key) below. + +> Note: YAML files used in this tutorial are stored in [docs/examples/cassandra](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/cassandra) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Cassandra with the [JMX Exporter](https://github.com/prometheus/jmx_exporter)-based `exporter` sidecar container, which scrapes the Cassandra JVM's JMX metrics and exposes them as Prometheus metrics on port `8080`. +- **Stats Service** (named `{cassandra-name}-stats`) is created automatically by KubeDB and fronts the exporter's metrics endpoint on port `56790`, which is proxied to the exporter's actual listening port `8080` inside the pod. +- **ServiceMonitor** (named `{cassandra-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `cassandra-alerts` chart and contains all Cassandra alert definitions grouped by concern: database health and provisioner. +- **Dashboard-import Job** — when `grafana.enabled` is `true` (default `false`), the chart also creates a one-shot `Job` that `POST`s a bundled dashboard JSON straight to your Grafana instance's `/api/dashboards/import` endpoint. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy Cassandra with Monitoring Enabled + +At first, let's deploy a Cassandra cluster with monitoring enabled. Below is the Cassandra object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Cassandra +metadata: + name: cas-alert-demo + namespace: alert-cas +spec: + version: 5.0.7 + topology: + rack: + - name: r0 + replicas: 2 + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageType: Durable + deletionPolicy: WipeOut + monitor: + agent: "prometheus.io/operator" + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.topology.rack[].replicas: 2` — Cassandra requires more than one replica per rack for its admission webhook to accept the object, so this demo uses the smallest allowed topology (1 rack, 2 replicas). +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the Cassandra resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/cassandra/monitoring/cas-alert-demo.yaml +cassandra.kubedb.com/cas-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get cassandra -n alert-cas cas-alert-demo +NAME VERSION STATUS AGE +cas-alert-demo 5.0.7 Ready 5m +``` + +KubeDB brings up 2 rack pods: + +```bash +$ kubectl get pods -n alert-cas +NAME READY STATUS RESTARTS AGE +cas-alert-demo-rack-r0-0 2/2 Running 0 5m +cas-alert-demo-rack-r0-1 2/2 Running 0 4m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-cas --selector="app.kubernetes.io/instance=cas-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cas-alert-demo ClusterIP 10.43.120.17 9042/TCP,7000/TCP,7199/TCP,7001/TCP 5m +cas-alert-demo-rack-r0-pods ClusterIP None 9042/TCP,7000/TCP,7199/TCP,7001/TCP 5m +cas-alert-demo-stats ClusterIP 10.43.90.186 56790/TCP 5m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-cas +NAME AGE +cas-alert-demo-stats 5m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-cas cas-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Create a Grafana API Key + +The chart's dashboard-import Job authenticates to Grafana with a bearer token, so create one first. + +* **Grafana 9+**: **Administration → Service accounts → Add service account** → role **Editor** → **Add token**. Copy the token. +* **Grafana 8.x and earlier** (no Service Accounts UI, e.g. the bundled `kube-prometheus-stack` Grafana 7.5.5 used while verifying this tutorial): use the legacy **API Keys** endpoint instead: + + ```bash + # Port-forward Grafana + $ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 + + # Retrieve the admin password + $ kubectl get secret -n monitoring prometheus-grafana \ + -o jsonpath='{.data.admin-password}' | base64 -d && echo + + # Create a service account with Editor role + $ curl -s -X POST -H "Content-Type: application/json" \ + -u admin: \ + http://localhost:3000/api/serviceaccounts \ + -d '{"name":"cas-alerts-demo","role":"Editor"}' + # Note the returned "id" + + # Create a token for the service account (replace with the returned service account ID) + $ curl -s -X POST -H "Content-Type: application/json" \ + -u admin: \ + http://localhost:3000/api/serviceaccounts//tokens \ + -d '{"name":"cas-alerts-demo-key","secondsToLive":0}' + # Note the returned "key" + + # Stop the port-forward + $ kill %1 + ``` + +Either way, you end up with a bearer token to use as `grafana.apikey` below. + +## Step 2 — Install cassandra-alerts + +The `cassandra-alerts` chart creates a `PrometheusRule` resource containing all Cassandra alert definitions, **and** (when `grafana.enabled=true`) a `Job` that imports a pre-built Grafana dashboard. + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the Cassandra object's name (`cas-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +> **Note:** The chart's default values leave `form.alert.groups.database.rules.cassandraDown.val` unset, which renders an incomplete PromQL expression (`... > `) that the `PrometheusRule` admission webhook rejects. Explicitly set it to `0` at install time — the `CassandraDown` expression counts down instances and fires whenever that count is greater than `0`. + +### Install + +```bash +$ helm upgrade -i cas-alert-demo appscode/cassandra-alerts \ + -n alert-cas \ + --create-namespace \ + --version=v2026.7.14 +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `cas-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`job="cas-alert-demo-stats"`) | +| `-n alert-cas` | `alert-cas` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | +| `form.alert.groups.database.rules.cassandraDown.val` | `0` | Fixes the chart's missing default so the `CassandraDown` expression is valid PromQL | +| `grafana.url` | in-cluster Grafana URL | The dashboard-import Job runs **inside the cluster**, so this must be a cluster-internal address, not `localhost` | +| `grafana.apikey` | token from Step 1 | Authenticates the dashboard-import `POST` request | +| `grafana.jobName` | `cas-alert-demo-stats` | **Required** — the chart's default (`kubedb-databases`) doesn't match any real Prometheus job, so half the dashboard's panels silently show "No data" unless you override it to your instance's actual stats-service name. See the caveat below. | + +> To install **alerts only, without the dashboard**, set `--set grafana.enabled=false`. + +> **Chart limitation found while verifying this tutorial:** even with `grafana.jobName` set correctly, the dashboard's **title** and its two "Cassandra Phase" panels (`KubeDB Cassandra Phase Not Ready` / `Critical`) remain hardcoded to the literal strings `demo` and `cassandra` inside the chart's bundled dashboard JSON — confirmed by rendering the chart with `helm template` against different release names/namespaces and observing the title and those two panels' PromQL (`kubedb_com_cassandra_status_phase{app="...",namespace="demo",...}`) never change. There is no values field that fixes this. Practically: the dashboard will always be titled `kubedb.com / Cassandra / demo / cassandra` in Grafana's UI regardless of your actual release/namespace, and the two Phase panels will only ever show data if you happen to deploy in a namespace literally named `demo`. The `Cassandra Server Status` row (six panels driven by the exporter's own metrics) is correctly scoped by `grafana.jobName` and does **not** have this problem. +> +> **Also note:** the dashboard-import payload has `overwrite: false` hardcoded (also not exposed as a value). Combined with the fixed title above, re-running the import Job (e.g. after changing `grafana.jobName`, or on a `helm upgrade`) fails with `"A dashboard with the same name in the folder already exists"` unless you first delete the existing one: `curl -s -X DELETE -H "Authorization: Bearer " http://localhost:3000/api/dashboards/uid/` (and `kubectl delete job -n alert-cas cas-alert-demo-post-job` so the Job re-runs on upgrade). + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-cas +NAME AGE +cas-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-cas cas-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Verify the dashboard-import Job + +```bash +$ kubectl get job -n alert-cas +NAME STATUS COMPLETIONS AGE +cas-alert-demo-post-job Complete 1/1 8s + +$ kubectl logs -n alert-cas job/cas-alert-demo-post-job +{"pluginId":"","title":"kubedb.com / Cassandra / demo / cassandra","imported":true, ...} +``` + +A `"imported":true` response confirms the dashboard now exists in Grafana — under the literal title `kubedb.com / Cassandra / demo / cassandra` regardless of this instance's real name/namespace (see the chart-limitation note above). + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=cassandra`. + +

+ Prometheus Rule Health +

+ +Both the `cassandra.database.alert-cas.cas-alert-demo.rules` and `cassandra.provisioner.alert-cas.cas-alert-demo.rules` groups are visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Cassandra alert definitions every 30 seconds. + +--- + +## Verify End-to-End + +### 1. Check the exporter is running + +The `exporter` sidecar inside the Cassandra pod scrapes JMX and serves metrics at `:8080/metrics`. A metric named `java:lang:runtime:uptime` confirms the exporter can reach the Cassandra JVM. + +```bash +$ kubectl exec -n alert-cas cas-alert-demo-rack-r0-0 -c cassandra -- \ + curl -s localhost:8080/metrics | grep 'name="java:lang:runtime:uptime"' +cassandra_stats{cluster="Test Cluster",datacenter="dc1",keyspace="",table="",name="java:lang:runtime:uptime",} 1867282.0 +``` + +> The `exporter` container's own image ships neither `curl` nor `wget`, so the check above runs from the `cassandra` container instead — both containers share the pod's network namespace, so `localhost:8080` reaches the exporter's HTTP server either way. + +### 2. Check the Prometheus target is UP + +Open `http://localhost:9090/targets?search=cas-alert-demo`. + +

+ Prometheus Target UP +

+ +The target `serviceMonitor/alert-cas/cas-alert-demo-stats/0` shows **2 / 2 up**, confirming metrics are being scraped from both `cas-alert-demo-rack-r0-0` and `cas-alert-demo-rack-r0-1` in the `alert-cas` namespace. + +### 3. Confirm all Cassandra alerts are inactive + +Open `http://localhost:9090/alerts?search=cassandra` to see the Cassandra alert groups. + +

+ Prometheus Alerts — All Inactive +

+ +All 6 rules in the `cassandra.database` group and both rules in the `cassandra.provisioner` group show **INACTIVE**, meaning the cluster is healthy and no thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy Cassandra cluster, no alerts for `cas-alert-demo` will be listed here. + +

+ AlertManager +

+ +### 5. Explore the Grafana dashboard + +Port-forward Grafana and log in. + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 +``` + +Open `http://localhost:3000` and navigate to the dashboard `kubedb.com / Cassandra / demo / cassandra` that the Job imported in Step 2 (search doesn't help here since the title never changes — see the chart-limitation note above; find it by `dashboardId`/`uid` from the Job's log output if you have several). + +

+ Grafana — Cassandra Alerts Dashboard +

+ +The dashboard mirrors the alert groups: **Cassandra Phase** (Not Ready / Critical — both show "No data" here since this tutorial deploys to `alert-cas`, not the hardcoded `demo` namespace the panels are wired to) and **Cassandra Server Status** (Down, Service Respawn, Connection Timeout, Dropped Messages, High Read/Write Latency — all live and correctly scoped once `grafana.jobName` is set as shown above). The **Cassandra Down** panel's flat, healthy line is the query `count(up{job="cas-alert-demo-stats",namespace="alert-cas"}) OR vector(0)` — note the `OR vector(0)` fallback means this panel reads as "0 down" both when the instance is genuinely healthy *and* when the job name is wrong and matches nothing, so don't mistake a flat 0 line for confirmation the wiring is correct — cross-check against the Prometheus target/alert pages above. + +--- + +## Simulating a Firing Alert + +The previous section confirmed that all alerts are **INACTIVE** while the cluster is healthy. This section walks through deliberately triggering the `CassandraDown` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +### 1. Stop the metrics endpoint + +Unlike some other KubeDB charts, `CassandraDown` is **not** driven by a custom exporter-reported gauge — it is built directly on Prometheus's own scrape-health metric, `up{job="cas-alert-demo-stats"}`. That metric only goes to `0` when Prometheus can no longer reach the scrape target's HTTP endpoint at all. + +On this build, the exporter's HTTP server (the process actually scraped by Prometheus on port `8080`) runs inside the `exporter` container — killing the `cassandra` container alone leaves the exporter's HTTP endpoint reachable (Prometheus would keep scraping it successfully), so `up` would stay `1` and `CassandraDown` would never fire. To reproduce a real scrape failure, stop the `exporter` container's process instead: + +```bash +$ end=$(( $(date +%s) + 45 )) +while [ $(date +%s) -lt $end ]; do + kubectl exec -n alert-cas cas-alert-demo-rack-r0-0 -c exporter -- kill 1 >/dev/null 2>&1 + sleep 1 +done + +``` + +Kubernetes restarts the crashed `exporter` container in the background. The restart is quick, so the outage window can be short — if the alert resolves before you finish inspecting it, repeat the `kill 1` command a few times in a row; Kubernetes' crash-loop backoff will keep the container down for a longer stretch on subsequent attempts, giving you a wider window to observe the firing state. + +Wait 30–60 seconds for the next Prometheus scrape cycle (configured at 10s) and rule-evaluation cycle (30s) to register the failure. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=cassandra`. + +

+ Prometheus Alerts — CassandraDown Firing +

+ +Because `CassandraDown` has `for: 0m` (instant), it moves directly from **INACTIVE** to **FIRING** within one evaluation cycle, while the rest of the `cassandra.database` group stays **INACTIVE**. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter={app_namespace="alert-cas"}`. + +

+ AlertManager — CassandraDown Firing +

+ +AlertManager shows the `CassandraDown` alert. The alert card displays labels including: + +- **alertname**: `CassandraDown` +- **severity**: `critical` +- **app**: `cas-alert-demo`, **app_namespace**: `alert-cas` +- **job**: `cas-alert-demo-stats` + +> Note: this chart's alert labels use `app_namespace` rather than a plain `namespace` label — filter or group on `app_namespace` when searching for these alerts in AlertManager. + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore Cassandra + +Delete the pod so KubeDB recreates it cleanly. + +```bash +$ kubectl delete pod -n alert-cas cas-alert-demo-rack-r0-1 +``` + +Once the exporter's `/metrics` endpoint is reachable again, Prometheus marks the alert **INACTIVE** and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `cas-alert-demo` instance in the `alert-cas` namespace via the PromQL label filters `job="cas-alert-demo-stats"` and `app_namespace="alert-cas"`. + +### Database Group + +Fired based on live metrics from the Cassandra JMX exporter. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `CassandraDown` | critical | instant | The Prometheus scrape target for this instance is unreachable — the exporter's metrics endpoint is down or the pod is unreachable. | +| `CassandraServiceRespawn` | critical | instant | Cassandra restarted recently (JVM uptime < 180s). | +| `ConnectionTimeouts` | warning | instant | More than 100 connection timeouts observed in the last minute. | +| `DroppedMessages` | warning | instant | One or more internal Cassandra messages have been dropped — a sign of overload or backpressure. | +| `HighReadLatency` | warning | instant | 99th-percentile coordinator read latency on the health-check table exceeds 7000 (µs). | +| `HighWriteLatency` | warning | instant | 99th-percentile coordinator write latency on the health-check table exceeds 7000 (µs). | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Cassandra resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBCassandraPhaseNotReady` | critical | 1m | KubeDB marked the Cassandra resource `NotReady` — the operator cannot reach the cluster. | +| `KubeDBCassandraPhaseCritical` | warning | 5m | The instance is in a degraded/critical phase. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + cassandraDown: + enabled: true + duration: "0m" + val: 0 + cassandraHighReadLatency: + enabled: true + duration: "1m" + val: 15000 # allow up to 15ms 99th-percentile read latency + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade cas-alert-demo appscode/cassandra-alerts \ + -n alert-cas \ + --version=v2026.7.14 \ + --set grafana.enabled=false \ + -f custom-alerts.yaml +``` + +> Note: `-f` values files don't merge `grafana.url`/`grafana.apikey`/`grafana.jobName` automatically — re-pass them (or set `grafana.enabled=false`) on every `helm upgrade`, otherwise the dashboard-import Job re-runs with an empty URL/token and fails, or re-imports with the broken default `jobName` again. + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the cassandra-alerts release (PrometheusRule + dashboard-import Job) +$ helm uninstall cas-alert-demo -n alert-cas + +# Remove the imported Grafana dashboard (it is not removed by helm uninstall) +$ curl -s -X DELETE -H "Authorization: Bearer " \ + http://localhost:3000/api/dashboards/uid/ + +# Remove the Cassandra instance +$ kubectl delete cassandra -n alert-cas cas-alert-demo + +# Delete namespace +$ kubectl delete ns alert-cas +``` + +## Next Steps + +- Monitor your Cassandra cluster with KubeDB using [builtin Prometheus](/docs/guides/cassandra/monitoring/using-builtin-prometheus.md). +- Monitor your Cassandra cluster with KubeDB using [Prometheus operator](/docs/guides/cassandra/monitoring/using-prometheus-operator.md). +- Visualise Cassandra metrics with [Grafana Dashboard](grafana-dashboard.md). +- Learn how to use KubeDB to run a Apache Cassandra cluster [here](/docs/guides/cassandra/README.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/clickhouse/monitoring/alerting.md b/docs/guides/clickhouse/monitoring/alerting.md new file mode 100644 index 0000000000..f317a556a7 --- /dev/null +++ b/docs/guides/clickhouse/monitoring/alerting.md @@ -0,0 +1,398 @@ +--- +title: ClickHouse Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: ch-monitoring-alerting + name: Alerting + parent: ch-monitoring-clickhouse + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# ClickHouse Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed ClickHouse instance using the `clickhouse-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-clickhouse` namespace: + + ```bash + $ kubectl create ns alert-clickhouse + namespace/alert-clickhouse created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/clickhouse/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/clickhouse](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/clickhouse) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys ClickHouse with metrics exposed by a [JMX Exporter](https://github.com/prometheus/jmx_exporter)-style Java agent running **inside the `clickhouse` container itself** on port `9363` — there is no separate exporter sidecar container, and only one container runs in the pod. +- **ServiceMonitor** (named `{clickhouse-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the metrics endpoint every 10 seconds. +- **PrometheusRule** is created by the `clickhouse-alerts` chart and contains ClickHouse alert definitions grouped by concern: database health, provisioner, ops-manager, and KubeStash backup/restore. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy ClickHouse with Monitoring Enabled + +Below is the ClickHouse object we are going to create — a single-node instance with monitoring enabled. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: ClickHouse +metadata: + name: clickhouse-alert-demo + namespace: alert-clickhouse +spec: + version: "26.2.6" + replicas: 1 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` matches the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/clickhouse/monitoring/clickhouse-alert-demo.yaml +clickhouse.kubedb.com/clickhouse-alert-demo created +``` + +Wait for the database to go into `Ready` state. + +```bash +$ kubectl get clickhouse -n alert-clickhouse clickhouse-alert-demo +NAME VERSION STATUS AGE +clickhouse-alert-demo 24.4.1 Ready 3m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-clickhouse --selector="app.kubernetes.io/instance=clickhouse-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +clickhouse-alert-demo ClusterIP 10.43.102.159 9000/TCP,8123/TCP 3m +clickhouse-alert-demo-pods ClusterIP None 9000/TCP,8123/TCP 3m +clickhouse-alert-demo-stats ClusterIP 10.43.76.0 9363/TCP 3m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-clickhouse +NAME AGE +clickhouse-alert-demo-stats 3m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-clickhouse clickhouse-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install clickhouse-alerts + +The `clickhouse-alerts` chart creates a `PrometheusRule` resource containing all ClickHouse alert definitions. + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the ClickHouse object's name (`clickhouse-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time tkubectl patch petset -n alert-solr solr-alert \ + --type=merge \ + -p '{"spec":{"replicas":0}}'o match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i clickhouse-alert-demo appscode/clickhouse-alerts \ + -n alert-clickhouse \ + --create-namespace \ + --version=v2026.7.14 +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `clickhouse-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`job="clickhouse-alert-demo-stats"`, `app="clickhouse-alert-demo"`) | +| `-n alert-clickhouse` | `alert-clickhouse` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +> Don't bother with `--set grafana.enabled=true` — as explained at the top of this tutorial, this chart version doesn't actually render anything for it. + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-clickhouse +NAME AGE +clickhouse-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-clickhouse clickhouse-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=clickhouse`. + +

+ Prometheus Rule Health +

+ +All four groups — `clickhouse.database`, `clickhouse.provisioner`, `clickhouse.opsManager`, and `clickhouse.kubeStash` — are visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the ClickHouse alert definitions every 30 seconds. The `database` group's `DiskUsageHigh`/`DiskAlmostFull` rules correctly divide by `kubelet_volume_stats_capacity_bytes` in this chart — unlike the same-named rules in a few other `*-alerts` charts in this project, they don't have the false-firing PVC-usage bug. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-clickhouse%22%7D&g0.tab=1`. + +

+ Prometheus up query — clickhouse-alert-demo-0 UP +

+ +The `clickhouse-alert-demo-0` pod reports `up == 1` via the `clickhouse-alert-demo-stats` service/job on port `9363`, confirming Prometheus is scraping the JMX agent successfully. + +### 2. Confirm the ClickHouse alerts are inactive + +Open `http://localhost:9090/alerts?search=clickhouse`. + +

+ Prometheus Alerts — ClickHouse groups inactive +

+ +All rules across the `clickhouse.database`, `clickhouse.provisioner`, `clickhouse.opsManager`, and `clickhouse.kubeStash` groups show **INACTIVE**, confirming the instance is healthy and no thresholds are breached. (`clickhouse.kubeStash` rules stay INACTIVE with no data unless KubeStash backups are configured on this instance.) + +### 3. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +No alerts are firing for the `alert-clickhouse` namespace. + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `ClickhouseInstanceDown` so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +The `clickhouse` container's `PID 1` is the `clickhouse-server` process itself, unlike most other KubeDB images in this project where `PID 1` is a supervisor (`tini`, a wrapper script) around the real database process. This matters: **`kubectl exec ... kill -9 1` does nothing here.** A container's `PID 1` is also PID 1 of that container's own PID namespace, and Linux unconditionally ignores `SIGKILL`/`SIGSTOP` sent to a namespace's PID 1 *from within that same namespace* (`kubectl exec` attaches to the same namespace) — confirmed by checking `readlink /proc/1/ns/pid` and `/proc/self/ns/pid` inside the container (identical), then observing `kill -9 1` return exit code `0` while `/proc/1`'s elapsed-time counter kept climbing, completely unaffected. This is a kernel-level protection, not a bug — only a signal delivered from *outside* the namespace (e.g. the container runtime stopping the container) can kill a namespace's PID 1 this way. + +**What works instead:** ask ClickHouse to shut itself down via SQL — `SYSTEM SHUTDOWN` — which exits the process voluntarily rather than relying on an external signal, and Kubernetes restarts the container normally afterward. A single shutdown recovers in only a few seconds (too fast to reliably observe), so loop it from *outside* the container: each `SYSTEM SHUTDOWN` call takes down the very `kubectl exec` session that issued it (its container just exited), so the retry has to be a fresh `kubectl exec` per iteration, not a loop running inside one exec session. + +### 1. Crash the ClickHouse process repeatedly + +```bash +$ end=$(( $(date +%s) + 150 )) +while [ $(date +%s) -lt $end ]; do + kubectl exec -n alert-clickhouse clickhouse-alert-demo-0 -c clickhouse -- kill 1 + sleep 1 +done +``` + +Retrieve the password first if you don't have it: `kubectl get secret -n alert-clickhouse clickhouse-alert-demo-auth -o jsonpath='{.data.password}' | base64 -d`. Run the loop in the background (or a separate terminal) — each iteration either succeeds (shutting the instance down again) or fails harmlessly while a previous shutdown is still restarting, so 90 seconds comfortably holds the instance in a crash loop. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=clickhouse`. + +

+ Prometheus Alerts — ClickhouseInstanceDown Firing +

+ +`ClickhouseInstanceDown` (`up{job="clickhouse-alert-demo-stats"} == 0`, `for: 1m`) transitions from **INACTIVE** to **FIRING** once the crash loop has kept the scrape target down continuously for the full window, while the rest of the `clickhouse.database` group stays **INACTIVE**. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter={namespace="alert-clickhouse"}`. + +

+ AlertManager — ClickhouseInstanceDown Firing +

+ +AlertManager shows the `ClickhouseInstanceDown` alert. The alert card displays: + +- **Severity**: `critical` +- **pod**: `clickhouse-alert-demo-0` +- **job**: `clickhouse-alert-demo-stats` +- **namespace** / **app_namespace**: `alert-clickhouse` +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore ClickHouse + +Let the loop from step 1 finish (or stop it early) — the pod recovers on its own once no further shutdowns land. + +```bash +$ kubectl get pods -n alert-clickhouse +NAME READY STATUS RESTARTS AGE +clickhouse-alert-demo-0 1/1 Running 5 18m +``` + +Once the pod is stably `1/1 Running` and the next scrape reports `up == 1`, Prometheus marks the alert **INACTIVE** again (took about a minute after the loop ended in testing, since the `for: 1m` window has to elapse clean) and AlertManager sends a **resolved** notification to all receivers. If the pod doesn't stabilize on its own, force a clean restart: `kubectl delete pod -n alert-clickhouse clickhouse-alert-demo-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `clickhouse-alert-demo` instance in the `alert-clickhouse` namespace via the PromQL label filters `job="clickhouse-alert-demo-stats"` / `namespace="alert-clickhouse"` (database group), or `app="clickhouse-alert-demo"` / `namespace="alert-clickhouse"` (provisioner/opsManager/kubeStash groups). + +### Database Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ClickhouseInstanceDown` | critical | 1m | The JMX agent's scrape target reports `up == 0`. | +| `ClickhouseTooManyConnections` | critical | 5m | Too many concurrent TCP connections. | +| `ClickhouseTooManyActiveQueries` | warning | 1m | Too many queries running concurrently. | +| `ClickhouseReplicationPartFetchFailed` | warning | 5m | Replicated part fetches are failing. | +| `ClickhouseBrokenPartsDetected` | critical | instant | Too many unexpected data parts detected. | +| `ClickhouseDataPartCorrupted` | warning | 5m | A data part failed a parse/assertion check. | +| `DiskUsageHigh` | warning | 5m | Persistent volume usage exceeds the configured threshold. | +| `DiskAlmostFull` | critical | 5m | Persistent volume usage is critically high. | + +### Provisioner Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBClickHousePhaseNotReady` | critical | 1m | KubeDB marked the ClickHouse resource `NotReady`. | +| `KubeDBClickHousePhaseCritical` | warning | 15m | ClickHouse is degraded but not fully unavailable. | + +### OpsManager Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBClickHouseOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes. | +| `KubeDBClickHouseOpsRequestFailed` | critical | instant | An ops request failed. | + +### KubeStash Group + +Only meaningful once KubeStash backup/restore is configured for this instance. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ClickHouseKubeStashBackupSessionFailed` | critical | instant | The most recent backup session failed. | +| `ClickHouseKubeStashRestoreSessionFailed` | critical | instant | The most recent restore session failed. | +| `ClickHouseKubeStashNoBackupSessionForTooLong` | warning | instant | No successful backup recorded recently. | +| `ClickHouseKubeStashRepositoryCorrupted` | critical | 5m | Backup repository integrity check failed. | +| `ClickHouseKubeStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage usage is high. | +| `ClickHouseKubeStashBackupSessionPeriodTooLong` | warning | instant | A backup session is taking unusually long. | +| `ClickHouseKubeStashRestoreSessionPeriodTooLong` | warning | instant | A restore session is taking unusually long. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + clickhouseTooManyConnections: + enabled: true + duration: "10m" + severity: warning + kubeStash: + enabled: "none" # disable if you don't use KubeStash +``` + +```bash +$ helm upgrade clickhouse-alert-demo appscode/clickhouse-alerts \ + -n alert-clickhouse \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the clickhouse-alerts release (PrometheusRule) +$ helm uninstall clickhouse-alert-demo -n alert-clickhouse + +# Remove the ClickHouse instance +$ kubectl delete clickhouse -n alert-clickhouse clickhouse-alert-demo + +# Delete namespace +$ kubectl delete ns alert-clickhouse +``` + +## Next Steps + +- Monitor your ClickHouse instance with KubeDB using [built-in Prometheus](/docs/guides/clickhouse/monitoring/using-builtin-prometheus.md). +- Monitor your ClickHouse instance with KubeDB using [Prometheus operator](/docs/guides/clickhouse/monitoring/using-prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/druid/monitoring/alerting.md b/docs/guides/druid/monitoring/alerting.md new file mode 100644 index 0000000000..213ed23066 --- /dev/null +++ b/docs/guides/druid/monitoring/alerting.md @@ -0,0 +1,617 @@ +--- +title: Druid Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: guides-druid-monitoring-alerting + name: Alerting + parent: guides-druid-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Druid Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Apache Druid instance using the `druid-alerts` Helm chart, and how to visualise live metrics using the `kubedb-grafana-dashboards` chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md), making sure to enable the `Druid` and `ZooKeeper` feature gates. + +* Deploy the database in the `alert-druid` namespace: + + ```bash + $ kubectl create ns alert-druid + namespace/alert-druid created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* Druid requires an external **deep storage** backend (for storing segments) and a **metadata storage** database before it can become `Ready`. This tutorial uses a MinIO tenant as S3-compatible deep storage, and lets KubeDB auto-provision a MySQL cluster for metadata storage. See the [Druid Quickstart](/docs/guides/druid/quickstart/guide/index.md#get-external-dependencies-ready) guide for the full walkthrough of setting these up — the short version is repeated in the [Deploy](#deploy-druid-with-monitoring-enabled) section below. + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/druid/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/guides/druid/monitoring/yamls](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/guides/druid/monitoring/yamls) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Druid nodes (router, broker, coordinator, historical, middleManager, overlord) with a **JMX Exporter** Java agent running inside each Druid container, exposing Prometheus metrics on port `9104` — unlike most other KubeDB databases which use a separate sidecar exporter container. +- **ServiceMonitor** (named `{druid-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape every Druid node's exporter every 10 seconds. +- **PrometheusRule** is created by the `druid-alerts` chart and contains all Druid alert definitions grouped by concern: database health and provisioner. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). +- **Grafana** visualises metrics through pre-built dashboards provisioned by the `kubedb-grafana-dashboards` chart. + +
+ Monitoring process of Druid using Prometheus Operator +
+ +--- + +## Deploy Druid with Monitoring Enabled + +### Prepare Deep Storage and Metadata Storage + +Druid cannot start without a deep storage backend for segments. We install a MinIO tenant to provide S3-compatible storage in the same namespace as the database (the cluster-wide `minio-operator` must already be installed — `helm upgrade --install minio-operator minio/operator -n minio-operator --create-namespace` if it isn't): + +```bash +$ helm repo add minio https://operator.min.io/ +$ helm repo update minio + +$ helm upgrade --install --namespace alert-druid druid-minio minio/tenant \ + --set tenant.pools[0].servers=1 \ + --set tenant.pools[0].volumesPerServer=1 \ + --set tenant.pools[0].size=1Gi \ + --set tenant.certificate.requestAutoCert=false \ + --set tenant.buckets[0].name="druid" \ + --set tenant.pools[0].name="default" +``` + +Once the tenant pod is `Running`, note the headless service it creates (typically `myminio-hl`) and create the `deep-storage-config` Secret. The tenant's root credentials live in the auto-created `myminio-env-configuration` Secret — `kubectl get secret -n alert-druid myminio-env-configuration -o jsonpath='{.data.config\.env}' | base64 -d` if you need to confirm them: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: deep-storage-config + namespace: alert-druid +stringData: + druid.storage.type: "s3" + druid.storage.bucket: "druid" + druid.storage.baseKey: "druid/segments" + druid.s3.accessKey: "minio" + druid.s3.secretKey: "minio123" + druid.s3.protocol: "http" + druid.s3.enablePathStyleAccess: "true" + druid.s3.endpoint.signingRegion: "us-east-1" + druid.s3.endpoint.url: "http://myminio-hl.alert-druid.svc.cluster.local:9000/" +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/druid/monitoring/yamls/deep-storage-config-alert-druid.yaml +secret/deep-storage-config created +``` + +KubeDB doesn't require you to set up metadata storage yourself — if `spec.metadataStorage` is left unset, the operator automatically provisions a dedicated MySQL cluster (3 replicas, for group-replication quorum) and a ZooKeeper ensemble (3 replicas) for cluster coordination. This means the Druid object stays in the `Provisioning` phase for several minutes on first deploy while these dependencies come up — this is expected, not a stuck deployment. In testing, the MySQL + ZooKeeper dependencies alone took about 6 minutes to reach `Ready` before Druid's own node pods even started, and the Druid image (~1 GB) took a further 4-5 minutes to pull on first use — budget **10-15 minutes** for a completely fresh cluster with nothing cached yet. + +### Deploy + +Below is the Druid object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Druid +metadata: + name: druid-alert-demo + namespace: alert-druid +spec: + version: 28.0.1 + deepStorage: + type: s3 + configSecret: + name: deep-storage-config + topology: + routers: + replicas: 1 + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s + deletionPolicy: WipeOut +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the Druid resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/druid/monitoring/yamls/druid-alert-demo.yaml +druid.kubedb.com/druid-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get druid -n alert-druid druid-alert-demo -w +NAME VERSION STATUS AGE +druid-alert-demo 28.0.1 Provisioning 30s +druid-alert-demo 28.0.1 Provisioning 9m50s +druid-alert-demo 28.0.1 Ready 15m +``` + +KubeDB brings up one pod per Druid node type, plus the auto-provisioned MySQL and ZooKeeper pods: + +```bash +$ kubectl get pods -n alert-druid +NAME READY STATUS RESTARTS AGE +druid-alert-demo-brokers-0 1/1 Running 0 15m +druid-alert-demo-coordinators-0 1/1 Running 0 15m +druid-alert-demo-historicals-0 1/1 Running 0 15m +druid-alert-demo-middlemanagers-0 1/1 Running 0 15m +druid-alert-demo-mysql-metadata-0 3/3 Running 0 21m +druid-alert-demo-mysql-metadata-1 3/3 Running 0 19m +druid-alert-demo-mysql-metadata-2 3/3 Running 0 17m +druid-alert-demo-routers-0 1/1 Running 0 15m +druid-alert-demo-zk-0 1/1 Running 0 21m +druid-alert-demo-zk-1 1/1 Running 0 18m +druid-alert-demo-zk-2 1/1 Running 0 17m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-druid --selector="app.kubernetes.io/instance=druid-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +druid-alert-demo-brokers ClusterIP 10.43.205.182 8082/TCP 15m +druid-alert-demo-coordinators ClusterIP 10.43.43.45 8081/TCP 15m +druid-alert-demo-pods ClusterIP None 8081/TCP,8090/TCP,8083/TCP,8091/TCP,8082/TCP,8888/TCP 15m +druid-alert-demo-routers ClusterIP 10.43.88.220 8888/TCP 15m +druid-alert-demo-stats ClusterIP 10.43.104.95 9104/TCP 15m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-druid +NAME AGE +druid-alert-demo-stats 15m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-druid druid-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install druid-alerts + +The `druid-alerts` chart creates a `PrometheusRule` resource containing all Druid alert definitions grouped by concern: database health and provisioner. + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the Druid object's name (`druid-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i druid-alert-demo appscode/druid-alerts \ + -n alert-druid \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `druid-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`service="druid-alert-demo-stats"`) | +| `-n alert-druid` | `alert-druid` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +> Don't bother with `--set grafana.enabled=true` — like `neo4j-alerts`/`cassandra-alerts`, this chart also exposes `grafana.enabled`/`jobName`/`url`/`apikey` values and *does* render a dashboard-import `Job` for them (confirmed via `helm template`), but this tutorial uses the separately-maintained `kubedb-grafana-dashboards` chart instead (see [Step 2](#step-2--install-kubedb-grafana-dashboards)) since it already ships real, well-tested Summary/Pod/Database dashboards for Druid — no need to fight the bundled Job's own quirks (see the Cassandra/ClickHouse alerting tutorials for what those quirks can look like in this chart family) when a proven alternative exists. + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-druid +NAME AGE +druid-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-druid druid-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=druid`. + +

+ Prometheus Rule Health +

+ +The `druid.database.alert-druid.druid-alert-demo.rules` and `druid.provisioner.alert-druid.druid-alert-demo.rules` groups are visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Druid alert definitions every 30 seconds. + +## Step 2 — Install kubedb-grafana-dashboards + +The `kubedb-grafana-dashboards` chart creates `GrafanaDashboard` CRDs containing pre-built Druid dashboard JSON. A separate controller, `grafana-operator`, watches these CRDs and pushes the dashboards into Grafana over its HTTP API — both pieces are required. If you've already set these up for another database on this cluster (see the [Elasticsearch alerting guide](/docs/guides/elasticsearch/monitoring/alerting.md) for the full walkthrough), skip straight to [Install the dashboards](#install-the-dashboards) below. + +### Install grafana-operator + +If your cluster doesn't already have it (check with `kubectl get crd grafanadashboards.openviz.dev`): + +```bash +$ helm upgrade -i grafana-operator appscode/grafana-operator \ + -n kubeops --create-namespace \ + --version=v2026.6.12 \ + --wait +``` + +### Mark your Grafana instance as the cluster default + +Skip this if you already have a Grafana `AppBinding` annotated as the cluster default (one is shared across every database). Otherwise: + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 & +$ GRAFANA_PW=$(kubectl get secret -n monitoring prometheus-grafana -o jsonpath='{.data.admin-password}' | base64 -d) +$ curl -s -X POST -H "Content-Type: application/json" -u admin:$GRAFANA_PW \ + http://localhost:3000/api/auth/keys \ + -d '{"name":"kubedb-dashboards","role":"Admin"}' +# Note the returned "key" +$ kill %1 +``` + +```yaml +# grafana-appbinding.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-token + namespace: kubeops +type: Opaque +stringData: + token: "" +--- +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + name: grafana + namespace: kubeops + annotations: + monitoring.appscode.com/is-default-grafana: "true" # must be an ANNOTATION, not a label +spec: + type: monitoring.appscode.com/grafana + clientConfig: + url: "http://prometheus-grafana.monitoring.svc:80" + secret: + name: grafana-admin-token +``` + +```bash +$ kubectl apply -f grafana-appbinding.yaml +``` + +### Install the dashboards + +```bash +$ helm repo add appscode https://charts.appscode.com/stable/ +$ helm repo update appscode + +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.Druid=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl apply -n kubeops -f - +``` + +> **Note:** `featureGates.` defaults to `true` for almost every database in this chart, so one `helm template | kubectl apply` installs dashboards for many databases at once, not just Druid — this is expected. + +### Verify dashboards are created + +```bash +$ kubectl get grafanadashboards -n kubeops | grep druid +NAME TITLE STATUS AGE +kubedb-druid-database KubeDB / Druid / Database Current 2m +kubedb-druid-pod KubeDB / Druid / Pod Current 2m +kubedb-druid-summary KubeDB / Druid / Summary Current 2m +``` + +--- + +## Verify End-to-End + +### 1. Check the exporter is running + +Every Druid node runs the JMX exporter Java agent, serving metrics on the stats service's `9104` port. A value of `druid_service_heartbeat 1` confirms the node is up and being scraped. + +```bash +$ kubectl exec -n alert-druid druid-alert-demo-routers-0 -c druid -- \ + wget -qO- localhost:9104/metrics | grep druid_service_heartbeat +druid_service_heartbeat{dataSource="unknown",type="unknown",} 1.0 +``` + +### 2. Check the Prometheus target is UP + +Prometheus discovers more than 20 scrape pools on a shared cluster, so instead of the Target health page, query `up` directly for a reliable view. + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-druid%22%7D&g0.tab=1`. + +

+ Prometheus up query — all Druid/MySQL/ZooKeeper targets UP +

+ +All 11 targets report `up == 1` — the 5 Druid nodes plus the 3 auto-provisioned MySQL replicas and 3 ZooKeeper replicas, confirming Prometheus is scraping every component of the cluster. + +### 3. Confirm all Druid alerts are inactive + +Open `http://localhost:9090/alerts?search=druid` to see the Druid alert groups. + +

+ Prometheus Alerts — All Inactive +

+ +All rules in the `druid.database` and `druid.provisioner` groups show **INACTIVE**, meaning the cluster is healthy and no thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy Druid instance, no alerts for `druid-alert-demo` will be listed here. + +

+ AlertManager +

+ +### 5. Explore Grafana dashboards + +Port-forward Grafana and log in. + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 +``` + +Open `http://localhost:3000` (username: `admin`). Search for **druid** in the Dashboards section. + +

+ Grafana — Druid Dashboard List +

+ +Three pre-built dashboards are available. The `Namespace` and `Druid` drop-downs at the top of each dashboard let you switch between instances. + +**KubeDB / Druid / Summary** — database status, version, node count (aggregated across Druid nodes *and* the auto-provisioned MySQL/ZooKeeper pods), CPU/memory/storage requests vs. usage. + +

+ Grafana — KubeDB Druid Summary +

+ +**KubeDB / Druid / Pod** — per-node status, ZooKeeper connection ratio, and JVM memory/pool/GC metrics for a single selected pod. + +

+ Grafana — KubeDB Druid Pod +

+ +**KubeDB / Druid / Database** — cluster-wide status, ZooKeeper connection ratio, datasource/segment counts and sizes, and query time/wait-time histograms across all nodes. + +

+ Grafana — KubeDB Druid Database +

+ +--- + +## Simulating a Firing Alert + +The previous section confirmed that all alerts are **INACTIVE** while the cluster is healthy. This section deliberately triggers the `ZKDisconnected` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +> **A note on `DruidDown` (and why this tutorial demonstrates `ZKDisconnected` instead):** every Druid container's `PID 1` is the Druid JVM process itself — there's no `tini`/supervisor wrapper like most other KubeDB images in this project. That means `kubectl exec ... kill -9 1` is a **guaranteed no-op**: confirmed via `readlink /proc/1/ns/pid` vs `/proc/self/ns/pid` (identical — `kubectl exec` shares the container's own PID namespace), and Linux unconditionally ignores `SIGKILL`/`SIGSTOP` sent to a namespace's PID 1 from within that same namespace (see the ClickHouse alerting tutorial for the full explanation of this kernel behavior). `kubectl delete pod` *does* work, but Druid's nodes restart fast enough once their image is cached that `druid_service_heartbeat` never actually reports `0` — the metric simply goes briefly absent and comes back as `1` directly, which can't satisfy `DruidDown`'s `min(...) == 0` condition. Repeatedly force-deleting the same pod for 90+ seconds straight didn't fire it either. `ZKDisconnected`, by contrast, is driven by the *ratio* of connected ZooKeeper sessions across the surviving Druid nodes, which genuinely dips below 1 while ZooKeeper itself is being disrupted — reliably reproducible, and arguably a more realistic incident anyway (ZooKeeper unavailability is a real dependency failure, not an artificial process kill). + +### 1. Disrupt the ZooKeeper ensemble + +```bash +$ end=$(( $(date +%s) + 90 )) +$ while [ $(date +%s) -lt $end ]; do + kubectl delete pod -n alert-druid -l app.kubernetes.io/instance=druid-alert-demo-zk --grace-period=0 --force >/dev/null 2>&1 + sleep 3 + done +``` + +Run this in the background (or a separate terminal) — repeatedly force-deleting all 3 ZooKeeper pods keeps the ensemble from re-forming quorum for the duration of the loop, which Druid's nodes report as a drop in ZooKeeper connectivity. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=druid`. + +

+ Prometheus Alerts — ZKDisconnected Firing +

+ +`ZKDisconnected` transitions from **INACTIVE** through **PENDING** to **FIRING** once the condition holds for the full `for: 1m` window, while `DruidDown` and the rest of the `druid.database` group stay **INACTIVE**. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter={app_namespace="alert-druid"}`. + +

+ AlertManager — ZKDisconnected Firing +

+ +AlertManager shows the `ZKDisconnected` alert. The alert card displays: + +- **Severity**: `critical` +- **app** / **app_namespace**: `druid-alert-demo` / `alert-druid` +- **k8s_kind**: `Druid` +- **Started**: timestamp when the alert first fired + +> Note: this chart's alert labels use `app_namespace` rather than a plain `namespace` label — filter or group on `app_namespace` when searching for these alerts in AlertManager. + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore ZooKeeper + +Let the loop from step 1 finish (or stop it early) — the ZooKeeper `PetSet` recreates all 3 pods on its own once nothing is deleting them anymore. + +```bash +$ kubectl get pods -n alert-druid -l app.kubernetes.io/instance=druid-alert-demo-zk +NAME READY STATUS RESTARTS AGE +druid-alert-demo-zk-0 1/1 Running 0 6m +druid-alert-demo-zk-1 1/1 Running 0 6m +druid-alert-demo-zk-2 1/1 Running 0 6m +``` + +Once all 3 ZooKeeper pods are stably `Running` and the ensemble has re-formed quorum, Prometheus marks the alert **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. In testing this took a few minutes after the disruption loop ended — ZooKeeper needs to elect a leader and Druid's nodes need to re-establish sessions, not an instant reconnect. + +--- + +## Alert Reference + +All alerts are scoped to the `druid-alert-demo` instance in the `alert-druid` namespace via the PromQL label filters `service="druid-alert-demo-stats"` and `namespace="alert-druid"`. + +### Database Group + +Fired based on live JMX-exporter metrics from the Druid nodes. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `DruidDown` | critical | 1m | One of the Druid services is down for more than the configured duration. See the caveat above — this only fires if a node reports a live `druid_service_heartbeat 0` reading, not when a pod simply disappears. | +| `ZKDisconnected` | critical | 1m | Druid lost connection to ZooKeeper. | +| `HighQueryTime` | warning | 1m | A query took more than 1 second to complete on a historical node. | +| `HighQueryWaitTime` | warning | 1m | Druid spent more than 1 second waiting for a segment to be scanned. | +| `HighSegmentScanPending` | warning | 1m | More than 2 segments are queued waiting to be scanned. | +| `HighSegmentUsage` | critical | 1m | More than 95% of space is used by served segments. | +| `HighJVMPoolUsage` | warning | 30s | More than 95% of a JVM memory pool is being used. | +| `HighJVMMemoryUsage` | critical | 30s | More than 95% of JVM memory is being used. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Druid resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBDruidPhaseNotReady` | critical | 1m | KubeDB marked the Druid resource `NotReady` — operator cannot reach the cluster. | +| `KubeDBDruidPhaseCritical` | warning | 15m | The instance is in a degraded/critical phase. | + +### OpsManager Group (declared but not rendered) + +The chart's `values.yaml` declares an `opsManager` group (under `form.alert.groups.opsManager`) meant to track `DruidOpsRequest` lifecycle during upgrades, scaling, and reconfiguration — following the same convention as the ops-manager group in other `*-alerts` charts (e.g. `memcached-alerts`). **At chart version `v2026.7.14`, this group is not actually rendered into the `PrometheusRule`** — `kubectl get prometheusrule -n alert-druid druid-alert-demo -o jsonpath='{.spec.groups[*].name}'` only ever returns the `database` and `provisioner` groups, confirmed both via `helm template` and against the live rule object on a real cluster. This is the same declared-but-unrendered pattern seen in several other `*-alerts` charts (rabbitmq, cassandra, zookeeper, pgbouncer, pgpool) — the values below are what the chart *would* produce if this gap is fixed in a future chart version, not alerts you can currently rely on. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `opsRequestOnProgress` | info | instant | An ops request is currently in progress. | +| `opsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes — likely stuck. | +| `opsRequestFailed` | critical | instant | An ops request failed — check the `OpsRequest` object for the error. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + highSegmentUsage: + enabled: true + duration: "5m" + val: 90 # fire at 90% segment usage instead of the default 95% + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade druid-alert-demo oci://ghcr.io/appscode-charts/druid-alerts \ + -n alert-druid \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the Grafana dashboards (installed via helm template | kubectl apply, not helm install) +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.Druid=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl delete -n kubeops -f - --ignore-not-found + +# Remove the druid-alerts release +$ helm uninstall druid-alert-demo -n alert-druid + +# Remove the Druid instance +$ kubectl delete druid -n alert-druid druid-alert-demo + +# Remove the MinIO tenant used for deep storage +$ helm uninstall druid-minio -n alert-druid + +# Delete namespace +$ kubectl delete ns alert-druid + +# Optional: only if nothing else in the cluster depends on them +$ kubectl delete appbinding -n kubeops grafana +$ kubectl delete secret -n kubeops grafana-admin-token +$ helm uninstall grafana-operator -n kubeops +$ helm uninstall minio-operator -n minio-operator +``` + +## Next Steps + +- Monitor your Druid database with KubeDB using [builtin Prometheus](/docs/guides/druid/monitoring/using-builtin-prometheus.md). +- Monitor your Druid database with KubeDB using [Prometheus operator](/docs/guides/druid/monitoring/using-prometheus-operator.md). +- Detail concepts of [DruidVersion object](/docs/guides/druid/concepts/druidversion.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/druid/monitoring/images/druid-alerting-alertmanager-firing.png b/docs/guides/druid/monitoring/images/druid-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..63ce815f3b Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-alertmanager-firing.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-alertmanager.png b/docs/guides/druid/monitoring/images/druid-alerting-alertmanager.png new file mode 100644 index 0000000000..4f92e36b41 Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-alertmanager.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-grafana-dashboards.png b/docs/guides/druid/monitoring/images/druid-alerting-grafana-dashboards.png new file mode 100644 index 0000000000..d656109aec Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-grafana-dashboards.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-grafana-database.png b/docs/guides/druid/monitoring/images/druid-alerting-grafana-database.png new file mode 100644 index 0000000000..c500aad9c7 Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-grafana-database.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-grafana-pod.png b/docs/guides/druid/monitoring/images/druid-alerting-grafana-pod.png new file mode 100644 index 0000000000..16aaf084df Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-grafana-pod.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-grafana-summary.png b/docs/guides/druid/monitoring/images/druid-alerting-grafana-summary.png new file mode 100644 index 0000000000..b66e1a93fe Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-grafana-summary.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-prom-alerts-firing.png b/docs/guides/druid/monitoring/images/druid-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..0fd87b0ab2 Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-prom-alerts-firing.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-prom-alerts.png b/docs/guides/druid/monitoring/images/druid-alerting-prom-alerts.png new file mode 100644 index 0000000000..aaea3293be Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-prom-alerts.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-prom-rules.png b/docs/guides/druid/monitoring/images/druid-alerting-prom-rules.png new file mode 100644 index 0000000000..af3a7cddb5 Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-prom-rules.png differ diff --git a/docs/guides/druid/monitoring/images/druid-alerting-prom-target.png b/docs/guides/druid/monitoring/images/druid-alerting-prom-target.png new file mode 100644 index 0000000000..e6459ee62b Binary files /dev/null and b/docs/guides/druid/monitoring/images/druid-alerting-prom-target.png differ diff --git a/docs/guides/druid/monitoring/yamls/deep-storage-config-alert-druid.yaml b/docs/guides/druid/monitoring/yamls/deep-storage-config-alert-druid.yaml new file mode 100644 index 0000000000..045b2952b3 --- /dev/null +++ b/docs/guides/druid/monitoring/yamls/deep-storage-config-alert-druid.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: deep-storage-config + namespace: alert-druid +stringData: + druid.storage.type: "s3" + druid.storage.bucket: "druid" + druid.storage.baseKey: "druid/segments" + druid.s3.accessKey: "minio" + druid.s3.secretKey: "minio123" + druid.s3.protocol: "http" + druid.s3.enablePathStyleAccess: "true" + druid.s3.endpoint.signingRegion: "us-east-1" + druid.s3.endpoint.url: "http://myminio-hl.alert-druid.svc.cluster.local:9000/" diff --git a/docs/guides/druid/monitoring/yamls/druid-alert-demo.yaml b/docs/guides/druid/monitoring/yamls/druid-alert-demo.yaml new file mode 100644 index 0000000000..6078255460 --- /dev/null +++ b/docs/guides/druid/monitoring/yamls/druid-alert-demo.yaml @@ -0,0 +1,22 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Druid +metadata: + name: druid-alert-demo + namespace: alert-druid +spec: + version: 28.0.1 + deepStorage: + type: s3 + configSecret: + name: deep-storage-config + topology: + routers: + replicas: 1 + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s + deletionPolicy: WipeOut diff --git a/docs/guides/elasticsearch/monitoring/alerting.md b/docs/guides/elasticsearch/monitoring/alerting.md new file mode 100644 index 0000000000..1b7de11232 --- /dev/null +++ b/docs/guides/elasticsearch/monitoring/alerting.md @@ -0,0 +1,667 @@ +--- +title: Elasticsearch Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: es-monitoring-alerting + name: Alerting + parent: es-monitoring-elasticsearch + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Elasticsearch Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Elasticsearch instance using the `elasticsearch-alerts` Helm chart, and how to visualise live metrics using the `kubedb-grafana-dashboards` chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in a dedicated namespace, so the alerting resources created in this tutorial stay isolated from other workloads: + + ```bash + $ kubectl create ns alert-elasticsearch + namespace/alert-elasticsearch created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. See the [Grafana Dashboard](grafana-dashboard.md#configuration) guide for how to deploy kube-prometheus-stack if you don't have it yet. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/elasticsearch/monitoring/overview.md). + +* You will also need a Grafana API key / token with **Admin** permission so the `kubedb-grafana-dashboards` chart's `grafana-operator` integration can push dashboards into Grafana. See [Step 2](#step-2--install-kubedb-grafana-dashboards) below. + +> Note: YAML files used in this tutorial are stored in [docs/examples/elasticsearch](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/elasticsearch) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +The diagram below shows the full alerting architecture — from Elasticsearch metric export through to alert delivery and Grafana visualisation. + +

+ Elasticsearch Alerting Architecture +

+ +- **KubeDB** deploys Elasticsearch with a built-in [elasticsearch_exporter](https://github.com/prometheus-community/elasticsearch_exporter) sidecar that exposes metrics on port `56790`. +- **ServiceMonitor** (named `{elasticsearch-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `elasticsearch-alerts` chart and contains all Elasticsearch alert definitions grouped by concern: database health, provisioner, ops-manager, Stash backup/restore, and KubeStash backup/restore. +- **Grafana** visualises metrics through pre-built dashboards provisioned by the `kubedb-grafana-dashboards` chart. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +Unlike some KubeDB databases, Elasticsearch's exporter does not publish a single boolean "is the database up" gauge. Instead, the chart watches the health signals a real Elasticsearch cluster actually exposes — JVM heap usage, filesystem usage on the data path, cluster health color (`green`/`yellow`/`red`), node/data-node counts, and shard state — and fires alerts when any of those cross a threshold. + +--- + +## Deploy Elasticsearch with Monitoring Enabled + +At first, let's deploy an Elasticsearch database with monitoring enabled. This tutorial uses a topology cluster (dedicated master, data, and ingest nodes) rather than a single-node instance, since that's representative of a real deployment and is what the rest of this guide's screenshots are taken from. Below is the Elasticsearch object we are going to create. + +```yaml +apiVersion: kubedb.com/v1 +kind: Elasticsearch +metadata: + name: es-alert + namespace: alert-elasticsearch +spec: + version: xpack-9.2.3 + deletionPolicy: WipeOut + topology: + master: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + data: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + ingest: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. +- `spec.topology.*.storage.storageClassName: "local-path"` — use whichever storage class is available/default in your cluster (`kubectl get storageclass`). Note that `local-path` is a `hostPath`-backed class with no capacity quota — the PVC's `1Gi` request is only used for scheduling, and the volume is really backed by however much space is free on the node's own disk. That's fine throughout this tutorial, including the [firing-alert simulation](#simulating-a-firing-alert) later, since that simulation scales the data-node count rather than filling disk. + +Let's create the Elasticsearch resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/elasticsearch/monitoring/es-alert.yaml +elasticsearch.kubedb.com/es-alert created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get elasticsearch -n alert-elasticsearch es-alert +NAME VERSION STATUS AGE +es-alert xpack-9.2.3 Ready 37m +``` + +KubeDB brings up 2 master, 2 data, and 2 ingest pods for this topology — 6 nodes total: + +```bash +$ kubectl get pods -n alert-elasticsearch +NAME READY STATUS RESTARTS AGE +es-alert-data-0 2/2 Running 0 37m +es-alert-data-1 2/2 Running 0 37m +es-alert-ingest-0 2/2 Running 0 37m +es-alert-ingest-1 2/2 Running 0 37m +es-alert-master-0 2/2 Running 0 37m +es-alert-master-1 2/2 Running 0 37m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-elasticsearch --selector="app.kubernetes.io/instance=es-alert" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +es-alert ClusterIP 10.43.126.120 9200/TCP 37m +es-alert-master ClusterIP None 9300/TCP 37m +es-alert-pods ClusterIP None 9200/TCP 37m +es-alert-stats ClusterIP 10.43.49.18 56790/TCP 37m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-elasticsearch +NAME AGE +es-alert-stats 115s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-elasticsearch es-alert-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install elasticsearch-alerts + +The `elasticsearch-alerts` chart creates a `PrometheusRule` resource containing all Elasticsearch alert definitions grouped by concern: database health, provisioner, ops-manager, Stash, and KubeStash. + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the Elasticsearch object's name (`es-alert`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### A note on chart defaults + +The chart's default `database` group rules assume a specific minimum topology: `elasticsearchHealthyNodes` and `elasticsearchHealthyDataNodes` both default to `val: 3` — i.e. "fire if fewer than 3 nodes / fewer than 3 data nodes are healthy." **These defaults do not match every topology, including this tutorial's own `es-alert` (2 master + 2 data + 2 ingest = 6 nodes total, 2 data nodes)** — confirmed live: installing with the chart's plain defaults left `ElasticsearchHealthyDataNodes` firing permanently (`2 < 3` is always true for this topology), even though the cluster was fully healthy. Always override both `val`s to match your actual node counts at install time — don't assume the defaults fit just because you're running a multi-node cluster instead of a single node. + +One rule pair needs overriding regardless of topology: + +- `diskUsageHigh` / `diskAlmostFull` compute PVC usage as `kubelet_volume_stats_used_bytes / (kubelet_volume_stats_used_bytes + kube_pod_spec_volumes_persistentvolumeclaims_info)`. The `..._info` series is a constant label metric (always `1`), not a byte count, so this expression evaluates to ~100% regardless of actual usage — a chart-level expression defect, confirmed live (real PVC usage was 69.6% while both alerts read ~100% and fired). We disable both and rely instead on `elasticsearchDiskOutOfSpace` / `elasticsearchDiskSpaceLow`, which are computed from the exporter's own accurate `elasticsearch_filesystem_data_available_bytes` / `elasticsearch_filesystem_data_size_bytes` metrics. + +### Install + +```bash +$ helm repo add appscode oci://ghcr.io/appscode-charts +$ helm repo update +$ helm search repo appscode/elasticsearch-alerts --version=v2026.7.14 +NAME CHART VERSION APP VERSION DESCRIPTION +appscode/elasticsearch-alerts v2026.7.14 v0.7.0 A Helm chart for Elasticsearch Alert by AppsCode + +$ helm upgrade -i es-alert appscode/elasticsearch-alerts -n alert-elasticsearch --create-namespace --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus \ + --set form.alert.groups.database.rules.diskUsageHigh.enabled=false \ + --set form.alert.groups.database.rules.diskAlmostFull.enabled=false \ + --set form.alert.groups.database.rules.elasticsearchHealthyNodes.val=6 \ + --set form.alert.groups.database.rules.elasticsearchHealthyDataNodes.val=2 +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `es-alert` (release name) | — | Scopes every PromQL expression to this instance (`job="es-alert-stats"`). **This must exactly match the Elasticsearch object's name** — see [above](#why-the-helm-release-name-matters). A mismatched release name is the most common cause of alerts silently never firing (and Grafana/Prometheus showing nothing for a healthy instance): the chart's rules end up scoped to a `job` label that no target ever carries. | +| `-n alert-elasticsearch` | `alert-elasticsearch` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | +| `...diskUsageHigh.enabled` / `...diskAlmostFull.enabled` | `false` | Works around the PVC-usage expression defect described above | +| `...elasticsearchHealthyNodes.val` | `6` | Matches this tutorial's real total node count (2 master + 2 data + 2 ingest) | +| `...elasticsearchHealthyDataNodes.val` | `2` | Matches this tutorial's real data-node count | + +> Whatever topology you actually deploy, set both `val`s to your real node counts — total nodes for `elasticsearchHealthyNodes`, data nodes for `elasticsearchHealthyDataNodes`. For a single-node instance that means `val: 1` for both, and you should also disable `elasticsearchUnassignedShards` (a single-node cluster can never assign a replica shard, so this rule fires permanently). + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-elasticsearch +NAME AGE +es-alert 22s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-elasticsearch es-alert \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=elasticsearch`. + +

+ Prometheus Rule Health +

+ +The `elasticsearch.database.alert-elasticsearch.es-alert.rules` group is visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Elasticsearch alert definitions every 30 seconds. + +## Step 2 — Install kubedb-grafana-dashboards + +The `kubedb-grafana-dashboards` chart creates `GrafanaDashboard` CRDs containing pre-built Elasticsearch dashboard JSON. A separate controller, `grafana-operator`, watches these CRDs and pushes the dashboards into Grafana over its HTTP API — both pieces are required. + +### Install grafana-operator + +If your cluster doesn't already have it (check with `kubectl get crd grafanadashboards.openviz.dev`), install the operator that reconciles `GrafanaDashboard`/`GrafanaDatasource` objects into a real Grafana instance: + +```bash +$ helm upgrade -i grafana-operator appscode/grafana-operator \ + -n kubeops --create-namespace \ + --version=v2026.6.12 \ + --wait +``` + +### Mark your Grafana instance as the cluster default + +The chart looks up Grafana connection details from an `AppBinding` annotated as the cluster's default Grafana. If you deployed Grafana via `kube-prometheus-stack` (as in this tutorial), that `AppBinding` doesn't exist yet and must be created once per cluster: + +```bash +# Create a Grafana API key (adjust the endpoint/payload shape for your Grafana version) +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 & +$ GRAFANA_PW=$(kubectl get secret -n monitoring prometheus-grafana -o jsonpath='{.data.admin-password}' | base64 -d) +$ curl -s -X POST -H "Content-Type: application/json" -u admin:$GRAFANA_PW \ + http://localhost:3000/api/auth/keys \ + -d '{"name":"kubedb-dashboards","role":"Admin"}' +# Note the returned "key" +$ kill %1 +``` + +```yaml +# grafana-appbinding.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-token + namespace: kubeops +type: Opaque +stringData: + token: "" +--- +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + name: grafana + namespace: kubeops + annotations: + monitoring.appscode.com/is-default-grafana: "true" # must be an ANNOTATION, not a label +spec: + type: monitoring.appscode.com/grafana + clientConfig: + url: "http://prometheus-grafana.monitoring.svc:80" + secret: + name: grafana-admin-token +``` + +```bash +$ kubectl apply -f grafana-appbinding.yaml +``` + +> **Why an AppBinding at all?** `GrafanaDashboard` objects don't carry connection details themselves — `grafana-operator` looks up the one `AppBinding` across the cluster marked with the `monitoring.appscode.com/is-default-grafana: "true"` **annotation** and uses its `clientConfig.url` + referenced `secret` (must contain a `token` key) to talk to Grafana. Skip this step only if your cluster already provisions Grafana through an Appscode-managed chart that creates this `AppBinding` automatically. + +### Install the dashboards + +```bash +$ helm repo add appscode https://charts.appscode.com/stable/ +$ helm repo update appscode + +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.Elasticsearch=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl apply -n kubeops -f - +``` + +> **Note:** The `kubedb-grafana-dashboards` chart bundles many large Grafana dashboard JSON files. Even with a single `featureGate` enabled, the rendered manifests can exceed Kubernetes' hard 1 MB Secret limit that Helm uses to store release state. To work around this, render the chart locally with `helm template` and apply the output directly with `kubectl apply`, which bypasses Helm's Secret storage entirely. Because this doesn't create a Helm release object, `helm uninstall` will not work for cleanup — use `kubectl delete` directly (see [Cleaning up](#cleaning-up)). Also note that `featureGates.` defaults to `true` for almost every database in this chart (only `Aerospike` defaults `false`), so one `helm template | kubectl apply` installs dashboards for many databases at once, not just Elasticsearch — this is expected. + +### Verify dashboards are created + +```bash +$ kubectl get grafanadashboards -n kubeops | grep elasticsearch +NAME TITLE STATUS AGE +kubedb-elasticsearch-database KubeDB / Elasticsearch / Database Current 2m +kubedb-elasticsearch-pod KubeDB / Elasticsearch / Pod Current 2m +kubedb-elasticsearch-summary KubeDB / Elasticsearch / Summary Current 2m +``` + +`Current` means `grafana-operator` successfully pushed the dashboard into Grafana. If a dashboard stays `Failed` with a message like `no default Grafana appbinding found`, revisit the AppBinding step above. + +--- + +## Verify End-to-End + +### 1. Check the exporter is running + +The `exporter` sidecar inside the Elasticsearch pod serves metrics at `:56790/metrics`. The `elasticsearch_cluster_health_status` series confirms the exporter can reach Elasticsearch and report cluster health. + +```bash +$ kubectl exec -n alert-elasticsearch es-alert-data-0 -c exporter -- \ + wget -qO- localhost:56790/metrics | grep elasticsearch_cluster_health_status +elasticsearch_cluster_health_status{cluster="es-alert",color="green"} 1 +elasticsearch_cluster_health_status{cluster="es-alert",color="red"} 0 +elasticsearch_cluster_health_status{cluster="es-alert",color="yellow"} 0 +``` + +With master, data, and ingest nodes all up, the cluster can fully assign both primary and replica shards, so it reports `green`. (A single-node cluster would instead report `yellow` — it can never assign replica shards without a second node to place them on — which is expected and not an outage.) + +### 2. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-elasticsearch%22%7D&g0.tab=1`. + +

+ Prometheus Target UP +

+ +All 6 series report `up == 1` — one entry per master/data/ingest pod, confirming metrics are being scraped from every node in the `alert-elasticsearch` namespace. + +### 3. Confirm all Elasticsearch alerts are inactive + +Open `http://localhost:9090/alerts?search=elasticsearch` to see the Elasticsearch alert groups. + +

+ Prometheus Alerts — All Inactive +

+ +All 6 rules in the `elasticsearch.database` group show **INACTIVE (6)**, meaning the database is healthy and no thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy Elasticsearch instance, no alerts for `es-alert` will be listed here. + +

+ AlertManager — No Active Alerts +

+ +### 5. Explore Grafana dashboards + +Port-forward Grafana and log in. + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 +``` + +Open `http://localhost:3000` (username: `admin`). Search for **elasticsearch** in the Dashboards section. + +

+ Grafana — Elasticsearch Dashboard List +

+ +Three pre-built dashboards are available. The `Namespace` and `app` drop-downs at the top of each dashboard let you switch between instances. + +**KubeDB / Elasticsearch / Summary** — cluster-wide health: database status, version, node count, CPU/memory/storage requests vs. usage. + +

+ Grafana — KubeDB Elasticsearch Summary +

+ +**KubeDB / Elasticsearch / Pod** — per-node detail: node status color, open file count, connected/active data nodes, memory and heap usage, GC time, documents indexed. + +

+ Grafana — KubeDB Elasticsearch Pod +

+ +**KubeDB / Elasticsearch / Database** — cluster status, shard counts, documents indexed, index size, indexing/query rate, and ingest-node system metrics. + +

+ Grafana — KubeDB Elasticsearch Database +

+ +--- + +## Simulating a Firing Alert + +The previous section confirmed that all alerts are **INACTIVE** while the database is healthy. This section walks through deliberately triggering `ElasticsearchHealthyDataNodes` — along with two alerts it drags along with it, see the note below — so you can observe the full alert lifecycle and then resolve it. + +Elasticsearch doesn't have a single "process down" style alert the way some other databases do — its exporter reports live cluster metrics rather than a boolean liveness gauge. Killing the `elasticsearch` process inside a pod doesn't work either: the container restarts in under 2 seconds (faster than the cluster's fault-detection window), so the other nodes never actually perceive the node as gone. Instead, we shrink the `data` role from 2 nodes to 1 — a real, sustained, cleanly-reversible change that reliably crosses this tutorial's `elasticsearchHealthyDataNodes.val: 2` threshold set in [Step 1](#install). + +### 1. Scale down the data nodes + +```bash +$ kubectl patch elasticsearch -n alert-elasticsearch es-alert \ + --type=merge -p '{"spec":{"topology":{"data":{"replicas":1}}}}' +elasticsearch.kubedb.com/es-alert patched +``` + +KubeDB terminates one data pod to bring the topology down to the new desired count: + +```bash +$ kubectl get pods -n alert-elasticsearch -l app.kubernetes.io/instance=es-alert +NAME READY STATUS RESTARTS AGE +es-alert-data-0 2/2 Running 0 37m +es-alert-ingest-0 2/2 Running 0 37m +es-alert-ingest-1 2/2 Running 0 37m +es-alert-master-0 2/2 Running 0 37m +es-alert-master-1 2/2 Running 0 37m +``` + +Wait 30–60 seconds for the next Prometheus scrape cycle (configured at 10 s) and rule-evaluation cycle (30 s) to register the smaller data-node count. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=elasticsearch`. + +

+ Prometheus Alerts — ElasticsearchHealthyDataNodes Firing +

+ +Dropping to 5 total nodes (1 data + 2 master + 2 ingest) crosses **three** thresholds at once, confirmed live — all `for: instant`, so all three move directly from **INACTIVE** to **FIRING** within one evaluation cycle, while the rest of the `elasticsearch.database` group stays **INACTIVE**: + +- `ElasticsearchHealthyDataNodes` — data-node count (1) is below `val: 2`. +- `ElasticsearchHealthyNodes` — total node count (5) is below `val: 6`. +- `ElasticsearchUnassignedShards` — with only one data node left, replica shards have nowhere to be placed. + +Each fires once per surviving node's exporter (5 series each here — one per remaining pod), since every node independently reports its own view of cluster-wide state; that's 15 alert instances in total, not 15 separate incidents. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter={namespace="alert-elasticsearch"}`. + +

+ AlertManager — ElasticsearchHealthyDataNodes Firing +

+ +AlertManager shows all three alerts grouped by namespace (15 alerts total). Each alert card displays: + +- **Severity**: `critical` +- **app** / **job**: `es-alert` / `es-alert-stats` +- **pod**: the surviving node reporting the condition (e.g. `es-alert-data-0`, `es-alert-master-1`, ...) +- **Started**: timestamp when the alert first fired + +AlertManager routes these alerts to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alerts are visible here but silently dropped. + +### 4. Restore the data nodes + +Scale the data role back to 2 to resolve the alert. + +```bash +$ kubectl patch elasticsearch -n alert-elasticsearch es-alert \ + --type=merge -p '{"spec":{"topology":{"data":{"replicas":2}}}}' +elasticsearch.kubedb.com/es-alert patched +``` + +Wait for the pod to rejoin and for the next scrape cycle to register the recovered count. + +```bash +$ kubectl get elasticsearch -n alert-elasticsearch es-alert +NAME VERSION STATUS AGE +es-alert xpack-9.2.3 Ready 41m +``` + +Once the Elasticsearch resource returns to `Ready` and `elasticsearch_cluster_health_number_of_data_nodes` reports `2` again, Prometheus marks all three alerts **INACTIVE** and AlertManager sends **resolved** notifications to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `es-alert` instance in the `alert-elasticsearch` namespace via the PromQL label filters `job="es-alert-stats"` and `namespace="alert-elasticsearch"`. + +### Database Group + +Fired based on live metrics from the Elasticsearch exporter. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ElasticsearchHeapUsageTooHigh` | critical | 2m | The JVM heap usage is over 90%. | +| `ElasticsearchHeapUsageWarning` | warning | 2m | The JVM heap usage is over 80%. | +| `ElasticsearchDiskOutOfSpace` | critical | instant | The disk usage is over 90%. | +| `ElasticsearchDiskSpaceLow` | warning | 2m | The disk usage is over 80%. | +| `ElasticsearchClusterRed` | critical | instant | Elastic Cluster Red status — one or more primary shards are not allocated. | +| `ElasticsearchClusterYellow` | warning | instant | Elastic Cluster Yellow status — one or more replica shards are not allocated. | +| `ElasticsearchHealthyNodes` | critical | instant | Fewer than the configured minimum number of nodes are healthy in the cluster (default `val: 3`; this tutorial overrides it to `6` — see [Step 1](#install)). | +| `ElasticsearchHealthyDataNodes` | critical | instant | Fewer than the configured minimum number of data nodes are healthy in the cluster (default `val: 3`; this tutorial overrides it to `2`). | +| `ElasticsearchRelocatingShards` | info | instant | Elasticsearch is relocating shards. | +| `ElasticsearchInitializingShards` | info | instant | Elasticsearch is initializing shards. | +| `ElasticsearchUnassignedShards` | critical | instant | Elasticsearch has unassigned shards. | +| `ElasticsearchPendingTasks` | warning | 15m | Elasticsearch has pending tasks — the cluster is working slowly. | +| `ElasticsearchNoNewDocuments10m` | info | instant | No new documents were indexed in the last 10 minutes (disabled by default). | +| `DiskUsageHigh` | warning | 1m | **Disabled by the install command above** — broken denominator always reads ~100% usage regardless of real usage (confirmed: real usage 69.6% while this alert read ~100%). | +| `DiskAlmostFull` | critical | 1m | **Disabled by the install command above** — same broken-denominator bug as `DiskUsageHigh`. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Elasticsearch resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBElasticsearchPhaseNotReady` | critical | 1m | KubeDB marked the Elasticsearch resource `NotReady` — operator cannot reach the database. | +| `KubeDBElasticsearchPhaseCritical` | warning | 15m | The instance is in a degraded/critical phase. | + +### OpsManager Group + +Tracks `ElasticsearchOpsRequest` lifecycle during upgrades, scaling, and reconfiguration. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBElasticsearchOpsRequestOnProgress` | info | instant | An ops request is currently in progress. | +| `KubeDBElasticsearchOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes — likely stuck. | +| `KubeDBElasticsearchOpsRequestFailed` | critical | instant | An ops request failed — check the `ElasticsearchOpsRequest` object for the error. | + +### Stash Group + +Tracks backup/restore health for Elasticsearch instances backed up with [Stash](https://stash.run/). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ElasticsearchStashBackupSessionFailed` | critical | instant | The most recent Stash backup session failed. | +| `ElasticsearchStashRestoreSessionFailed` | critical | instant | The most recent Stash restore session failed. | +| `ElasticsearchStashNoBackupSessionForTooLong` | warning | instant | No successful backup session in the last 18000s (5 hours). | +| `ElasticsearchStashRepositoryCorrupted` | critical | 5m | The Stash backup repository failed its integrity check. | +| `ElasticsearchStashRepositoryStorageRunningLow` | warning | 5m | The Stash repository has grown beyond 10 GB. | +| `ElasticsearchStashBackupSessionPeriodTooLong` | warning | instant | A backup session took longer than 1800s (30 minutes) to complete. | +| `ElasticsearchStashRestoreSessionPeriodTooLong` | warning | instant | A restore session took longer than 1800s (30 minutes) to complete. | + +### KubeStash Group + +Tracks backup/restore health for Elasticsearch instances backed up with [KubeStash](https://kubestash.com/). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ElasticsearchKubeStashBackupSessionFailed` | critical | instant | The most recent KubeStash backup session failed. | +| `ElasticsearchKubeStashRestoreSessionFailed` | critical | instant | The most recent KubeStash restore session failed. | +| `ElasticsearchKubeStashNoBackupSessionForTooLong` | warning | instant | No successful backup session in the last 18000s (5 hours). | +| `ElasticsearchKubeStashRepositoryCorrupted` | critical | 5m | The KubeStash repository failed its integrity check. | +| `ElasticsearchKubeStashRepositoryStorageRunningLow` | warning | 5m | The KubeStash repository has grown beyond 10 GB. | +| `ElasticsearchKubeStashBackupSessionPeriodTooLong` | warning | instant | A backup session took longer than 1800s (30 minutes) to complete. | +| `ElasticsearchKubeStashRestoreSessionPeriodTooLong` | warning | instant | A restore session took longer than 1800s (30 minutes) to complete. | + +> Stash and KubeStash alerts are only relevant if you've configured backups for this Elasticsearch instance. This tutorial doesn't set up backups — the tables above are included so you know what's available in the chart if you do. + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + elasticsearchHeapUsageWarning: + enabled: true + duration: "5m" + val: 70 # fire at 70% heap usage instead of the default 80% + severity: warning + opsManager: + enabled: "none" # disable all ops-manager alerts +``` + +```bash +$ helm upgrade es-alert oci://ghcr.io/appscode-charts/elasticsearch-alerts \ + -n alert-elasticsearch \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the Grafana dashboards (installed via helm template | kubectl apply, not helm install) +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.Elasticsearch=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl delete -n kubeops -f - --ignore-not-found + +# Remove the elasticsearch-alerts release +$ helm uninstall es-alert -n alert-elasticsearch + +# Remove the Elasticsearch instance +$ kubectl delete elasticsearch -n alert-elasticsearch es-alert + +# Delete namespace +$ kubectl delete ns alert-elasticsearch + +# Optional: only if nothing else in the cluster depends on them +$ kubectl delete appbinding -n kubeops grafana +$ kubectl delete secret -n kubeops grafana-admin-token +$ helm uninstall grafana-operator -n kubeops +``` + +## Next Steps + +- Monitor your Elasticsearch database with KubeDB using [builtin Prometheus](/docs/guides/elasticsearch/monitoring/using-builtin-prometheus.md). +- Monitor your Elasticsearch database with KubeDB using [Prometheus operator](/docs/guides/elasticsearch/monitoring/using-prometheus-operator.md). +- Visualise Elasticsearch metrics with [Grafana Dashboard](grafana-dashboard.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/hazelcast/monitoring/alerting.md b/docs/guides/hazelcast/monitoring/alerting.md new file mode 100644 index 0000000000..633f202e65 --- /dev/null +++ b/docs/guides/hazelcast/monitoring/alerting.md @@ -0,0 +1,361 @@ +--- +title: Hazelcast Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: hz-monitoring-alerting + name: Alerting + parent: hz-monitoring-hazelcast + weight: 60 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Hazelcast Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Hazelcast instance using the `hazelcast-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-hazelcast` namespace: + + ```bash + $ kubectl create ns alert-hazelcast + namespace/alert-hazelcast created + ``` + +* Hazelcast requires an Enterprise license. Create a secret with your license key before deploying: + + ```bash + $ kubectl create secret generic hz-license-key -n alert-hazelcast \ + --from-literal=licenseKey='your hazelcast license key' + secret/hz-license-key created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/hazelcast/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/hazelcast](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/hazelcast) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Hazelcast with a metrics-exporter sidecar (container `exporter`) that exposes Hazelcast's own JMX-derived metrics (`com_hazelcast_Metrics_*`). +- **ServiceMonitor** (named `{hazelcast-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `hazelcast-alerts` chart and contains alert definitions grouped by concern: database health (which also embeds KubeDB-operator-sourced `hazelcastDown`/`hazelcastPhaseCritical` alerts) and provisioner. +- **Grafana** dashboards for Hazelcast are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy Hazelcast with Monitoring Enabled + +At first, let's deploy a single-node Hazelcast instance with monitoring enabled. Below is the Hazelcast object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Hazelcast +metadata: + name: hazelcast-alert-demo + namespace: alert-hazelcast +spec: + version: "5.5.2" + replicas: 1 + licenseSecret: + name: hz-license-key + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.replicas: 1` creates a single-node Hazelcast instance. +- `spec.licenseSecret.name: hz-license-key` points to the Enterprise license secret created in [Before You Begin](#before-you-begin). +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the Hazelcast resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/hazelcast/monitoring/hazelcast-alert-demo.yaml +hazelcast.kubedb.com/hazelcast-alert-demo created +``` + +Wait for the database to go into `Ready` state. + +```bash +$ kubectl get hazelcast -n alert-hazelcast hazelcast-alert-demo +NAME VERSION STATUS AGE +hazelcast-alert-demo 5.5.2 Ready 3m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-hazelcast --selector="app.kubernetes.io/instance=hazelcast-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +hazelcast-alert-demo ClusterIP 10.43.10.20 5701/TCP 3m +hazelcast-alert-demo-pods ClusterIP None 5701/TCP 3m +hazelcast-alert-demo-stats ClusterIP 10.43.10.21 8080/TCP 3m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-hazelcast +NAME AGE +hazelcast-alert-demo-stats 3m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-hazelcast hazelcast-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install hazelcast-alerts + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression from the **Helm release name** — so the release name must match the Hazelcast object's name (`hazelcast-alert-demo`). + +### Install + +```bash +$ helm upgrade -i hazelcast-alert-demo oci://ghcr.io/appscode-charts/hazelcast-alerts \ + -n alert-hazelcast \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-hazelcast +NAME AGE +hazelcast-alert-demo 30s + +$ kubectl get prometheusrule -n alert-hazelcast hazelcast-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `hazelcast.database` and `hazelcast.provisioner` groups. + +

+ Prometheus Rule Health +

+ +Both groups should show **OK**. `hazelcast-alerts` v2026.7.14 has no `opsManager`/`stash`/`kubeStash` groups at all — only `database` and `provisioner`. + +> **Note the overlap:** the `database` group's `hazelcastDown` (`for: 30s`) and `hazelcastPhaseCritical` (`for: 3m`) key off the exact same `kubedb_com_hazelcast_status_phase` metric as the `provisioner` group's `KubeDBhazelcastPhaseNotReady`/`KubeDBhazelcastPhaseCritical` (`for: 1m`/`15m`) — a real outage fires **both** pairs of alerts (at different times, since the `for` windows differ), not a bug exactly, but worth knowing so you don't mistake it for two independent problems. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-hazelcast%22%7D&g0.tab=1`. + +

+ Prometheus up query — hazelcast-alert-demo-0 UP +

+ +### 2. Confirm the Hazelcast alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — Hazelcast groups inactive +

+ +All rules should show **INACTIVE**. + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +### 4. Explore the Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the Hazelcast dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.Hazelcast=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `hazelcastDown` (`for: 30s`, the fastest down-signal) by crashing the main Hazelcast JVM process. + +### 1. Crash the Hazelcast process + +```bash +$ kubectl exec -n alert-hazelcast hazelcast-alert-demo-0 -c hazelcast -- sh -c ' + end=$(( $(date +%s) + 60 )); + while [ $(date +%s) -lt $end ]; do + pid=$(pgrep -f "java.*hazelcast" | head -1); + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; + sleep 1; + done' +``` + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — hazelcastDown Firing +

+ +`hazelcastDown` (`kubedb_com_hazelcast_status_phase{phase!="Ready"} == 1`, `for: 30s`) should transition to **FIRING** first; if the crash loop runs long enough, `KubeDBhazelcastPhaseNotReady` (`for: 1m`, provisioner group) fires shortly after. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — hazelcastDown Firing +

+ +### 4. Restore Hazelcast + +Stop the loop from step 1. + +```bash +$ kubectl get hazelcast -n alert-hazelcast hazelcast-alert-demo -w +NAME VERSION STATUS AGE +hazelcast-alert-demo 5.5.2 Ready 24m +``` + +If Hazelcast does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-hazelcast hazelcast-alert-demo-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `hazelcast-alert-demo` instance in the `alert-hazelcast` namespace, mostly via `namespace`/`service` label filters matching `$app-stats` (database group), or `app="hazelcast-alert-demo"` / `namespace="alert-hazelcast"` (provisioner group and the two operator-phase alerts embedded in the database group). + +### Database Group + +Fired based on live metrics from the Hazelcast exporter sidecar (JMX-derived) and, for the `hazelcastDown`/`hazelcastPhaseCritical` pair, the KubeDB operator's own view of the resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `hazelcastPartitionCountExceed` | warning | 30s | Active partition count is unusually high. | +| `hazelcastHighHeapPercentage` | warning | 30s | JVM heap usage is high. | +| `hazelcastHighMemoryUsage` | warning | 30s | Hazelcast memory usage is high. | +| `hazelcastHighPhysicalMemoryUsage` | warning | 30s | Physical memory usage is high relative to total. | +| `hazelcastHighLatency` | warning | 30s | Get-operation latency is elevated. | +| `hazelcastSystemCPULoadExceed` | warning | 30s | System CPU load is high. | +| `hazelcastPhaseCritical` | warning | 3m | KubeDB operator view: resource `Critical` (duplicates the provisioner group's own version at a different `for`). | +| `hazelcastDown` | critical | 30s | KubeDB operator view: resource not `Ready`. Fastest down-signal available. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Hazelcast resource phase (sourced from Panopticon, not the Hazelcast metrics endpoint). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBhazelcastPhaseNotReady` | critical | 1m | KubeDB marked the Hazelcast resource `NotReady`. | +| `KubeDBhazelcastPhaseCritical` | warning | 15m | Hazelcast is degraded but not fully unavailable. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + hazelcastHighHeapPercentage: + enabled: true + duration: "2m" + severity: warning +``` + +```bash +$ helm upgrade hazelcast-alert-demo oci://ghcr.io/appscode-charts/hazelcast-alerts \ + -n alert-hazelcast \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +$ helm uninstall hazelcast-alert-demo -n alert-hazelcast +$ kubectl delete hazelcast -n alert-hazelcast hazelcast-alert-demo +$ kubectl delete secret -n alert-hazelcast hz-license-key +$ kubectl delete ns alert-hazelcast +``` + +## Next Steps + +- Monitor your Hazelcast instance with KubeDB using [built-in Prometheus](/docs/guides/hazelcast/monitoring/prometheus-builtin.md). +- Monitor your Hazelcast instance with KubeDB using [Prometheus operator](/docs/guides/hazelcast/monitoring/prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/ignite/monitoring/alerting.md b/docs/guides/ignite/monitoring/alerting.md new file mode 100644 index 0000000000..7db141db74 --- /dev/null +++ b/docs/guides/ignite/monitoring/alerting.md @@ -0,0 +1,343 @@ +--- +title: Ignite Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: ig-monitoring-alerting + name: Alerting + parent: ig-monitoring-ignite + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Ignite Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Ignite instance using the `ignite-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-ignite` namespace: + + ```bash + $ kubectl create ns alert-ignite + namespace/alert-ignite created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/ignite/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/ignite](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/ignite) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Ignite with a metrics-exporter sidecar (container `exporter`) that exposes Ignite's own JMX-derived metrics (`sys_*`, `io_*`, `cluster_*`, `ignite_*`). +- **ServiceMonitor** (named `{ignite-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `ignite-alerts` chart and contains alert definitions grouped by concern: database health (which also embeds KubeDB-operator-sourced `IgniteDown`/`IgnitePhaseCritical` alerts) and provisioner. +- **Grafana** dashboards for Ignite are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy Ignite with Monitoring Enabled + +At first, let's deploy an Ignite instance with monitoring enabled. Below is the Ignite object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Ignite +metadata: + name: ignite-alert-demo + namespace: alert-ignite +spec: + replicas: 1 + version: "2.17.0" + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.replicas: 1` deploys a single-node Ignite instance. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/ignite/monitoring/ignite-alert-demo.yaml +ignite.kubedb.com/ignite-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get ignite -n alert-ignite ignite-alert-demo +NAME VERSION STATUS AGE +ignite-alert-demo 2.17.0 Ready 3m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-ignite --selector="app.kubernetes.io/instance=ignite-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +ignite-alert-demo ClusterIP 10.43.10.20 10800/TCP 3m +ignite-alert-demo-pods ClusterIP None 10800/TCP 3m +ignite-alert-demo-stats ClusterIP 10.43.10.21 8080/TCP 3m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-ignite +NAME AGE +ignite-alert-demo-stats 3m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-ignite ignite-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install ignite-alerts + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression from the **Helm release name** — so the release name must match the Ignite object's name (`ignite-alert-demo`). + +### Install + +```bash +$ helm upgrade -i ignite-alert-demo oci://ghcr.io/appscode-charts/ignite-alerts \ + -n alert-ignite \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-ignite +NAME AGE +ignite-alert-demo 30s + +$ kubectl get prometheusrule -n alert-ignite ignite-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `ignite.database` and `ignite.provisioner` groups. + +

+ Prometheus Rule Health +

+ +Both groups should show **OK**. `ignite-alerts` v2026.7.14 has no `opsManager`/`stash`/`kubeStash` groups — only `database` and `provisioner`. + +> **Note the overlap:** the `database` group's `IgniteDown` (`for: 30s`) and `IgnitePhaseCritical` (`for: 1m`) key off the same `kubedb_com_ignite_status_phase` metric as the `provisioner` group's `KubeDBIgnitePhaseNotReady`/`KubeDBIgnitePhaseCritical` (`for: 1m`/`15m`) — expect both pairs to eventually fire together during a real outage, at different times. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-ignite%22%7D&g0.tab=1`. + +

+ Prometheus up query — ignite-alert-demo-0 UP +

+ +### 2. Confirm the Ignite alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — Ignite groups inactive +

+ +All rules should show **INACTIVE**. `IgniteClusterNoBaselineNode` will only have data once the cluster's baseline topology is activated (a normal part of Ignite persistence setup, not KubeDB-specific). + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +### 4. Explore the Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the Ignite dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.Ignite=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `IgniteDown` (`for: 30s`, the fastest down-signal) by crashing the main Ignite JVM process. + +### 1. Crash the Ignite process + +```bash +$ kubectl exec -n alert-ignite ignite-alert-demo-0 -c ignite -- sh -c ' + end=$(( $(date +%s) + 60 )); + while [ $(date +%s) -lt $end ]; do + pid=$(pgrep -f "org.apache.ignite" | head -1); + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; + sleep 1; + done' +``` + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — IgniteDown Firing +

+ +`IgniteDown` (`kubedb_com_ignite_status_phase{phase!="Ready"} == 1`, `for: 30s`) should transition to **FIRING** first. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — IgniteDown Firing +

+ +### 4. Restore Ignite + +Stop the loop from step 1. + +```bash +$ kubectl get ignite -n alert-ignite ignite-alert-demo -w +NAME VERSION STATUS AGE +ignite-alert-demo 2.17.0 Ready 24m +``` + +If Ignite does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-ignite ignite-alert-demo-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `ignite-alert-demo` instance in the `alert-ignite` namespace via `job="ignite-alert-demo-stats"` / `namespace="alert-ignite"` (database group), or `app="ignite-alert-demo"` / `namespace="alert-ignite"` (provisioner group and the two operator-phase alerts embedded in the database group). + +### Database Group + +Fired based on live metrics from the Ignite exporter sidecar and node/kubelet metrics, plus two KubeDB-operator-sourced phase alerts (`IgniteDown`, `IgnitePhaseCritical`). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `IgniteDown` | critical | 30s | KubeDB operator view: resource not `Ready`. Fastest down-signal available. | +| `IgnitePhaseCritical` | warning | 1m | KubeDB operator view: resource `Critical` (duplicates the provisioner group's own version at a different `for`). | +| `IgniteClusterNoBaselineNode` | warning | 1m | The cluster has no baseline topology node registered. | +| `IgniteRestarted` | warning | 1m | Uptime indicates a recent restart. | +| `IgniteHighCPULoad` | warning | 1m | System CPU load exceeds 80%. | +| `IgniteHighHeapMemoryUsed` | warning | 1m | JVM heap usage is high. | +| `IgniteHighDataregionOffHeapUsed` | warning | 1m | Off-heap data region usage is high. | +| `IgniteJVMPausesTotalDuration` | warning | 1m | Long JVM GC pauses detected. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Ignite resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBIgnitePhaseNotReady` | critical | 1m | KubeDB marked the Ignite resource `NotReady`. | +| `KubeDBIgnitePhaseCritical` | warning | 15m | Ignite is degraded but not fully unavailable. | + +--- + +## Customising Alerts + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + igniteHighCPULoad: + enabled: true + duration: "5m" + severity: warning +``` + +```bash +$ helm upgrade ignite-alert-demo oci://ghcr.io/appscode-charts/ignite-alerts \ + -n alert-ignite \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +```bash +$ helm uninstall ignite-alert-demo -n alert-ignite +$ kubectl delete ignite -n alert-ignite ignite-alert-demo +$ kubectl delete ns alert-ignite +``` + +## Next Steps + +- Monitor your Ignite instance with KubeDB using [built-in Prometheus](/docs/guides/ignite/monitoring/using-builtin-prometheus.md). +- Monitor your Ignite instance with KubeDB using [Prometheus operator](/docs/guides/ignite/monitoring/using-prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/kafka/monitoring/alerting.md b/docs/guides/kafka/monitoring/alerting.md new file mode 100644 index 0000000000..f8df871fc2 --- /dev/null +++ b/docs/guides/kafka/monitoring/alerting.md @@ -0,0 +1,352 @@ +--- +title: Kafka Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: kf-monitoring-alerting + name: Alerting + parent: kf-monitoring-kafka + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Kafka Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Kafka cluster using the `kafka-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-kafka` namespace: + + ```bash + $ kubectl create ns alert-kafka + namespace/alert-kafka created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/kafka/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/kafka](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/kafka) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Kafka with metrics exposed by a [JMX Exporter](https://github.com/prometheus/jmx_exporter) running as a **Java agent inside the `kafka` container itself** — not a separate sidecar container. KubeDB uses the JMX agent because the officially recognized Kafka exporter image does not yet expose metrics for the KRaft-mode versions KubeDB supports. +- **ServiceMonitor** (named `{kafka-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the JMX agent's HTTP endpoint every 10 seconds. +- **PrometheusRule** is created by the `kafka-alerts` chart and contains alert definitions grouped by concern: database health (which also embeds KubeDB-operator-sourced `KafkaDown`/`KafkaPhaseCritical` alerts) and provisioner. +- **Grafana** dashboards for Kafka are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy Kafka with Monitoring Enabled + +Below is a single-broker Kafka object for this tutorial (a production cluster would use `spec.topology` for separate broker/controller roles). + +```yaml +apiVersion: kubedb.com/v1 +kind: Kafka +metadata: + name: kafka-alert-demo + namespace: alert-kafka +spec: + replicas: 1 + version: "3.9.0" + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.replicas: 1` deploys a single-broker Kafka instance for this tutorial. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the Kafka resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/kafka/monitoring/kafka-alert-demo.yaml +kafka.kubedb.com/kafka-alert-demo created +``` + +Wait for the cluster to go into `Ready` state. + +```bash +$ kubectl get kafka -n alert-kafka kafka-alert-demo +NAME VERSION STATUS AGE +kafka-alert-demo 3.9.0 Ready 3m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-kafka --selector="app.kubernetes.io/instance=kafka-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kafka-alert-demo ClusterIP 10.43.10.20 9092/TCP 3m +kafka-alert-demo-pods ClusterIP None 9092/TCP 3m +kafka-alert-demo-stats ClusterIP 10.43.10.21 56790/TCP 3m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-kafka +NAME AGE +kafka-alert-demo-stats 3m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-kafka kafka-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install kafka-alerts + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression from the **Helm release name** — so the release name must match the Kafka object's name (`kafka-alert-demo`). + +### Install + +```bash +$ helm upgrade -i kafka-alert-demo oci://ghcr.io/appscode-charts/kafka-alerts \ + -n alert-kafka \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-kafka +NAME AGE +kafka-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-kafka kafka-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `kafka.database` and `kafka.provisioner` groups. + +

+ Prometheus Rule Health +

+ +Both groups should show **OK**. `kafka-alerts` v2026.7.14 has no `opsManager`/`stash`/`kubeStash` groups — only `database` and `provisioner`. + +> **Note the overlap:** the `database` group's `KafkaDown` (`for: 30s`) and `KafkaPhaseCritical` (`for: 3m`) key off the same `kubedb_com_kafka_status_phase` metric as the `provisioner` group's `KubeDBKafkaPhaseNotReady`/`KubeDBKafkaPhaseCritical` (`for: 1m`/`15m`) — expect both pairs to eventually fire together during a real outage, at different times. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-kafka%22%7D&g0.tab=1`. + +

+ Prometheus up query — kafka-alert-demo-0 UP +

+ +### 2. Confirm all Kafka alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — Kafka groups inactive +

+ +All rules should show **INACTIVE**. `KafkaTopicCount` and the replication-related alerts (`KafkaUnderReplicatedPartitions`, `KafkaUnderMinIsrPartitionCount`, `KafkaISRExpandRate`/`KafkaISRShrinkRate`) are naturally quiet on a single-broker cluster with no topics yet. + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +### 4. Explore the Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the Kafka dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.Kafka=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `KafkaDown` (`for: 30s`, the fastest down-signal) by crashing the main Kafka JVM process. + +### 1. Crash the Kafka process + +```bash +$ kubectl exec -n alert-kafka kafka-alert-demo-0 -c kafka -- sh -c ' + end=$(( $(date +%s) + 60 )); + while [ $(date +%s) -lt $end ]; do + pid=$(pgrep -f "kafka.Kafka" | head -1); + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; + sleep 1; + done' +``` + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — KafkaDown Firing +

+ +`KafkaDown` (`kubedb_com_kafka_status_phase{phase!="Ready"} == 1`, `for: 30s`) should transition to **FIRING** first. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — KafkaDown Firing +

+ +### 4. Restore Kafka + +Stop the loop from step 1. + +```bash +$ kubectl get kafka -n alert-kafka kafka-alert-demo -w +NAME VERSION STATUS AGE +kafka-alert-demo 3.9.0 Ready 24m +``` + +If Kafka does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-kafka kafka-alert-demo-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `kafka-alert-demo` instance in the `alert-kafka` namespace via `job="kafka-alert-demo-stats"` / `namespace="alert-kafka"` (database group), or `app="kafka-alert-demo"` / `namespace="alert-kafka"` (provisioner group and the two operator-phase alerts embedded in the database group). + +### Database Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KafkaUnderReplicatedPartitions` | warning | 10s | Partitions have fewer in-sync replicas than expected. | +| `KafkaAbnormalControllerState` | warning | 10s | More or fewer than one active controller in the cluster. | +| `KafkaOfflinePartitions` | warning | 10s | One or more partitions have no leader. | +| `KafkaUnderMinIsrPartitionCount` | warning | 10s | Partitions below the minimum in-sync-replica count. | +| `KafkaOfflineLogDirectoryCount` | warning | 10s | A log directory has gone offline (likely a disk issue). | +| `KafkaISRExpandRate` | warning | 1m | ISR set is expanding frequently — sign of flakiness. | +| `KafkaISRShrinkRate` | warning | 1m | ISR set is shrinking frequently — sign of flakiness. | +| `KafkaBrokerCount` | critical | 1m | Broker count has dropped. | +| `KafkaNetworkProcessorIdlePercent` | critical | 1m | Network processor threads are saturated. | +| `KafkaRequestHandlerIdlePercent` | critical | 1m | Request handler threads are saturated. | +| `KafkaReplicaFetcherManagerMaxLag` | critical | 1m | Replica fetcher lag is high. | +| `KafkaTopicCount` | warning | 1m | Topic count changed unexpectedly. | +| `KafkaPhaseCritical` | warning | 3m | KubeDB operator view: resource `Critical` (duplicates the provisioner group's own version at a different `for`). | +| `KafkaDown` | critical | 30s | KubeDB operator view: resource not `Ready`. Fastest down-signal available. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Provisioner Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBKafkaPhaseNotReady` | critical | 1m | KubeDB marked the Kafka resource `NotReady`. | +| `KubeDBKafkaPhaseCritical` | warning | 15m | Kafka is degraded but not fully unavailable. | + +--- + +## Customising Alerts + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + kafkaBrokerCount: + enabled: true + duration: "2m" + severity: critical +``` + +```bash +$ helm upgrade kafka-alert-demo oci://ghcr.io/appscode-charts/kafka-alerts \ + -n alert-kafka \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +```bash +$ helm uninstall kafka-alert-demo -n alert-kafka +$ kubectl delete kafka -n alert-kafka kafka-alert-demo +$ kubectl delete ns alert-kafka +``` + +## Next Steps + +- Monitor your Kafka cluster with KubeDB using [built-in Prometheus](/docs/guides/kafka/monitoring/using-builtin-prometheus.md). +- Monitor your Kafka cluster with KubeDB using [Prometheus operator](/docs/guides/kafka/monitoring/using-prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/mariadb/monitoring/alerting.md b/docs/guides/mariadb/monitoring/alerting.md new file mode 100644 index 0000000000..c199dd0360 --- /dev/null +++ b/docs/guides/mariadb/monitoring/alerting.md @@ -0,0 +1,587 @@ +--- +title: MariaDB Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: guides-mariadb-monitoring-alerting + name: Alerting + parent: guides-mariadb-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# MariaDB Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed MariaDB instance using the `mariadb-alerts` Helm chart, and how to visualise live metrics using the `kubedb-grafana-dashboards` chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-mariadb` namespace: + + ```bash + $ kubectl create ns alert-mariadb + namespace/alert-mariadb created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/mariadb/monitoring/overview/index.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/mariadb](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/mariadb) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +The diagram below shows the full alerting architecture — from MariaDB metric export through to alert delivery and Grafana visualisation. + +

+ MariaDB Alerting Architecture +

+ +- **KubeDB** deploys MariaDB with a `mysqld_exporter`-compatible sidecar (container `exporter`) that exposes metrics used by both MySQL and MariaDB alert charts (`mysql_*` metric names). +- **ServiceMonitor** (named `{mariadb-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `mariadb-alerts` chart and contains MariaDB alert definitions grouped by concern: database health, Galera cluster, provisioner, ops-manager, Stash backup/restore, KubeStash backup/restore, and schema manager. +- **Grafana** visualises metrics through pre-built dashboards provisioned by the `kubedb-grafana-dashboards` chart. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy MariaDB with Monitoring Enabled + +Below is the MariaDB object we are going to create — a 3-node Galera cluster (Primary-Primary multi-master replication) with monitoring enabled. This tutorial uses a real Galera cluster rather than a standalone instance since that's representative of a real deployment and is what the rest of this guide's screenshots are taken from — the `cluster` group's `GaleraReplicationLatencyTooLong` alert only produces real data on a Galera topology; a standalone instance simply leaves it permanently INACTIVE with no series at all. + +```yaml +apiVersion: kubedb.com/v1 +kind: MariaDB +metadata: + name: mariadb-alert-demo + namespace: alert-mariadb +spec: + version: "12.1.2" + deletionPolicy: WipeOut + replicas: 3 + topology: + mode: GaleraCluster + wsrepSSTMethod: rsync + storageType: Durable + storage: + storageClassName: "longhorn" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.topology.mode: GaleraCluster` tells KubeDB to bootstrap a multi-master Galera cluster instead of a standalone/async-replica instance. +- `spec.wsrepSSTMethod: rsync` selects the State Snapshot Transfer method Galera uses to bring a rejoining node's dataset back in sync with the cluster. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/mariadb/monitoring/mariadb-alert-demo.yaml +mariadb.kubedb.com/mariadb-alert-demo created +``` + +Wait for the database to go into `Ready` state. + +```bash +$ kubectl get mariadb -n alert-mariadb mariadb-alert-demo +NAME VERSION STATUS AGE +mariadb-alert-demo 12.1.2 Ready 21h +``` + +KubeDB brings up 3 Galera pods, each running as a Primary: + +```bash +$ kubectl get pods -n alert-mariadb +NAME READY STATUS RESTARTS AGE +mariadb-alert-demo-0 3/3 Running 0 21h +mariadb-alert-demo-1 3/3 Running 0 21h +mariadb-alert-demo-2 3/3 Running 0 21h +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-mariadb --selector="app.kubernetes.io/instance=mariadb-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +mariadb-alert-demo ClusterIP 10.43.111.157 3306/TCP 21h +mariadb-alert-demo-pods ClusterIP None 3306/TCP 21h +mariadb-alert-demo-stats ClusterIP 10.43.30.195 56790/TCP 21h +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-mariadb +NAME AGE +mariadb-alert-demo-stats 21h +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-mariadb mariadb-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install mariadb-alerts + +The `mariadb-alerts` chart creates a `PrometheusRule` resource containing all MariaDB alert definitions. + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the MariaDB object's name (`mariadb-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### A note on chart defaults + +`mariadb-alerts` has one chart-level bug worth knowing about before you install: `diskUsageHigh` / `diskAlmostFull` compute PVC usage as `kubelet_volume_stats_used_bytes / (kubelet_volume_stats_used_bytes + kube_pod_spec_volumes_persistentvolumeclaims_info)`. The `..._info` series is a constant label metric (always `1`), not a byte count, so this expression evaluates to ~100% regardless of actual usage — a chart-level expression defect (the same one documented for several other `*-alerts` charts in this project). Confirmed on this instance: `df -h /var/lib/mysql` showed real usage at **31%**, but both alerts were firing permanently because the broken expression read them as ~100%. Unlike Elasticsearch's `elasticsearch-alerts` chart, `mariadb-alerts` has **no accurate alternative** disk-space rule to fall back on — so the fix here is simply to disable both. + +### Install + +```bash +$ helm upgrade -i mariadb-alert-demo oci://ghcr.io/appscode-charts/mariadb-alerts \ + -n alert-mariadb \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `mariadb-alert-demo` (release name) | — | Scopes every PromQL expression to this instance. **This must exactly match the MariaDB object's name** — see [above](#why-the-helm-release-name-matters). | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | +| `...diskUsageHigh.enabled` / `...diskAlmostFull.enabled` | `false` | Works around the PVC-usage expression defect described above | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-mariadb +NAME AGE +mariadb-alert-demo 30s + +$ kubectl get prometheusrule -n alert-mariadb mariadb-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=mariadb` and locate the `mariadb.database`, `mariadb.cluster`, `mariadb.provisioner`, `mariadb.opsManager`, `mariadb.stash`, `mariadb.kubeStash`, and `mariadb.schemaManager` groups. + +

+ Prometheus Rule Health +

+ +All groups show **OK**, confirming that Prometheus has loaded and is evaluating the MariaDB alert definitions every 30 seconds. Unlike several other `*-alerts` charts in this project, `mariadb-alerts` v2026.7.14 renders every group declared in its `values.yaml` — no missing-group gap found here. Note `mariadb.database` now has 11 rules, not 13 — `diskUsageHigh`/`diskAlmostFull` are gone entirely rather than merely disabled-and-hidden, since a disabled rule isn't rendered into the `PrometheusRule` at all. + +--- + +## Step 2 — Install kubedb-grafana-dashboards + +The `kubedb-grafana-dashboards` chart creates `GrafanaDashboard` CRDs containing pre-built MariaDB dashboard JSON. A separate controller, `grafana-operator`, watches these CRDs and pushes the dashboards into Grafana over its HTTP API — both pieces are required. If you've already set these up for another database on this cluster (see the [Elasticsearch alerting guide](/docs/guides/elasticsearch/monitoring/alerting.md) for the full walkthrough), skip straight to [Install the dashboards](#install-the-dashboards) below. + +### Install grafana-operator + +If your cluster doesn't already have it (check with `kubectl get crd grafanadashboards.openviz.dev`): + +```bash +$ helm upgrade -i grafana-operator appscode/grafana-operator \ + -n kubeops --create-namespace \ + --version=v2026.6.12 \ + --wait +``` + +### Mark your Grafana instance as the cluster default + +Skip this if you already have a Grafana `AppBinding` annotated as the cluster default (one is shared across every database). Otherwise: + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 & +$ GRAFANA_PW=$(kubectl get secret -n monitoring prometheus-grafana -o jsonpath='{.data.admin-password}' | base64 -d) +$ curl -s -X POST -H "Content-Type: application/json" -u admin:$GRAFANA_PW \ + http://localhost:3000/api/auth/keys \ + -d '{"name":"kubedb-dashboards","role":"Admin"}' +# Note the returned "key" +$ kill %1 +``` + +```yaml +# grafana-appbinding.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-token + namespace: kubeops +type: Opaque +stringData: + token: "" +--- +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + name: grafana + namespace: kubeops + annotations: + monitoring.appscode.com/is-default-grafana: "true" # must be an ANNOTATION, not a label +spec: + type: monitoring.appscode.com/grafana + clientConfig: + url: "http://prometheus-grafana.monitoring.svc:80" + secret: + name: grafana-admin-token +``` + +```bash +$ kubectl apply -f grafana-appbinding.yaml +``` + +### Install the dashboards + +```bash +$ helm repo add appscode https://charts.appscode.com/stable/ +$ helm repo update appscode + +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.MariaDB=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ +``` + +> **Note:** `featureGates.` defaults to `true` for almost every database in this chart, so one `helm template | kubectl apply` installs dashboards for many databases at once, not just MariaDB — this is expected. See the render-vs-Secret-size caveat in the [Elasticsearch alerting guide](/docs/guides/elasticsearch/monitoring/alerting.md#install-the-dashboards) for why `helm template | kubectl apply` is used instead of `helm install`. + +### Verify dashboards are created + +```bash +$ kubectl get grafanadashboards -n kubeops | grep mariadb +NAME TITLE STATUS AGE +kubedb-mariadb-database KubeDB / MariaDB / Database Current 2m +kubedb-mariadb-galera-cluster KubeDB / MariaDB / Galera-Cluster Current 2m +kubedb-mariadb-pod KubeDB / MariaDB / Pod Current 2m +kubedb-mariadb-summary KubeDB / MariaDB / Summary Current 2m +``` + +Four dashboards this time, not three — MariaDB's chart ships a dedicated **Galera-Cluster** dashboard alongside the usual Summary/Pod/Database triplet. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-mariadb%22%7D&g0.tab=1`. + +

+ Prometheus up query — all 3 mariadb-alert-demo pods UP +

+ +All 3 pods (`mariadb-alert-demo-0/1/2`) should report `up == 1` via the `mariadb-alert-demo-stats` service/job. + +### 2. Confirm the MariaDB alerts are inactive + +Open `http://localhost:9090/alerts?search=mariadb`. + +

+ Prometheus Alerts — MariaDB groups inactive +

+ +All rules show **INACTIVE**, including `GaleraReplicationLatencyTooLong` (the `cluster` group) — on a real Galera topology this rule has live data (unlike a standalone instance, where it would have none at all), it's just currently below threshold. + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +No alerts should be firing for the `alert-mariadb` namespace. + +### 4. Explore Grafana dashboards + +Port-forward Grafana and log in. + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 +``` + +Open `http://localhost:3000` (username: `admin`). Search for **mariadb** in the Dashboards section. + +

+ Grafana — MariaDB Dashboard List +

+ +Four pre-built dashboards are available. The `Namespace` and `MariaDB` drop-downs at the top of each dashboard let you switch between instances. + +**KubeDB / MariaDB / Summary** — database status, version, node count, CPU/memory/storage requests vs. usage. + +

+ Grafana — KubeDB MariaDB Summary +

+ +**KubeDB / MariaDB / Galera-Cluster** — cluster name, per-node ONLINE/Primary status, and Galera replication latency (average, standard deviation, sample size) per node. + +

+ Grafana — KubeDB MariaDB Galera-Cluster +

+ +**KubeDB / MariaDB / Database** — per-pod service status/uptime, cluster size, primary status, QPS, connections, disk I/O, and top command counters. + +

+ Grafana — KubeDB MariaDB Database +

+ +**KubeDB / MariaDB / Pod** — per-pod CPU/memory/file descriptors, connections, thread activity, temporary objects, slow queries, table locks, and network traffic. + +

+ Grafana — KubeDB MariaDB Pod +

+ +--- + +## Simulating a Firing Alert + +This section deliberately triggers `MariaDBInstanceDown` (instant, `for: 0m`) by crashing the main `mariadb` process, and observes the alert through Prometheus and AlertManager. + +Unlike Elasticsearch, killing the main process here works well: MariaDB's container `PID 1` is `tini` supervising a wrapper script, not `mariadbd` itself, so killing `mariadbd` doesn't take the container down — it just leaves `mysql_up` at `0` until the script notices and restarts the daemon. A single `kill -9` self-heals in roughly 20–30 seconds (too fast to reliably catch, since it beats one evaluation cycle), so hold it down with a short kill-loop instead. + +### 1. Crash the MariaDB process + +```bash +$ kubectl exec -n alert-mariadb mariadb-alert-demo-0 -c mariadb -- sh -c ' + end=$(( $(date +%s) + 45 )); + while [ $(date +%s) -lt $end ]; do + pid=$(pgrep -x mariadbd | head -1); + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; + sleep 1; + done' +``` + +Run this in the background (or a separate terminal) — it holds `mariadbd` down for 45 seconds, comfortably past one Prometheus scrape (10s) and evaluation (30s) cycle. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=mariadb`. + +

+ Prometheus Alerts — MariaDBInstanceDown Firing +

+ +`MariaDBInstanceDown` (`mysql_up == 0`) transitions straight to **FIRING** since it has no `for` delay, while the rest of the `mariadb.database` group stays **INACTIVE**. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter={namespace="alert-mariadb"}`. + +

+ AlertManager — MariaDBInstanceDown Firing +

+ +AlertManager shows the `MariaDBInstanceDown` alert. The alert card displays: + +- **Severity**: `critical` +- **pod**: `mariadb-alert-demo-0` +- **job**: `mariadb-alert-demo-stats` +- **Started**: timestamp when the alert first fired + +### 4. Restore MariaDB + +Let the loop from step 1 finish (or stop it early). `run.sh` inside the container restarts `mariadbd` on its own — no pod restart needed. + +```bash +$ kubectl get mariadb -n alert-mariadb mariadb-alert-demo -w +NAME VERSION STATUS AGE +mariadb-alert-demo 12.1.2 Critical 21h +mariadb-alert-demo 12.1.2 Ready 21h +``` + +Recovery took about 10–15 seconds after the kill-loop ended in testing — `mariadbd` restarts, performs a quick Galera State Snapshot Transfer (SST via `rsync`) to catch back up with the other two nodes, and `mysql_up` returns to `1`. The **KubeDB / MariaDB / Galera-Cluster** dashboard's replication-latency panel is a good place to watch this recovery happen in real time. Once `mysql_up` is back to `1`, Prometheus marks the alert **INACTIVE** and AlertManager sends a **resolved** notification. If MariaDB does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-mariadb mariadb-alert-demo-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `mariadb-alert-demo` instance in the `alert-mariadb` namespace via the PromQL label filters `job="mariadb-alert-demo-stats"` / `namespace="alert-mariadb"` (database/cluster groups), or `app="mariadb-alert-demo"` / `namespace="alert-mariadb"` (provisioner/opsManager/stash/kubeStash/schemaManager groups). + +### Database Group + +Fired from metrics exposed by the `mysqld_exporter`-compatible sidecar and node/kubelet metrics. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MariaDBInstanceDown` | critical | instant | `mysql_up == 0` on this instance. | +| `MariaDBServiceDown` | critical | instant | No replica behind the service is answering. | +| `MariaDBTooManyConnections` | warning | 2m | Connection count is high relative to `max_connections`. | +| `MariaDBHighThreadsRunning` | warning | 2m | Too many threads actively running. | +| `MariaDBSlowQueries` | warning | 2m | Slow-query count is increasing. | +| `MariaDBInnoDBLogWaits` | warning | instant | InnoDB log waits are occurring — I/O may be a bottleneck. | +| `MariaDBRestarted` | warning | instant | Uptime indicates a recent restart. | +| `MariaDBHighQPS` | critical | instant | Query rate is unusually high. | +| `MariaDBHighIncomingBytes` | critical | instant | Inbound network traffic is unusually high. | +| `MariaDBHighOutgoingBytes` | critical | instant | Outbound network traffic is unusually high. | +| `MariaDBTooManyOpenFiles` | warning | 2m | Open file count is high relative to the limit. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. **Disabled in this tutorial** — see [above](#a-note-on-chart-defaults): the expression is a chart-level defect that always reads ~100% regardless of real usage, with no accurate alternative in this chart. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. **Disabled in this tutorial** — same defect as `DiskUsageHigh` above. | + +### Cluster Group + +Only produces data when `spec.topology` (Galera) is configured — this tutorial's instance is a Galera cluster, so this group has live data. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `GaleraReplicationLatencyTooLong` | warning | 5m | Galera replication latency is high. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the MariaDB resource phase (sourced from Panopticon, not the exporter's metrics). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMariaDBPhaseNotReady` | critical | 1m | KubeDB marked the MariaDB resource `NotReady`. | +| `KubeDBMariaDBPhaseCritical` | warning | 15m | MariaDB is degraded but not fully unavailable. | + +### OpsManager Group + +Monitors the lifecycle of ops requests (upgrades, scaling, reconfiguration, etc.) issued against this MariaDB instance. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMariaDBOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes. | +| `KubeDBMariaDBOpsRequestFailed` | critical | instant | An ops request failed. | + +### Stash / KubeStash Groups + +Only meaningful once Stash or KubeStash backup/restore is configured. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MariaDBStashBackupSessionFailed` / `MariaDBKubeStashBackupSessionFailed` | critical | instant | The most recent backup session failed. | +| `MariaDBStashRestoreSessionFailed` / `MariaDBKubeStashRestoreSessionFailed` | critical | instant | The most recent restore session failed. | +| `MariaDBStashNoBackupSessionForTooLong` / `MariaDBKubeStashNoBackupSessionForTooLong` | warning | instant | No recent successful backup. | +| `MariaDBStashRepositoryCorrupted` / `MariaDBKubeStashRepositoryCorrupted` | critical | 5m | Backup repository integrity check failed. | +| `MariaDBStashRepositoryStorageRunningLow` / `MariaDBKubeStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage usage is high. | +| `MariaDBStashBackupSessionPeriodTooLong` / `MariaDBKubeStashBackupSessionPeriodTooLong` | warning | instant | A backup session is taking unusually long. | +| `MariaDBStashRestoreSessionPeriodTooLong` / `MariaDBKubeStashRestoreSessionPeriodTooLong` | warning | instant | A restore session is taking unusually long. | + +### SchemaManager Group + +Only meaningful when using `MariaDBDatabase` schema-manager objects. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMariaDBSchemaPendingForTooLong` | warning | 30m | A `MariaDBDatabase` object stuck `Pending`. | +| `KubeDBMariaDBSchemaInProgressForTooLong` | warning | 30m | A `MariaDBDatabase` object stuck `InProgress`. | +| `KubeDBMariaDBSchemaTerminatingForTooLong` | warning | 30m | A `MariaDBDatabase` object stuck `Terminating`. | +| `KubeDBMariaDBSchemaFailed` | warning | instant | A `MariaDBDatabase` object failed. | +| `KubeDBMariaDBSchemaExpired` | warning | instant | A `MariaDBDatabase` object expired. | + +--- + +## Customising Alerts + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + mariadbTooManyConnections: + enabled: true + duration: "5m" + severity: warning + cluster: + enabled: "none" # disable if you don't run Galera +``` + +```bash +$ helm upgrade mariadb-alert-demo oci://ghcr.io/appscode-charts/mariadb-alerts \ + -n alert-mariadb \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +```bash +# Remove the Grafana dashboards (installed via helm template | kubectl apply, not helm install) +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.MariaDB=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl delete -n kubeops -f - --ignore-not-found + +# Remove the mariadb-alerts release +$ helm uninstall mariadb-alert-demo -n alert-mariadb + +# Remove the MariaDB instance +$ kubectl delete mariadb -n alert-mariadb mariadb-alert-demo + +# Delete namespace +$ kubectl delete ns alert-mariadb + +# Optional: only if nothing else in the cluster depends on them +$ kubectl delete appbinding -n kubeops grafana +$ kubectl delete secret -n kubeops grafana-admin-token +$ helm uninstall grafana-operator -n kubeops +``` + +## Next Steps + +- Monitor your MariaDB database with KubeDB using [built-in Prometheus](/docs/guides/mariadb/monitoring/builtin-prometheus/index.md). +- Monitor your MariaDB database with KubeDB using [Prometheus operator](/docs/guides/mariadb/monitoring/prometheus-operator/index.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/memcached/monitoring/alerting.md b/docs/guides/memcached/monitoring/alerting.md new file mode 100644 index 0000000000..2a3c679a98 --- /dev/null +++ b/docs/guides/memcached/monitoring/alerting.md @@ -0,0 +1,385 @@ +--- +title: Memcached Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: mc-monitoring-alerting + name: Alerting + parent: mc-monitoring-memcached + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Memcached Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Memcached instance using the `memcached-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. See the [Grafana Dashboard](grafana-dashboard.md#configuration) guide for how to deploy kube-prometheus-stack if you don't have it yet. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/memcached/monitoring/overview.md). + +* For dashboards and visualisation, see [Grafana Dashboard](grafana-dashboard.md) for Memcached. + +> Note: YAML files used in this tutorial are stored in [docs/examples/memcached](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/memcached) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Memcached with a built-in exporter sidecar that exposes metrics on port `56790`. +- **ServiceMonitor** (named `{memcached-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `memcached-alerts` chart and contains all Memcached alert definitions grouped by concern: database health, provisioner, and ops-manager. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +

+ Monitoring process of Memcached using Prometheus Operator +

+ +--- + +## Deploy Memcached with Monitoring Enabled + +At first, let's deploy a Memcached database with monitoring enabled. Below is the Memcached object we are going to create. + +```yaml +apiVersion: kubedb.com/v1 +kind: Memcached +metadata: + name: mc-alert-demo + namespace: demo +spec: + replicas: 1 + version: "1.6.40" + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the Memcached resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/memcached/monitoring/mc-alert-demo.yaml +memcached.kubedb.com/mc-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get memcached -n demo mc-alert-demo +NAME VERSION STATUS AGE +mc-alert-demo 1.6.40 Ready 30s +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n demo --selector="app.kubernetes.io/instance=mc-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +mc-alert-demo ClusterIP 10.43.157.50 11211/TCP 30s +mc-alert-demo-pods ClusterIP None 11211/TCP 30s +mc-alert-demo-stats ClusterIP 10.43.157.43 56790/TCP 30s +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n demo +NAME AGE +mc-alert-demo-stats 30s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n demo mc-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install memcached-alerts + +The `memcached-alerts` chart creates a `PrometheusRule` resource containing all Memcached alert definitions grouped by concern: database health, provisioner, and ops-manager. + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the Memcached object's name (`mc-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i mc-alert-demo oci://ghcr.io/appscode-charts/memcached-alerts \ + -n demo \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `mc-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`job="mc-alert-demo-stats"`) | +| `-n demo` | `demo` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n demo +NAME AGE +mc-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n demo mc-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=memcached`. + +

+ Prometheus Rule Health +

+ +The `memcached.database.demo.mc-alert-demo.rules` group is visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Memcached alert definitions every 30 seconds. + +--- + +## Verify End-to-End + +### 1. Check the exporter is running + +The `exporter` sidecar inside the Memcached pod serves metrics at `:56790/metrics`. A value of `memcached_up 1` confirms the exporter can reach Memcached. + +```bash +$ kubectl exec -n demo mc-alert-demo-0 -c exporter -- \ + wget -qO- localhost:56790/metrics | grep memcached_up +memcached_up 1 +``` + +### 2. Check the Prometheus target is UP + +Open `http://localhost:9090/targets?search=mc-alert-demo`. + +

+ Prometheus Target UP +

+ +The target `serviceMonitor/demo/mc-alert-demo-stats/0` shows **UP**, confirming metrics are being scraped from `mc-alert-demo-0` in the `demo` namespace. + +### 3. Confirm all Memcached alerts are inactive + +Open `http://localhost:9090/alerts?search=memcached` to see the Memcached alert groups. + +

+ Prometheus Alerts — All Inactive +

+ +All 6 rules in the `memcached.database` group show **INACTIVE (6)**, meaning the database is healthy and no thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy Memcached instance, no alerts for `mc-alert-demo` will be listed here. + +--- + +## Simulating a Firing Alert + +The previous section confirmed that all alerts are **INACTIVE** while the database is healthy. This section walks through deliberately triggering the `MemcachedDown` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +### 1. Stop the Memcached process + +Kill the `memcached` process inside the pod. This crashes the main container so the `exporter` sidecar can no longer reach it and reports `memcached_up 0` on the next scrape, while Kubernetes restarts the crashed container in the background. + +```bash +$ kubectl exec -n demo mc-alert-demo-0 -c memcached -- kill 1 +``` + +Wait 30–60 seconds for the next Prometheus scrape cycle (configured at 10 s) and rule-evaluation cycle (30 s) to register the failure. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=memcached`. + +

+ Prometheus Alerts — MemcachedDown Firing +

+ +Because `MemcachedDown` has `for: 0m` (instant), it moves directly from **INACTIVE** to **FIRING** within one evaluation cycle. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — MemcachedDown Firing +

+ +AlertManager shows the `MemcachedDown` alert. The alert card displays: + +- **Severity**: `critical` +- **Instance**: `mc-alert-demo-0` in the `demo` namespace +- **job**: `mc-alert-demo-stats` +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore Memcached + +Delete the pod so KubeDB recreates it cleanly. + +```bash +$ kubectl delete pod -n demo mc-alert-demo-0 +``` + +Once `memcached_up` returns to `1`, Prometheus marks the alert **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `mc-alert-demo` instance in the `demo` namespace via the PromQL label filters `job="mc-alert-demo-stats"` and `namespace="demo"`. + +### Database Group + +Fired based on live metrics from the Memcached exporter. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MemcachedDown` | critical | instant | Exporter cannot reach Memcached — instance is down or crashed. | +| `MemcachedServiceRespawn` | critical | instant | Memcached restarted recently (uptime < 180s). | +| `MemcachedConnectionThrottled` | warning | 2m | More than 10 connections were yielded/throttled in the last minute — client pool nearly exhausted. | +| `MemcachedConnectionsNoneMinor` | warning | 2m | No open client connections — application may not be talking to this instance. | +| `MemcachedItemsNoneMinor` | warning | 2m | Cache is empty — no items stored, possibly indicating a flush or misconfiguration. | +| `memcachedEvictionsLimit` | critical | instant | More than 10 evictions — cache is undersized for the current workload. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Memcached resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `appPhaseNotReady` | critical | 1m | KubeDB marked the Memcached resource `NotReady` — operator cannot reach the database. | +| `appPhaseCritical` | warning | 15m | The instance is in a degraded/critical phase. | + +### OpsManager Group + +Tracks `MemcachedOpsRequest` lifecycle during upgrades, scaling, and reconfiguration. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `opsRequestOnProgress` | info | instant | An ops request is currently in progress. | +| `opsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes — likely stuck. | +| `opsRequestFailed` | critical | instant | An ops request failed — check the `OpsRequest` object for the error. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + memcachedConnectionThrottled: + enabled: true + duration: "5m" + val: 20 # fire at 20 throttled connections instead of the default 10 + severity: warning + opsManager: + enabled: "none" # disable all ops-manager alerts +``` + +```bash +$ helm upgrade mc-alert-demo oci://ghcr.io/appscode-charts/memcached-alerts \ + -n demo \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the memcached-alerts release +$ helm uninstall mc-alert-demo -n demo + +# Remove the Memcached instance +$ kubectl delete memcached -n demo mc-alert-demo + +# Delete namespace +$ kubectl delete ns demo +``` + +## Next Steps + +- Monitor your Memcached database with KubeDB using [builtin Prometheus](/docs/guides/memcached/monitoring/using-builtin-prometheus.md). +- Monitor your Memcached database with KubeDB using [Prometheus operator](/docs/guides/memcached/monitoring/using-prometheus-operator.md). +- Visualise Memcached metrics with [Grafana Dashboard](grafana-dashboard.md). +- Use [private Docker registry](/docs/guides/memcached/private-registry/using-private-registry.md) to deploy Memcached with KubeDB. +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/mongodb/monitoring/alerting.md b/docs/guides/mongodb/monitoring/alerting.md new file mode 100644 index 0000000000..001d3dd1b0 --- /dev/null +++ b/docs/guides/mongodb/monitoring/alerting.md @@ -0,0 +1,389 @@ +--- +title: MongoDB Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: mg-monitoring-alerting + name: Alerting + parent: mg-monitoring-mongodb + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# MongoDB Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed MongoDB instance using the `mongodb-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-mongodb` namespace: + + ```bash + $ kubectl create ns alert-mongodb + namespace/alert-mongodb created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/mongodb/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/mongodb](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/mongodb) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys MongoDB with a `mongodb_exporter` sidecar (container `exporter`) that exposes metrics (`mongodb_*`). +- **ServiceMonitor** (named `{mongodb-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `mongodb-alerts` chart and contains MongoDB alert definitions grouped by concern: database health (which also embeds the KubeDB-operator-sourced `MongoDBDown`/`MongoDBPhaseCritical` pair), provisioner, ops-manager, Stash backup/restore, KubeStash backup/restore, and schema manager. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). +- **Grafana** dashboards for MongoDB are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. + +--- + +## Deploy MongoDB with Monitoring Enabled + +At first, let's deploy a 3-member MongoDB replica set with monitoring enabled. Below is the MongoDB object we are going to create. + +```yaml +apiVersion: kubedb.com/v1 +kind: MongoDB +metadata: + name: mongodb-alert-demo + namespace: alert-mongodb +spec: + version: "8.0.17" + replicaSet: + name: "rs1" + replicas: 3 + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.replicaSet.name: "rs1"` and `spec.replicas: 3` create a 3-member MongoDB replica set. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the MongoDB resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/mongodb/monitoring/mongodb-alert-demo.yaml +mongodb.kubedb.com/mongodb-alert-demo created +``` + +Wait for the database to go into `Ready` state. + +```bash +$ kubectl get mongodb -n alert-mongodb mongodb-alert-demo +NAME VERSION STATUS AGE +mongodb-alert-demo 8.0.17 Ready 3m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-mongodb --selector="app.kubernetes.io/instance=mongodb-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +mongodb-alert-demo ClusterIP 10.43.10.20 27017/TCP 3m +mongodb-alert-demo-pods ClusterIP None 27017/TCP 3m +mongodb-alert-demo-stats ClusterIP 10.43.10.21 56790/TCP 3m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-mongodb +NAME AGE +mongodb-alert-demo-stats 3m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-mongodb mongodb-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install mongodb-alerts + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the MongoDB object's name (`mongodb-alert-demo`). + +### Install + +> **Chart bug — disable `diskUsageHigh`/`diskAlmostFull` at install time:** at chart version `v2026.7.14`, both alerts divide by the wrong denominator — `kubelet_volume_stats_used_bytes / (kubelet_volume_stats_used_bytes + kube_pod_spec_volumes_persistentvolumeclaims_info)` instead of `kubelet_volume_stats_capacity_bytes`. `kube_pod_spec_volumes_persistentvolumeclaims_info` is just a presence marker that's always `1`, so the denominator ≈ the numerator and the computed usage reads ~99.9999% **regardless of real usage** — confirmed on a live cluster where the real PVC usage (`kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes`) was 68.5% while both alerts read 99.9999% and fired. Same bug already found in `elasticsearch-alerts` and `mariadb-alerts` — no accurate alternative expression exists in this chart, so disable both rules rather than chase a working threshold: + +```bash +$ helm upgrade -i mongodb-alert-demo oci://ghcr.io/appscode-charts/mongodb-alerts \ + -n alert-mongodb \ + --create-namespace \ + --version=v2026.7.14 +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-mongodb +NAME AGE +mongodb-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-mongodb mongodb-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `mongodb.database`, `mongodb.provisioner`, `mongodb.opsManager`, `mongodb.stash`, `mongodb.kubeStash`, and `mongodb.schemaManager` groups. + +

+ Prometheus Rule Health +

+ +All groups should show **OK**. Every group declared in `values.yaml` is rendered for `mongodb-alerts` v2026.7.14 — no missing-group gap found here. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-mongodb%22%7D&g0.tab=1`. + +

+ Prometheus up query — mongodb-alert-demo-0 UP +

+ +### 2. Confirm the MongoDB alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — MongoDB groups inactive +

+ +All rules should show **INACTIVE**, including `MongoDBDown` and `MongoDBPhaseCritical` — note these two are placed inside the `database` group even though, like the `provisioner` group's alerts, they key off `kubedb_com_mongodb_status_phase` (the KubeDB operator's own view), not a MongoDB-native metric. `MongoDBDown` fires much faster (`for: 30s`) than the provisioner group's `KubeDBMongoDBPhaseNotReady` (`for: 1m`). + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +### 4. Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the MongoDB dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.MongoDB=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `MongoDBDown` (the fastest of the down-detection alerts, `for: 30s`) so you can observe the full alert lifecycle. + +> **`kill -9` does not work on this image — confirmed, not assumed.** MongoDB's container has no `tini`/supervisor wrapper: `mongod` runs directly as the container's own `PID 1` (`kubectl exec ... ps aux` shows `PID 1 mongod ...`). `readlink /proc/1/ns/pid` and `readlink /proc/self/ns/pid` inside the container return the identical inode, confirming a `kubectl exec` session shares that same PID namespace — and Linux unconditionally ignores `SIGKILL`/`SIGSTOP` sent to a namespace's PID 1 from within that same namespace (`man 7 pid_namespaces`). A kill-loop targeting `pgrep -x mongod` therefore reports success on every iteration while doing nothing: 0 pod restarts, phase stays `Ready` throughout. Any KubeDB image without a supervisor wrapper in front of its main process will behave the same way. +> +> **What actually works:** `MongoDBDown` (like `KubeDBMongoDBPhaseNotReady`) is driven by the KubeDB operator's own view of the resource `phase`, not a per-pod metric — so disrupting just one pod of a 3-member replica set isn't enough (the other two keep serving and the operator still reports `Ready`). Force-deleting *all* member pods in a repeating loop reliably denies the resource enough healthy members to stay `Ready` for the duration of the loop. + +### 1. Disrupt every MongoDB pod + +```bash +$ end=$(( $(date +%s) + 90 )) +while [ $(date +%s) -lt $end ]; do + kubectl delete pod -n alert-mongodb -l app.kubernetes.io/instance=mongodb-alert-demo --grace-period=0 --force >/dev/null 2>&1 + sleep 3 + done +``` + +Run this in the background (or a separate terminal) — repeatedly force-deleting all 3 pods keeps the replica set from having a stable `Ready` majority for the duration of the loop, which the KubeDB operator reports as the resource leaving `Ready`. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — MongoDBDown Firing +

+ +`MongoDBDown` (`kubedb_com_mongodb_status_phase{phase!="Ready"} == 1`, `for: 30s`) should transition to **FIRING** once the KubeDB operator observes the resource leaving `Ready`. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — MongoDBDown Firing +

+ +### 4. Restore MongoDB + +Let the loop from step 1 finish (or stop it early) — the StatefulSet recreates all 3 pods on its own once nothing is deleting them anymore. + +```bash +$ kubectl get pods -n alert-mongodb +NAME READY STATUS RESTARTS AGE +mongodb-alert-demo-0 3/3 Running 2 65s +mongodb-alert-demo-1 3/3 Running 0 37s +mongodb-alert-demo-2 3/3 Running 0 21s + +$ kubectl get mongodb -n alert-mongodb mongodb-alert-demo -w +NAME VERSION STATUS AGE +mongodb-alert-demo 8.0.17 Ready 65m +``` + +Once all 3 pods are stably `Running` and the replica set has re-elected a primary, Prometheus marks `MongoDBDown` **INACTIVE** again and AlertManager sends a **resolved** notification. In testing this took well under a minute after the disruption loop ended. + +--- + +## Alert Reference + +All alerts are scoped to the `mongodb-alert-demo` instance in the `alert-mongodb` namespace via the PromQL label filters `job="mongodb-alert-demo-stats"` / `namespace="alert-mongodb"` (most of the database group), or `app="mongodb-alert-demo"` / `namespace="alert-mongodb"` (provisioner/opsManager/stash/kubeStash/schemaManager groups, plus the two operator-phase alerts embedded in the database group). + +### Database Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MongodbVirtualMemoryUsage` | warning | 1m | Virtual memory usage is high. | +| `MongodbReplicationLag` | critical | instant | Replica set member is lagging behind the primary. | +| `MongodbNumberCursorsOpen` | warning | 2m | Too many open cursors. | +| `MongodbCursorsTimeouts` | warning | 2m | Cursor timeout rate is increasing. | +| `MongodbTooManyConnections` | warning | 2m | Connection growth rate is high. | +| `MongoDBPhaseCritical` | warning | 10m | KubeDB operator view: resource `Critical` (embedded here, duplicates provisioner group's own version at a different `for`). | +| `MongoDBDown` | critical | 30s | KubeDB operator view: resource not `Ready`. Fastest down-signal available for MongoDB. | +| `MongoDBHighLatency` | warning | 10m | Operation latency is elevated. | +| `MongoDBHighTicketUtilization` | warning | 10m | WiredTiger concurrency tickets are close to exhausted. | +| `MongoDBRecurrentCursorTimeout` | warning | 30m | Cursor timeouts recurring over a longer window. | +| `MongoDBRecurrentMemoryPageFaults` | warning | 30m | Page faults recurring over a longer window. | +| `DiskUsageHigh` | warning | 1m | **Disabled by the install command above** — broken denominator always reads ~100% usage regardless of real usage. See the callout in [Step 1](#install). | +| `DiskAlmostFull` | critical | 1m | **Disabled by the install command above** — same broken-denominator bug as `DiskUsageHigh`. | + +### Provisioner Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMongoDBPhaseNotReady` | critical | 1m | KubeDB marked the MongoDB resource `NotReady`. | +| `KubeDBMongoDBPhaseCritical` | warning | 15m | MongoDB is degraded but not fully unavailable. | + +### OpsManager Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMongoDBOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes. | +| `KubeDBMongoDBOpsRequestFailed` | critical | instant | An ops request failed. | + +### Stash / KubeStash Groups + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MongoDBStashBackupSessionFailed` / `MongoDBKubeStashBackupSessionFailed` | critical | instant | Most recent backup session failed. | +| `MongoDBStashRestoreSessionFailed` / `MongoDBKubeStashRestoreSessionFailed` | critical | instant | Most recent restore session failed. | +| `MongoDBStashNoBackupSessionForTooLong` / `MongoDBKubeStashNoBackupSessionForTooLong` | warning | instant | No recent successful backup. | +| `MongoDBStashRepositoryCorrupted` / `MongoDBKubeStashRepositoryCorrupted` | critical | 5m | Backup repository integrity check failed. | +| `MongoDBStashRepositoryStorageRunningLow` / `MongoDBKubeStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage usage is high. | +| `MongoDBStashBackupSessionPeriodTooLong` / `MongoDBKubeStashBackupSessionPeriodTooLong` | warning | instant | Backup session taking unusually long. | +| `MongoDBStashRestoreSessionPeriodTooLong` / `MongoDBKubeStashRestoreSessionPeriodTooLong` | warning | instant | Restore session taking unusually long. | + +### SchemaManager Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMongoDBSchemaPendingForTooLong` | warning | 30m | A `MongoDBDatabase` object stuck `Pending`. | +| `KubeDBMongoDBSchemaInProgressForTooLong` | warning | 30m | A `MongoDBDatabase` object stuck `InProgress`. | +| `KubeDBMongoDBSchemaTerminatingForTooLong` | warning | 30m | A `MongoDBDatabase` object stuck `Terminating`. | +| `KubeDBMongoDBSchemaFailed` | warning | instant | A `MongoDBDatabase` object failed. | +| `KubeDBMongoDBSchemaExpired` | warning | instant | A `MongoDBDatabase` object expired. | + +--- + +## Customising Alerts + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + mongodbTooManyConnections: + enabled: true + duration: "5m" + severity: warning +``` + +```bash +$ helm upgrade mongodb-alert-demo oci://ghcr.io/appscode-charts/mongodb-alerts \ + -n alert-mongodb \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +```bash +$ helm uninstall mongodb-alert-demo -n alert-mongodb +$ kubectl delete mongodb -n alert-mongodb mongodb-alert-demo +$ kubectl delete ns alert-mongodb +``` + +## Next Steps + +- Monitor your MongoDB database with KubeDB using [built-in Prometheus](/docs/guides/mongodb/monitoring/using-builtin-prometheus.md). +- Monitor your MongoDB database with KubeDB using [Prometheus operator](/docs/guides/mongodb/monitoring/using-prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/mssqlserver/monitoring/alerting.md b/docs/guides/mssqlserver/monitoring/alerting.md new file mode 100644 index 0000000000..a4d3420568 --- /dev/null +++ b/docs/guides/mssqlserver/monitoring/alerting.md @@ -0,0 +1,407 @@ +--- +title: MSSQLServer Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: ms-monitoring-alerting + name: Alerting + parent: ms-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# MSSQLServer Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Microsoft SQL Server instance using the `mssqlserver-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Install [cert-manager](https://cert-manager.io/docs/installation/) — MSSQLServer requires TLS, issued via a cert-manager `Issuer`. + +* Deploy the database in the `alert-mssqlserver` namespace: + + ```bash + $ kubectl create ns alert-mssqlserver + namespace/alert-mssqlserver created + ``` + +* Create a self-signed CA and an `Issuer` for MSSQLServer to use: + + ```bash + $ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./ca.key -out ./ca.crt -subj "/CN=MSSQLServer/O=kubedb" + + $ kubectl create secret tls mssqlserver-ca --cert=ca.crt --key=ca.key --namespace=alert-mssqlserver + secret/mssqlserver-ca created + + $ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/mssqlserver/monitoring/mssqlserver-ca-issuer.yaml + issuer.cert-manager.io/mssqlserver-ca-issuer created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/mssqlserver/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/mssqlserver](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/mssqlserver) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys MSSQLServer with a metrics-exporter sidecar (container `exporter`) that exposes metrics on the `{mssqlserver-name}-stats` service. +- **ServiceMonitor** (named `{mssqlserver-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `mssqlserver-alerts` chart and contains MSSQLServer alert definitions grouped by concern: database health, provisioner, and ops-manager. A fourth group (`kubeStash`) is declared in the chart but currently fails to render — see the chart-bug callout below. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). +- **Grafana** dashboards for MSSQLServer are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. + +--- + +## Deploy MSSQLServer with Monitoring Enabled + +At first, let's deploy an MSSQLServer instance with monitoring enabled. Below is the MSSQLServer object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: MSSQLServer +metadata: + name: mssqlserver-alert-demo + namespace: alert-mssqlserver +spec: + version: "2025-cu0" + replicas: 1 + storageType: Durable + tls: + issuerRef: + name: mssqlserver-ca-issuer + kind: Issuer + apiGroup: "cert-manager.io" + clientTLS: false + podTemplate: + spec: + containers: + - name: mssql + env: + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_PID + value: Evaluation # Change to a licensed edition for production use + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.tls.issuerRef` points at the `mssqlserver-ca-issuer` Issuer created above, so the SQL Server endpoint is served over TLS. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. +- `ACCEPT_EULA=Y` and `MSSQL_PID` are required by the upstream SQL Server image itself, not KubeDB — see [Microsoft's environment variable reference](https://learn.microsoft.com/en-us/sql/linux/sql-server-linux-configure-environment-variables) for valid `MSSQL_PID` values. + +Let's create the MSSQLServer resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/mssqlserver/monitoring/mssqlserver-alert-demo.yaml +mssqlserver.kubedb.com/mssqlserver-alert-demo created +``` + +Wait for the database to go into `Ready` state. + +```bash +$ kubectl get mssqlserver -n alert-mssqlserver mssqlserver-alert-demo +NAME VERSION STATUS AGE +mssqlserver-alert-demo 2025-cu0 Ready 5m +``` + +> First boot takes several minutes — SQL Server doesn't write to its error log or create system databases (`master.mdf`, `msdbdata.mdf`, ...) until it's actually ready to initialize, so `kubectl get pods` showing `2/2 Running` doesn't by itself mean the instance is `Ready` yet. Give it 3-5 minutes on a cold node before assuming something's stuck. + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-mssqlserver --selector="app.kubernetes.io/instance=mssqlserver-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +mssqlserver-alert-demo ClusterIP 10.43.10.20 1433/TCP 5m +mssqlserver-alert-demo-pods ClusterIP None 1433/TCP 5m +mssqlserver-alert-demo-stats ClusterIP 10.43.10.21 56790/TCP 5m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-mssqlserver +NAME AGE +mssqlserver-alert-demo-stats 5m + +$ kubectl get servicemonitor -n alert-mssqlserver mssqlserver-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install mssqlserver-alerts + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the MSSQLServer object's name (`mssqlserver-alert-demo`). + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector` — installing without `--set form.alert.labels.release=prometheus` produces a `PrometheusRule` that Prometheus silently never loads (confirmed: the rules exist in the cluster but never appear under `/rules` or `/alerts` until the label is corrected with a `helm upgrade`). + +### Chart bug: the `kubeStash` group fails to render + +> **Chart bug found (v2026.7.14):** `helm template`/`helm install` for `mssqlserver-alerts` fails outright with `error converting YAML to JSON: yaml: line 149: mapping values are not allowed in this context` if the `kubeStash` alert group is left at its default `enabled: warning`. The group's rule template has a real indentation bug — the `labels:`/`annotations:` block for each `kubeStash` rule is emitted one level shallower than required, so keys like `k8s_group`, `k8s_kind`, `app` end up as siblings of `labels:` instead of nested under it, breaking YAML parsing entirely. **Workaround:** disable the group at install time with `--set form.alert.groups.kubeStash.enabled=none` — the rest of the chart (`database`, `provisioner`, `opsManager`) renders correctly once this group is disabled. + +### Install + +```bash +$ helm upgrade -i mssqlserver-alert-demo appscode/mssqlserver-alerts \ + -n alert-mssqlserver \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus \ + --set form.alert.groups.kubeStash.enabled=none +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-mssqlserver +NAME AGE +mssqlserver-alert-demo 30s + +$ kubectl get prometheusrule -n alert-mssqlserver mssqlserver-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `mssqlserver.database`, `mssqlserver.provisioner`, and `mssqlserver.opsManager` groups. + +

+ Prometheus Rule Health +

+ +All three groups should show **OK** (with `kubeStash` absent, per the workaround above). + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-mssqlserver%22%7D&g0.tab=1`. + +

+ Prometheus up query — mssqlserver-alert-demo-0 UP +

+ +Both series report `up == 1` — the exporter's own target and its `target="mssql_database"` sub-target — confirming Prometheus is scraping `mssqlserver-alert-demo-0` successfully. + +### 2. Confirm the MSSQLServer alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — MSSQLServer groups inactive +

+ +All rules should show **INACTIVE**. + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +This view isn't filtered to a namespace, so alerts from other databases on a shared cluster may also appear here — use the filter box (`app_namespace="alert-mssqlserver"`) to confirm none are specific to this instance. + +### 4. Explore the Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the MSSQLServer dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.MSSQLServer=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `MSSQLServerInstanceDown` (instant, `for: 0m`) by crashing the main `sqlservr` process. + +### 1. Crash the MSSQLServer process + +```bash +$ kubectl exec -n alert-mssqlserver mssqlserver-alert-demo-0 -c mssql -- sh -c ' + end=$(( $(date +%s) + 30 )); + while [ $(date +%s) -lt $end ]; do + pid=$(pgrep -x sqlservr | head -1); + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; + sleep 1; + done' +``` + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — MSSQLServerInstanceDown Firing +

+ +`MSSQLServerInstanceDown` (`up == 0`) should transition straight to **FIRING**. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — MSSQLServerInstanceDown Firing +

+ +AlertManager shows the `MSSQLServerInstanceDown` alert. The alert card displays labels including: + +- **alertname**: `MSSQLServerInstanceDown` +- **severity**: `critical` +- **app**: `mssqlserver-alert-demo`, **app_namespace**: `alert-mssqlserver` +- **k8s_kind**: `MSSQLServer` +- **job**: `mssqlserver-alert-demo-stats` + +> Note: this chart's alert labels use `app_namespace` rather than a plain `namespace` label — filter or group on `app_namespace` when searching for these alerts in AlertManager. + +### 4. Restore MSSQLServer + +Stop the loop from step 1. + +```bash +$ kubectl get mssqlserver -n alert-mssqlserver mssqlserver-alert-demo -w +NAME VERSION STATUS AGE +mssqlserver-alert-demo 2025-cu0 Ready 24m +``` + +If MSSQLServer does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-mssqlserver mssqlserver-alert-demo-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `mssqlserver-alert-demo` instance in the `alert-mssqlserver` namespace via the PromQL label filters `job="mssqlserver-alert-demo-stats"` / `namespace="alert-mssqlserver"` (database group), or `app="mssqlserver-alert-demo"` / `namespace="alert-mssqlserver"` (provisioner/opsManager groups). + +### Database Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MSSQLServerInstanceDown` | critical | instant | `up == 0` on this instance. | +| `MSSQLServerServiceDown` | critical | instant | No replica behind the service is answering. | +| `MSSQLServerRestarted` | critical | instant | Uptime indicates a recent restart. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Provisioner Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMSSQLServerPhaseNotReady` | critical | 1m | KubeDB marked the MSSQLServer resource `NotReady`. | +| `KubeDBMSSQLServerPhaseCritical` | warning | 15m | MSSQLServer is degraded but not fully unavailable. | + +### OpsManager Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMSSQLServerOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes. | +| `KubeDBMSSQLServerOpsRequestFailed` | critical | instant | An ops request failed. | + +### KubeStash Group (currently broken — see chart-bug callout above) + +Declared in `values.yaml` but not currently installable at chart v2026.7.14 without disabling it. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MSSQLServerKubeStashBackupSessionFailed` | critical | instant | Most recent backup session failed. | +| `MSSQLServerKubeStashRestoreSessionFailed` | critical | instant | Most recent restore session failed. | +| `MSSQLServerKubeStashNoBackupSessionForTooLong` | warning | instant | No recent successful backup. | +| `MSSQLServerKubeStashRepositoryCorrupted` | critical | 5m | Backup repository integrity check failed. | +| `MSSQLServerKubeStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage usage is high. | + +--- + +## Customising Alerts + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + kubeStash: + enabled: "none" # required until the chart's indentation bug above is fixed upstream + database: + enabled: warning + rules: + mssqlserverRestarted: + enabled: true + severity: warning +``` + +```bash +$ helm upgrade mssqlserver-alert-demo appscode/mssqlserver-alerts \ + -n alert-mssqlserver \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +> Once the upstream chart fixes the `kubeStash` group's indentation, re-check whether `enabled: "none"` is still necessary before upgrading further. + +--- + +## Cleaning up + +```bash +$ helm uninstall mssqlserver-alert-demo -n alert-mssqlserver +$ kubectl delete mssqlserver -n alert-mssqlserver mssqlserver-alert-demo +$ kubectl delete issuer -n alert-mssqlserver mssqlserver-ca-issuer +$ kubectl delete secret -n alert-mssqlserver mssqlserver-ca +$ kubectl delete ns alert-mssqlserver +``` + +## Next Steps + +- Monitor your MSSQLServer instance with KubeDB using [Prometheus operator](/docs/guides/mssqlserver/monitoring/using-prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/mysql/monitoring/alerting.md b/docs/guides/mysql/monitoring/alerting.md new file mode 100644 index 0000000000..216fe525c9 --- /dev/null +++ b/docs/guides/mysql/monitoring/alerting.md @@ -0,0 +1,645 @@ +--- +title: MySQL Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: guides-mysql-monitoring-alerting + name: Alerting + parent: guides-mysql-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# MySQL Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed MySQL instance using the `mysql-alerts` Helm chart, and how to visualise live metrics using the `kubedb-grafana-dashboards` chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-mysql` namespace: + + ```bash + $ kubectl create ns alert-mysql + namespace/alert-mysql created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. See the [Grafana Dashboard](grafana-dashboard.md#configuration) guide for how to deploy kube-prometheus-stack if you don't have it yet. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/mysql/monitoring/overview/index.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/mysql](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/mysql) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +The diagram below shows the full alerting architecture — from MySQL metric export through to alert delivery and Grafana visualisation. + +

+ MySQL Alerting Architecture +

+ +- **KubeDB** deploys MySQL with a built-in `mysqld_exporter` sidecar (container name `exporter`) that exposes metrics on port `56790`. +- **ServiceMonitor** (named `{mysql-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `mysql-alerts` chart and contains all MySQL alert definitions grouped by concern: database health, group replication, provisioner, ops-manager, and backups (Stash and KubeStash). +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). +- **Grafana** visualises metrics through pre-built dashboards provisioned by the `kubedb-grafana-dashboards` chart. + +--- + +## Deploy MySQL with Monitoring Enabled + +At first, let's deploy a MySQL database with monitoring enabled. This tutorial uses a 3-node Group Replication cluster (Single-Primary mode) rather than a standalone instance, since that's representative of a real deployment and is what the rest of this guide's screenshots are taken from — the `group` alert group only produces real data on a Group Replication topology; a standalone instance simply leaves it permanently INACTIVE with no series at all. `spec.storage.storageClassName` is set to `longhorn` so each node's data volume is backed by Longhorn-replicated block storage rather than node-local storage. + +```yaml +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: mysql-alert + namespace: alert-mysql +spec: + version: "9.6.0" + deletionPolicy: WipeOut + replicas: 3 + topology: + mode: GroupReplication + group: + mode: Single-Primary + storageType: Durable + storage: + storageClassName: "longhorn" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.storage.storageClassName: "longhorn"` provisions the data volume from the `longhorn` `StorageClass` instead of the default `local-path`, giving the volume replicated, node-independent storage. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the namespace and the MySQL resource. + +```bash +$ kubectl create ns alert-mysql +namespace/alert-mysql created + +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/mysql/monitoring/mysql-alert.yaml +mysql.kubedb.com/mysql-alert created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get mysql -n alert-mysql mysql-alert +NAME VERSION STATUS AGE +mysql-alert 9.6.0 Ready 28m +``` + +KubeDB brings up 3 Group Replication pods: + +```bash +$ kubectl get pods -n alert-mysql +NAME READY STATUS RESTARTS AGE +mysql-alert-0 3/3 Running 0 28m +mysql-alert-1 3/3 Running 0 24m +mysql-alert-2 3/3 Running 0 24m +``` + +Confirm each data volume is actually `Bound` on the `longhorn` `StorageClass`. + +```bash +$ kubectl get pvc -n alert-mysql +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +data-mysql-alert-0 Bound pvc-97cac013-9493-4943-9ec0-6f571878f0f2 1Gi RWO longhorn 28m +data-mysql-alert-1 Bound pvc-93dfb146-0e3f-4860-b347-404b88a3ce95 1Gi RWO longhorn 24m +data-mysql-alert-2 Bound pvc-177aee56-529b-4d07-92c8-48bfab3abf10 1Gi RWO longhorn 24m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-mysql --selector="app.kubernetes.io/instance=mysql-alert" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +mysql-alert ClusterIP 10.43.62.19 3306/TCP 28m +mysql-alert-pods ClusterIP None 3306/TCP 28m +mysql-alert-standby ClusterIP 10.43.44.245 3306/TCP 28m +mysql-alert-stats ClusterIP 10.43.241.184 56790/TCP 28m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-mysql +NAME AGE +mysql-alert-stats 28m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-mysql mysql-alert-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install mysql-alerts + +The `mysql-alerts` chart creates a `PrometheusRule` resource containing all MySQL alert definitions grouped by concern: database health, group replication, provisioner, ops-manager, and backups (Stash / KubeStash). + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the MySQL object's name (`mysql-alert`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i mysql-alert oci://ghcr.io/appscode-charts/mysql-alerts \ + -n alert-mysql \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `mysql-alert` (release name) | — | Scopes every PromQL expression to this instance (`job="mysql-alert-stats"`) | +| `-n alert-mysql` | `alert-mysql` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-mysql +NAME AGE +mysql-alert 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-mysql mysql-alert \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and search for **mysql-alert**. + +

+ Prometheus Rule Health +

+ +The `mysql.database.alert-mysql.mysql-alert.rules` group (and the accompanying `mysql.group`, `mysql.provisioner`, `mysql.opsManager`, `mysql.stash`, `mysql.kubeStash`, and `mysql.schemaManager` groups) are visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the MySQL alert definitions every 30 seconds. + +> **Chart note:** unlike some other `*-alerts` charts, every alert group declared in `mysql-alerts`' `values.yaml` (`database`, `group`, `provisioner`, `opsManager`, `stash`, `kubeStash`, `schemaManager`) is actually rendered into the `PrometheusRule` at v2026.7.14 — there is no group silently missing from the template. + +## Step 2 — Install kubedb-grafana-dashboards + +The `kubedb-grafana-dashboards` chart creates `GrafanaDashboard` CRDs containing pre-built MySQL dashboard JSON. A separate controller, `grafana-operator`, watches these CRDs and pushes the dashboards into Grafana over its HTTP API — both pieces are required. If you've already set these up for another database on this cluster (see the [Elasticsearch alerting guide](/docs/guides/elasticsearch/monitoring/alerting.md) for the full walkthrough), skip straight to [Install the dashboards](#install-the-dashboards) below. + +### Install grafana-operator + +If your cluster doesn't already have it (check with `kubectl get crd grafanadashboards.openviz.dev`): + +```bash +$ helm upgrade -i grafana-operator appscode/grafana-operator \ + -n kubeops --create-namespace \ + --version=v2026.6.12 \ + --wait +``` + +### Mark your Grafana instance as the cluster default + +Skip this if you already have a Grafana `AppBinding` annotated as the cluster default (one is shared across every database). Otherwise: + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 & +$ GRAFANA_PW=$(kubectl get secret -n monitoring prometheus-grafana -o jsonpath='{.data.admin-password}' | base64 -d) +$ curl -s -X POST -H "Content-Type: application/json" -u admin:$GRAFANA_PW \ + http://localhost:3000/api/auth/keys \ + -d '{"name":"kubedb-dashboards","role":"Admin"}' +# Note the returned "key" +$ kill %1 +``` + +```yaml +# grafana-appbinding.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-token + namespace: kubeops +type: Opaque +stringData: + token: "" +--- +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + name: grafana + namespace: kubeops + annotations: + monitoring.appscode.com/is-default-grafana: "true" # must be an ANNOTATION, not a label +spec: + type: monitoring.appscode.com/grafana + clientConfig: + url: "http://prometheus-grafana.monitoring.svc:80" + secret: + name: grafana-admin-token +``` + +```bash +$ kubectl apply -f grafana-appbinding.yaml +``` + +### Install the dashboards + +```bash +$ helm repo add appscode https://charts.appscode.com/stable/ +$ helm repo update appscode + +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.MySQL=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl apply -n kubeops -f - +``` + +> **Note:** `featureGates.` defaults to `true` for almost every database in this chart, so one `helm template | kubectl apply` installs dashboards for many databases at once, not just MySQL — this is expected. + +### Verify dashboards are created + +```bash +$ kubectl get grafanadashboards -n kubeops | grep mysql +NAME TITLE STATUS AGE +kubedb-mysql-database KubeDB / MySQL / Database Current 2m +kubedb-mysql-group-replication-summary KubeDB / MySQL / Group-Replication-Summary Current 2m +kubedb-mysql-pod KubeDB / MySQL / Pod Current 2m +kubedb-mysql-summary KubeDB / MySQL / Summary Current 2m +``` + +Four dashboards this time, not three — MySQL's chart ships a dedicated **Group-Replication-Summary** dashboard alongside the usual Summary/Pod/Database triplet. + +--- + +## Verify End-to-End + +### 1. Check the exporter is running + +The `exporter` sidecar inside the MySQL pod serves metrics at `:56790/metrics`. A value of `mysql_up 1` confirms the exporter can reach MySQL. + +```bash +$ kubectl exec -n alert-mysql mysql-alert-0 -c exporter -- \ + wget -qO- localhost:56790/metrics | grep mysql_up +mysql_up 1 +``` + +### 2. Check the Prometheus target is UP + +Prometheus discovers more than 20 scrape pools on a shared cluster, so instead of the Target health page, query `up` directly for a reliable view. + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-mysql%22%7D&g0.tab=1`. + +

+ Prometheus up query — target UP +

+ +The target reports `up == 1` for all 3 pods (`mysql-alert-0/1/2`) in the `alert-mysql` namespace, confirming Prometheus is scraping the exporter on every longhorn-backed pod. + +### 3. Confirm all MySQL alerts are inactive + +Open `http://localhost:9090/alerts?search=mysql-alert` and locate the `mysql-alert` groups. + +

+ Prometheus Alerts — All Inactive +

+ +All 13 rules in the `mysql.database` group show **INACTIVE (13)**, meaning the database is healthy and no thresholds are breached. This also confirms `DiskUsageHigh`/`DiskAlmostFull` are inactive — this chart's disk-usage PromQL correctly divides against `kubelet_volume_stats_capacity_bytes`, so (unlike some other `*-alerts` charts) it does not falsely fire on a healthy volume regardless of storage backend. The `mysql.group` group's 4 rules are also INACTIVE — on this real Group Replication cluster they have live data (unlike a standalone instance, where they'd have none at all), just currently below threshold. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy MySQL instance, no alerts for `mysql-alert` will be listed here. + +

+ AlertManager +

+ +### 5. Explore Grafana dashboards + +Port-forward Grafana and log in. + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 +``` + +Open `http://localhost:3000` (username: `admin`). Search for **mysql** in the Dashboards section. + +

+ Grafana — MySQL Dashboard List +

+ +Four pre-built dashboards are available. The `Namespace` and `mysql` drop-downs at the top of each dashboard let you switch between instances. + +**KubeDB / MySQL / Summary** — database status, version, node count, CPU/memory/storage requests vs. usage. + +

+ Grafana — KubeDB MySQL Summary +

+ +**KubeDB / MySQL / Group-Replication-Summary** — per-node ONLINE status, the current Primary, and replication delay/transport-time/apply-time/transaction-queue metrics per member. + +

+ Grafana — KubeDB MySQL Group-Replication-Summary +

+ +**KubeDB / MySQL / Database** — per-pod service status/uptime, QPS, connections, disk I/O, network, and top command counters. + +

+ Grafana — KubeDB MySQL Database +

+ +**KubeDB / MySQL / Pod** — per-pod CPU/memory/file descriptors, connections, thread activity, temporary objects, slow queries, table locks, and network traffic. + +

+ Grafana — KubeDB MySQL Pod +

+ +--- + +## Simulating a Firing Alert + +The previous section confirmed that all alerts are **INACTIVE** while the database is healthy. This section walks through deliberately triggering the `MySQLInstanceDown` critical alert on one node so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +The `exporter` sidecar runs as a **separate container** from `mysql`, so it keeps running even after `mysqld` crashes, and the pod's `PID 1` is `tini` (not `mysqld` itself) — killing just the `mysqld` process doesn't take the container down, unlike killing PID 1 would. A single kill self-heals in well under 10 seconds though (the wrapper notices and respawns it), so hold it down with a short kill-loop to reliably catch it on a scrape. + +### 1. Crash the MySQL process on one node + +```bash +$ kubectl exec -n alert-mysql mysql-alert-0 -c mysql -- sh -c ' + end=$(( $(date +%s) + 45 )); + while [ $(date +%s) -lt $end ]; do + for p in /proc/[0-9]*; do + pid=$(basename $p) + if [ -r $p/comm ] && [ "$(cat $p/comm 2>/dev/null)" = "mysqld" ]; then + kill -9 $pid 2>/dev/null + fi + done + sleep 1 + done' +``` + +(The image has no `pgrep`/`ps`, so the loop scans `/proc` directly to find the `mysqld` PID each iteration.) Run this in the background — it holds `mysqld` down on `mysql-alert-0` for 45 seconds, comfortably past one scrape (10s) and evaluation (30s) cycle. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=mysql-alert`. + +

+ Prometheus Alerts — MySQLInstanceDown Firing +

+ +`MySQLInstanceDown` transitions from **INACTIVE** to **FIRING** as soon as the exporter on `mysql-alert-0` reports `mysql_up == 0` — it has `for: 0m`, so it fires on the very next evaluation cycle. Note that `MySQLServiceDown` correctly stays **INACTIVE** here: that rule fires only when *no* pod behind the stats service reports `mysql_up == 1`, and `mysql-alert-1`/`mysql-alert-2` are still healthy — a real distinction a single-node deployment wouldn't show you. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter={namespace="alert-mysql"}`. + +

+ AlertManager — MySQLInstanceDown Firing +

+ +AlertManager shows the `MySQLInstanceDown` alert. The alert card displays: + +- **Severity**: `critical` +- **pod**: `mysql-alert-0` +- **job**: `mysql-alert-stats` +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore MySQL + +Let the loop from step 1 finish (or stop it early) — the wrapper script inside the container restarts `mysqld` on its own, no pod restart needed. + +```bash +$ kubectl get pods -n alert-mysql +NAME READY STATUS RESTARTS AGE +mysql-alert-0 3/3 Running 0 29m +``` + +Recovery took about 60 seconds after the kill-loop ended in testing — `mysqld` restarts and rejoins Group Replication before `mysql_up` returns to `1`. Expect a brief `MySQLRestarted` (uptime-based) alert and possibly a transient `MySQLSlowQueries` alert immediately afterward — both clear on their own within a couple of minutes and aren't a sign anything is wrong. Once `mysql_up` is back to `1` and the resource phase returns to `Ready`, Prometheus marks `MySQLInstanceDown` **INACTIVE** and AlertManager sends a **resolved** notification. + +--- + +## Alert Reference + +All alerts are scoped to the `mysql-alert` instance in the `alert-mysql` namespace via the PromQL label filters `job="mysql-alert-stats"` and `namespace="alert-mysql"` (database/group groups), or `app="mysql-alert"` and `namespace="alert-mysql"` (provisioner/opsManager/stash/kubeStash/schemaManager groups). + +### Database Group + +Fired based on live metrics from `mysqld_exporter`. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MySQLInstanceDown` | critical | instant | Exporter cannot reach MySQL — instance is down or `mysqld` crashed. | +| `MySQLServiceDown` | critical | instant | No pod behind the stats service reports `mysql_up == 1` — the service has no healthy backend. | +| `MySQLTooManyConnections` | warning | 2m | More than 80% of `max_connections` are in use — connection pool nearing exhaustion. | +| `MySQLHighThreadsRunning` | warning | 2m | More than 60% of `max_connections` worth of threads are actively running — the server is under heavy load. | +| `MySQLSlowQueries` | warning | 2m | New slow queries have been logged in the last minute. | +| `MySQLInnoDBLogWaits` | warning | instant | InnoDB log writes are stalling (>10 waits in 15m) — I/O may be a bottleneck. | +| `MySQLRestarted` | warning | instant | MySQL uptime is under 60 seconds — the server restarted recently. | +| `MySQLHighQPS` | critical | instant | Queries per second exceed 1000 — unusually high query load. | +| `MySQLHighIncomingBytes` | critical | instant | Incoming network traffic exceeds 1MB/s. | +| `MySQLHighOutgoingBytes` | critical | instant | Outgoing network traffic exceeds 1MB/s. | +| `MySQLTooManyOpenFiles` | warning | 2m | More than 80% of `open_files_limit` are in use. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80% — plan for expansion. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95% — MySQL may become read-only or crash. | + +### Group Replication Group + +Fired based on MySQL Group Replication performance-schema metrics; only meaningful for clustered/group-replication deployments. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MySQLHighReplicationDelay` | warning | 5m | Group Replication apply time on a member exceeds 0.5s. | +| `MySQLHighReplicationTransportTime` | warning | 5m | Group Replication transport time exceeds 0.5s — network lag between members. | +| `MySQLHighReplicationApplyTime` | warning | 5m | Group Replication apply time exceeds 0.5s. | +| `MySQLReplicationHighTransactionTime` | warning | 5m | Transaction time on the local relay queue exceeds 0.5s — the member is falling behind. | + +> **Chart bug (v2026.7.14):** `MySQLHighReplicationDelay` and `MySQLHighReplicationApplyTime` render with the **identical** PromQL expression (`mysql_perf_schema_replication_group_worker_apply_time_seconds`) — confirmed by reading the rendered `PrometheusRule` directly. They will always fire and clear together. `MySQLHighReplicationDelay` was almost certainly intended to alert on a different metric (e.g. an actual replication-lag/delay gauge). This tutorial's 3-node Group Replication cluster gives these rules live data (unlike a standalone instance, where they'd have none at all) — both stayed INACTIVE throughout normal operation and the single-node crash test in this guide, since replication delay/apply-time on the two surviving nodes stayed low. + +### Provisioner Group + +Monitors the KubeDB operator's view of the MySQL resource phase (sourced from Panopticon, not the MySQL metrics endpoint). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMySQLPhaseNotReady` | critical | 1m | KubeDB marked the MySQL resource `NotReady` — operator cannot reach a healthy instance. | +| `KubeDBMySQLPhaseCritical` | warning | 15m | The MySQL resource is in a degraded/critical phase. | + +### OpsManager Group + +Tracks `MySQLOpsRequest` lifecycle during upgrades, scaling, reconfiguration, and certificate rotations. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMySQLOpsRequestStatusProgressingToLong` | critical | 30m | A `MySQLOpsRequest` has been in progress for 30+ minutes — likely stuck. | +| `KubeDBMySQLOpsRequestFailed` | critical | instant | A `MySQLOpsRequest` failed — check the `MySQLOpsRequest` object for the error. | + +### Stash Group + +Tracks Stash-driven backup/restore health for this instance. (You do not need to configure backups to see these rules; they are included in the `PrometheusRule` regardless.) + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MySQLStashBackupSessionFailed` | critical | instant | A Stash backup session failed. | +| `MySQLStashRestoreSessionFailed` | critical | instant | A Stash restore session failed. | +| `MySQLStashNoBackupSessionForTooLong` | warning | instant | No successful backup session for more than 18000s (5 hours). | +| `MySQLStashRepositoryCorrupted` | critical | 5m | The Stash backup repository integrity check failed — repository is corrupted. | +| `MySQLStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage size has exceeded 10GB. | +| `MySQLStashBackupSessionPeriodTooLong` | warning | instant | A backup session took more than 1800s (30 minutes) to complete. | +| `MySQLStashRestoreSessionPeriodTooLong` | warning | instant | A restore session took more than 1800s (30 minutes) to complete. | + +### KubeStash Group + +Tracks KubeStash-driven backup/restore health for this instance. Same semantics as the Stash group above, sourced from KubeStash metrics instead. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `MySQLKubeStashBackupSessionFailed` | critical | instant | A KubeStash backup session failed. | +| `MySQLKubeStashRestoreSessionFailed` | critical | instant | A KubeStash restore session failed. | +| `MySQLKubeStashNoBackupSessionForTooLong` | warning | instant | No successful backup session for more than 18000s (5 hours). | +| `MySQLKubeStashRepositoryCorrupted` | critical | 5m | The KubeStash backup repository integrity check failed — repository is corrupted. | +| `MySQLKubeStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage size has exceeded 10GB. | +| `MySQLKubeStashBackupSessionPeriodTooLong` | warning | instant | A backup session took more than 1800s (30 minutes) to complete. | +| `MySQLKubeStashRestoreSessionPeriodTooLong` | warning | instant | A restore session took more than 1800s (30 minutes) to complete. | + +### SchemaManager Group + +Monitors `MySQLDatabase` schema lifecycle objects managed by KubeDB Schema Manager. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBMySQLSchemaPendingForTooLong` | warning | 30m | Schema object stuck in `Pending` — may be waiting on a dependency. | +| `KubeDBMySQLSchemaInProgressForTooLong` | warning | 30m | Schema migration running for 30+ minutes — may be stuck. | +| `KubeDBMySQLSchemaTerminatingForTooLong` | warning | 30m | Schema deletion stuck — a finalizer may be blocking it. | +| `KubeDBMySQLSchemaFailed` | warning | instant | Schema operation failed. | +| `KubeDBMySQLSchemaExpired` | warning | instant | A schema with a TTL has expired and been revoked. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + mysqlTooManyConnections: + enabled: true + duration: "5m" + val: 90 # fire at 90% instead of the default 80% + severity: warning + opsManager: + enabled: "none" # disable all ops-manager alerts +``` + +```bash +$ helm upgrade mysql-alert oci://ghcr.io/appscode-charts/mysql-alerts \ + -n alert-mysql \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the Grafana dashboards (installed via helm template | kubectl apply, not helm install) +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.7.10 \ + --set featureGates.MySQL=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl delete -n kubeops -f - --ignore-not-found + +# Remove the mysql-alerts release +$ helm uninstall mysql-alert -n alert-mysql + +# Remove the MySQL instance +$ kubectl delete mysql -n alert-mysql mysql-alert + +# Delete namespace +$ kubectl delete ns alert-mysql + +# Optional: only if nothing else in the cluster depends on them +$ kubectl delete appbinding -n kubeops grafana +$ kubectl delete secret -n kubeops grafana-admin-token +$ helm uninstall grafana-operator -n kubeops +``` + +## Next Steps + +- Monitor your MySQL database with KubeDB using [builtin Prometheus](/docs/guides/mysql/monitoring/builtin-prometheus/index.md). +- Monitor your MySQL database with KubeDB using [Prometheus operator](/docs/guides/mysql/monitoring/prometheus-operator/index.md). +- Visualise MySQL metrics with [Grafana Dashboard](grafana-dashboard.md). +- Detail concepts of [MySQL object](/docs/guides/mysql/concepts/database/index.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/neo4j/monitoring/alerting.md b/docs/guides/neo4j/monitoring/alerting.md new file mode 100644 index 0000000000..d45db2e996 --- /dev/null +++ b/docs/guides/neo4j/monitoring/alerting.md @@ -0,0 +1,483 @@ +--- +title: Neo4j Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: neo4j-monitoring-alerting + name: Alerting + parent: neo4j-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Neo4j Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Neo4j cluster using the `neo4j-alerts` Helm chart. Unlike most other `*-alerts` charts, `neo4j-alerts` also bundles a Grafana dashboard that it imports automatically through a post-install Job — no separate dashboard chart is required. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-neo4j` namespace: + + ```bash + $ kubectl create ns alert-neo4j + namespace/alert-neo4j created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/neo4j/monitoring/overview.md). + +* You will also need a Grafana API key / token with **Editor** permission so the chart's dashboard-import Job can push the dashboard. See [Step 2](#create-a-grafana-api-key) below. + +> Note: YAML files used in this tutorial are stored in [docs/examples/neo4j](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/neo4j) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Neo4j with metrics exposed directly by the `neo4j` container itself on port `2004` (Neo4j's built-in Prometheus metrics endpoint) — there is no separate exporter sidecar. +- **ServiceMonitor** (named `{neo4j-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the metrics endpoint every 10 seconds. +- **PrometheusRule** is created by the `neo4j-alerts` chart and contains all Neo4j alert definitions grouped by concern: database health/resource usage and provisioner. +- **Dashboard-import Job** — when `grafana.enabled` is `true` (the default), the chart also creates a one-shot `Job` that `POST`s a bundled dashboard JSON straight to your Grafana instance's `/api/dashboards/import` endpoint. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy Neo4j with Monitoring Enabled + +At first, let's deploy a 3-node Neo4j cluster with monitoring enabled. Below is the Neo4j object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-alert-demo + namespace: alert-neo4j +spec: + replicas: 3 + version: "2025.12.1" + deletionPolicy: WipeOut + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus +``` + +Here, + +- `spec.replicas: 3` creates a 3-member Neo4j cluster. Neo4j Enterprise clustering requires a minimum of 3 core members. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the namespace and the Neo4j resource. + +```bash +$ kubectl create ns alert-neo4j +namespace/alert-neo4j created + +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/neo4j/monitoring/neo4j-alert-demo.yaml +neo4j.kubedb.com/neo4j-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get neo4j -n alert-neo4j neo4j-alert-demo +NAME VERSION STATUS AGE +neo4j-alert-demo 2025.12.1 Ready 3m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-neo4j --selector="app.kubernetes.io/instance=neo4j-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +neo4j-alert-demo ClusterIP 10.43.30.142 6362/TCP,7687/TCP,7474/TCP 3m +neo4j-alert-demo-0 ClusterIP None 6362/TCP,7687/TCP,7474/TCP,7688/TCP,7000/TCP,6000/TCP 3m +neo4j-alert-demo-1 ClusterIP None 6362/TCP,7687/TCP,7474/TCP,7688/TCP,7000/TCP,6000/TCP 3m +neo4j-alert-demo-2 ClusterIP None 6362/TCP,7687/TCP,7474/TCP,7688/TCP,7000/TCP,6000/TCP 3m +neo4j-alert-demo-stats ClusterIP 10.43.63.217 2004/TCP 3m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-neo4j +NAME AGE +neo4j-alert-demo-stats 3m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-neo4j neo4j-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Create a Grafana API Key + +The chart's dashboard-import Job authenticates to Grafana with a bearer token, so create one first. + +* **Grafana 9+**: **Administration → Service accounts → Add service account** → role **Editor** → **Add token**. Copy the token. +* **Grafana 8.x and earlier** (no Service Accounts UI, e.g. the bundled `kube-prometheus-stack` Grafana 7.5.5 used while verifying this tutorial): use the legacy **API Keys** endpoint instead: + + ```bash +# Port-forward Grafana +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 + +# Retrieve the admin password +$ kubectl get secret -n monitoring prometheus-grafana \ + -o jsonpath='{.data.admin-password}' | base64 -d && echo + +# Create a service account with Editor role +$ curl -s -X POST -H "Content-Type: application/json" \ + -u admin: \ + http://localhost:3000/api/serviceaccounts \ + -d '{"name":"neo4j-alerts-demo","role":"Editor"}' +# Note the returned "id" + +# Create a token for the service account (replace with the returned service account ID) +$ curl -s -X POST -H "Content-Type: application/json" \ + -u admin: \ + http://localhost:3000/api/serviceaccounts//tokens \ + -d '{"name":"neo4j-alerts-demo-key","secondsToLive":0}' +# Note the returned "key" + +# Stop the port-forward +$ kill %1 +``` +Either way, you end up with a bearer token to use as `grafana.apikey` below. + +## Step 2 — Install neo4j-alerts + +The `neo4j-alerts` chart creates a `PrometheusRule` resource containing all Neo4j alert definitions, **and** (by default) a `Job` that imports a pre-built Grafana dashboard. + +### Why the Helm release name matters + +The chart derives the PromQL `pod`/`container` scoping (via `.Release.Name`/`.Release.Namespace`) and the `PrometheusRule` name from the **Helm release name**, not from a values field — so the release name must match the Neo4j object's name (`neo4j-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm repo add appscode https://charts.appscode.com/stable/ +$ helm repo update +$ helm search repo appscode/neo4j-alerts --version=v2026.7.14 +$ helm upgrade -i neo4j-alerts appscode/neo4j-alerts -n alert-neo4j \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus \ + --set grafana.enabled=false +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `neo4j-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`pod=~"neo4j-alert-demo-.+$"`) | +| `-n alert-neo4j` | `alert-neo4j` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | +| `grafana.url` | in-cluster Grafana URL | The dashboard-import Job runs **inside the cluster**, so this must be a cluster-internal address, not `localhost` | +| `grafana.apikey` | token from Step 1 | Authenticates the dashboard-import `POST` request | + +> To install **alerts only, without the dashboard**, set `--set grafana.enabled=false`. + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-neo4j +NAME AGE +neo4j-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-neo4j neo4j-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Verify the dashboard-import Job + +```bash +$ kubectl get job -n alert-neo4j +NAME STATUS COMPLETIONS AGE +neo4j-alert-demo-post-job Complete 1/1 17s + +$ kubectl logs -n alert-neo4j job/neo4j-alert-demo-post-job +{"pluginId":"","title":"kubedb.com / Neo4j / alert-neo4j / neo4j-alert-demo","imported":true, ...} +``` + +A `"imported":true` response confirms the dashboard `kubedb.com / Neo4j / alert-neo4j / neo4j-alert-demo` now exists in Grafana. + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `neo4j.database` and `neo4j.provisioner` groups. + +

+ Prometheus Rule Health +

+ +Both groups are visible with all 10 rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Neo4j alert definitions every 30 seconds. + +--- + +## Verify End-to-End + +### 1. Check the metrics endpoint + +The `neo4j` container serves its own Prometheus metrics at `:2004/metrics` — no exporter sidecar is involved. + +```bash +$ kubectl exec -n alert-neo4j neo4j-alert-demo-0 -c neo4j -- \ + wget -qO- localhost:2004/metrics | grep neo4j_dbms_page_cache_hit_ratio +# HELP neo4j_dbms_page_cache_hit_ratio Generated from Dropwizard metric import (metric=neo4j.dbms.page_cache.hit_ratio, type=com.neo4j.metrics.source.db.PageCacheHitRatioGauge) +# TYPE neo4j_dbms_page_cache_hit_ratio gauge +neo4j_dbms_page_cache_hit_ratio 1.0 + +``` + +### 2. Check the Prometheus target is UP + +Prometheus discovers more than 20 scrape pools on a shared cluster, so instead of the Target health page, query `up` directly for a reliable view. + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-neo4j%22%7D&g0.tab=1`. + +

+ Prometheus up query — all 3 pods UP +

+ +All three `neo4j-alert-demo-{0,1,2}` pods report `up == 1`, confirming Prometheus is scraping every pod in the cluster. + +### 3. Confirm the Neo4j alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — Neo4j groups inactive +

+ +All 10 rules across the `neo4j.database` and `neo4j.provisioner` groups show **INACTIVE**, confirming the cluster is healthy and no alert thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +No alerts are firing for the `alert-neo4j` namespace. + +### 5. Explore the Grafana dashboard + +Port-forward Grafana and log in. + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 +``` + +Open `http://localhost:3000` and navigate to the dashboard `kubedb.com / Neo4j / alert-neo4j / neo4j-alert-demo` that the Job imported in Step 2. + +

+ Grafana — Neo4j Alerts Dashboard +

+ +The dashboard mirrors the alert groups: **Neo4j Phase & Availability** (Down / Critical Phase), **Neo4j Resource Usage** (CPU, memory, disk), and **Neo4j Page Cache Metrics** (usage ratio, hit ratio, page faults). Note that **Neo4j High CPU Usage** shows "No data" on clusters without cAdvisor's `container_cpu_usage_seconds_total` metric exposed (common on some k3s setups) — this is an environment limitation, not a chart bug. + +--- + +## Simulating a Firing Alert + +The previous section showed that all genuine health alerts (everything except the buggy disk rules) are **INACTIVE** while the cluster is healthy. This section deliberately triggers the `KubeDBNeo4jPhaseNotReady` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +Neo4j runs as a **single container per pod** — there is no separate exporter sidecar to keep reporting metrics once the database process dies, and Kubernetes will restart a killed process within seconds. Because `KubeDBNeo4jPhaseNotReady` requires the condition to persist for `for: 1m`, a single `kill` is not enough: you need to keep the pods crashing long enough for the KubeDB operator to mark the resource `NotReady` and hold it there past the one-minute window. + +### 1. Crash the Neo4j process repeatedly + +```bash +$ while true; do + for i in 0 1 2; do + kubectl exec -n alert-neo4j neo4j-alert-demo-$i -c neo4j -- kill 1 >/dev/null 2>&1 + done + sleep 3 + done +``` + +Let this loop run for a couple of minutes (leave it running while you check the next steps), then stop it once you've captured the firing state. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — KubeDBNeo4jPhaseNotReady Firing +

+ +`KubeDBNeo4jPhaseNotReady` transitions from **INACTIVE** to **FIRING** once `kubedb_com_neo4j_status_phase{phase="NotReady"}` has read `1` continuously for the full `for: 1m` duration — this metric comes from the KubeDB operator's own view of the resource (exported via Panopticon), not from the Neo4j metrics endpoint itself. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — KubeDBNeo4jPhaseNotReady Firing +

+ +AlertManager shows the `KubeDBNeo4jPhaseNotReady` alert. The alert card displays: + +- **Severity**: `critical` +- **neo4j**: `neo4j-alert-demo` in the `alert-neo4j` namespace +- **phase**: `NotReady` +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore Neo4j + +Stop the loop from step 1. The pods recover on their own — KubeDB just needs a few uninterrupted scrape/reconcile cycles to mark the resource `Ready` again. + +```bash +$ kubectl get neo4j -n alert-neo4j neo4j-alert-demo -w +NAME VERSION STATUS AGE +neo4j-alert-demo 2025.12.1 Ready 24m +``` + +Once the phase returns to `Ready`, Prometheus marks the alert **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `neo4j-alert-demo` instance in the `alert-neo4j` namespace via the PromQL label filters `pod=~"neo4j-alert-demo-.+$"` and `namespace=~"alert-neo4j"` (database group), or `app="neo4j-alert-demo"` and `namespace="alert-neo4j"` (provisioner group). + +### Database Group + +Fired based on live metrics from the Neo4j container's built-in metrics endpoint and node/kubelet metrics. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `Neo4jHighCPUUsage` | warning | 1m | Average pod CPU usage exceeds 80%. Requires cAdvisor `container_cpu_usage_seconds_total` — shows no data if unavailable. | +| `Neo4jHighMemoryUsage` | warning | 1m | Average pod memory usage exceeds 80% of the configured limit. | +| `Neo4jPageCacheUsageRatioHigh` | warning | 5m | More than 85% of the allocated page cache is in use — consider allocating more page cache. | +| `Neo4jPageCacheHitRatioLow` | warning | 5m | Page cache hit ratio has dropped below 98% — the database is going to disk too often. | +| `Neo4jPageFaultsHigh` | warning | 5m | More than 5000 page faults in the last 5 minutes — may indicate more page cache is required. | +| `Neo4jPageFaultFailuresHigh` | critical | 5m | Any failed page faults in the last 5 minutes — indicates potential disk I/O issues or corruption. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Neo4j resource phase (sourced from Panopticon, not the Neo4j metrics endpoint). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBNeo4jPhaseNotReady` | critical | 1m | KubeDB marked the Neo4j resource `NotReady` — operator cannot reach a healthy cluster majority. | +| `KubeDBNeo4jPhaseCritical` | warning | 15m | One or more Neo4j core members are down; the cluster is degraded but not fully unavailable. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + neo4jHighMemoryUsage: + enabled: true + duration: "5m" + val: 90 # fire at 90% instead of the default 80% + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade neo4j-alert-demo appscode/neo4j-alerts \ + -n alert-neo4j \ + --version=v2026.7.14 \ + --set grafana.enabled=false \ + -f custom-alerts.yaml +``` + +> Note: `-f` values files don't merge `grafana.url`/`grafana.apikey` automatically — re-pass them (or set `grafana.enabled=false`) on every `helm upgrade`, otherwise the dashboard-import Job re-runs with an empty URL/token and fails. + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the neo4j-alerts release (PrometheusRule + dashboard-import Job) +$ helm uninstall neo4j-alert-demo -n alert-neo4j + +# Remove the imported Grafana dashboard (it is not removed by helm uninstall) +$ curl -s -X DELETE -H "Authorization: Bearer " \ + http://localhost:3000/api/dashboards/uid/lhSzgLYDk + +# Remove the Neo4j instance +$ kubectl delete neo4j -n alert-neo4j neo4j-alert-demo + +# Delete namespace +$ kubectl delete ns alert-neo4j +``` + +## Next Steps + +- Monitor your Neo4j database with KubeDB using [built-in Prometheus](/docs/guides/neo4j/monitoring/using-builtin-prometheus.md). +- Monitor your Neo4j database with KubeDB using [Prometheus operator](/docs/guides/neo4j/monitoring/using-prometheus-operator.md). +- Use [private Docker registry](/docs/guides/neo4j/private-registry/using-private-registry.md) to deploy Neo4j with KubeDB. +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/percona-xtradb/monitoring/alerting.md b/docs/guides/percona-xtradb/monitoring/alerting.md new file mode 100644 index 0000000000..70062a0927 --- /dev/null +++ b/docs/guides/percona-xtradb/monitoring/alerting.md @@ -0,0 +1,382 @@ +--- +title: PerconaXtraDB Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: guides-perconaxtradb-monitoring-alerting + name: Alerting + parent: guides-perconaxtradb-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# PerconaXtraDB Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed PerconaXtraDB instance using the `perconaxtradb-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-perconaxtradb` namespace: + + ```bash + $ kubectl create ns alert-perconaxtradb + namespace/alert-perconaxtradb created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/percona-xtradb/monitoring/overview/index.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/percona-xtradb](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/percona-xtradb) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys PerconaXtraDB with a `mysqld_exporter`-compatible sidecar (container `exporter`) that exposes metrics (`mysql_*`), the same exporter family used by MySQL/MariaDB. +- **ServiceMonitor** (named `{perconaxtradb-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `perconaxtradb-alerts` chart and contains alert definitions grouped by concern: database health, Galera cluster, provisioner, ops-manager, Stash backup/restore, and schema manager. +- **Grafana** dashboards for PerconaXtraDB are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy PerconaXtraDB with Monitoring Enabled + +Below is the PerconaXtraDB object we are going to create — a single standalone instance with monitoring enabled. (The chart's `cluster` group only produces data if you deploy a Galera cluster via `spec.topology`; a standalone instance simply leaves that alert INACTIVE.) + +```yaml +apiVersion: kubedb.com/v1 +kind: PerconaXtraDB +metadata: + name: perconaxtradb-alert-demo + namespace: alert-perconaxtradb +spec: + version: "8.0.40" + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the PerconaXtraDB resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/percona-xtradb/monitoring/perconaxtradb-alert-demo.yaml +perconaxtradb.kubedb.com/perconaxtradb-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get perconaxtradb -n alert-perconaxtradb perconaxtradb-alert-demo +NAME VERSION STATUS AGE +perconaxtradb-alert-demo 8.0.40 Ready 3m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-perconaxtradb --selector="app.kubernetes.io/instance=perconaxtradb-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +perconaxtradb-alert-demo ClusterIP 10.43.10.20 3306/TCP 3m +perconaxtradb-alert-demo-pods ClusterIP None 3306/TCP 3m +perconaxtradb-alert-demo-stats ClusterIP 10.43.10.21 56790/TCP 3m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-perconaxtradb +NAME AGE +perconaxtradb-alert-demo-stats 3m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-perconaxtradb perconaxtradb-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install perconaxtradb-alerts + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the PerconaXtraDB object's name (`perconaxtradb-alert-demo`). Note the chart/repo name has no hyphen (`perconaxtradb-alerts`), even though the KubeDB guides directory uses `percona-xtradb`. + +### Install + +```bash +$ helm upgrade -i perconaxtradb-alert-demo oci://ghcr.io/appscode-charts/perconaxtradb-alerts \ + -n alert-perconaxtradb \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-perconaxtradb +NAME AGE +perconaxtradb-alert-demo 30s + +$ kubectl get prometheusrule -n alert-perconaxtradb perconaxtradb-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `perconaxtradb.database`, `perconaxtradb.cluster`, `perconaxtradb.provisioner`, `perconaxtradb.opsManager`, `perconaxtradb.stash`, and `perconaxtradb.schemaManager` groups. + +

+ Prometheus Rule Health +

+ +All groups should show **OK**. Unlike MariaDB's chart, `perconaxtradb-alerts` v2026.7.14 has a `stash` group but **no** `kubeStash` group — every group it does declare in `values.yaml` renders correctly. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-perconaxtradb%22%7D&g0.tab=1`. + +

+ Prometheus up query — perconaxtradb-alert-demo-0 UP +

+ +### 2. Confirm all PerconaXtraDB alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — PerconaXtraDB groups inactive +

+ +All rules should show **INACTIVE**. `GaleraReplicationLatencyTooLong` has no data on a standalone instance. + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +### 4. Explore the Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the PerconaXtraDB dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.PerconaXtraDB=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `PerconaXtraDBInstanceDown` (instant, `for: 0m`) by crashing the main database process. + +### 1. Crash the PerconaXtraDB process + +```bash +$ kubectl exec -n alert-perconaxtradb perconaxtradb-alert-demo-0 -c perconaxtradb -- sh -c ' + end=$(( $(date +%s) + 30 )); + while [ $(date +%s) -lt $end ]; do + pid=$(pgrep -x mysqld | head -1); + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; + sleep 1; + done' +``` + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — PerconaXtraDBInstanceDown Firing +

+ +`PerconaXtraDBInstanceDown` (`mysql_up == 0`) should transition straight to **FIRING**. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — PerconaXtraDBInstanceDown Firing +

+ +### 4. Restore PerconaXtraDB + +Stop the loop from step 1. + +```bash +$ kubectl get perconaxtradb -n alert-perconaxtradb perconaxtradb-alert-demo -w +NAME VERSION STATUS AGE +perconaxtradb-alert-demo 8.0.40 Ready 24m +``` + +If PerconaXtraDB does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-perconaxtradb perconaxtradb-alert-demo-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `perconaxtradb-alert-demo` instance in the `alert-perconaxtradb` namespace via the PromQL label filters `job="perconaxtradb-alert-demo-stats"` / `namespace="alert-perconaxtradb"` (database/cluster groups), or `app="perconaxtradb-alert-demo"` / `namespace="alert-perconaxtradb"` (provisioner/opsManager/stash/schemaManager groups). + +### Database Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `PerconaXtraDBInstanceDown` | critical | instant | `mysql_up == 0` on this instance. | +| `PerconaXtraDBServiceDown` | critical | instant | No replica behind the service is answering. | +| `PerconaXtraDBTooManyConnections` | warning | 2m | Connection count is high relative to `max_connections`. | +| `PerconaXtraDBHighThreadsRunning` | warning | 2m | Too many threads actively running. | +| `PerconaXtraDBSlowQueries` | warning | 2m | Slow-query count is increasing. | +| `PerconaXtraDBInnoDBLogWaits` | warning | instant | InnoDB log waits are occurring. | +| `PerconaXtraDBRestarted` | warning | instant | Uptime indicates a recent restart. | +| `PerconaXtraDBHighQPS` | critical | instant | Query rate is unusually high. | +| `PerconaXtraDBHighIncomingBytes` | critical | instant | Inbound network traffic is unusually high. | +| `PerconaXtraDBHighOutgoingBytes` | critical | instant | Outbound network traffic is unusually high. | +| `PerconaXtraDBTooManyOpenFiles` | warning | 2m | Open file count is high relative to the limit. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Cluster Group + +Only produces data when `spec.topology` (Galera) is configured. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `GaleraReplicationLatencyTooLong` | warning | 5m | Galera replication latency is high. | + +### Provisioner Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBPerconaXtraDBPhaseNotReady` | critical | 1m | KubeDB marked the PerconaXtraDB resource `NotReady`. | +| `KubeDBPerconaXtraDBPhaseCritical` | warning | 15m | PerconaXtraDB is degraded but not fully unavailable. | + +### OpsManager Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBPerconaXtraDBOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes. | +| `KubeDBPerconaXtraDBOpsRequestFailed` | critical | instant | An ops request failed. | + +### Stash Group + +Only meaningful once Stash backup/restore is configured. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `PerconaXtraDBStashBackupSessionFailed` | critical | instant | Most recent backup session failed. | +| `PerconaXtraDBStashRestoreSessionFailed` | critical | instant | Most recent restore session failed. | +| `PerconaXtraDBStashNoBackupSessionForTooLong` | warning | instant | No recent successful backup. | +| `PerconaXtraDBStashRepositoryCorrupted` | critical | 5m | Backup repository integrity check failed. | +| `PerconaXtraDBStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage usage is high. | +| `PerconaXtraDBStashBackupSessionPeriodTooLong` | warning | instant | A backup session is taking unusually long. | +| `PerconaXtraDBStashRestoreSessionPeriodTooLong` | warning | instant | A restore session is taking unusually long. | + +### SchemaManager Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBPerconaXtraDBSchemaPendingForTooLong` | warning | 30m | A `PerconaXtraDBDatabase` object stuck `Pending`. | +| `KubeDBPerconaXtraDBSchemaInProgressForTooLong` | warning | 30m | A `PerconaXtraDBDatabase` object stuck `InProgress`. | +| `KubeDBPerconaXtraDBSchemaTerminatingForTooLong` | warning | 30m | A `PerconaXtraDBDatabase` object stuck `Terminating`. | +| `KubeDBPerconaXtraDBSchemaFailed` | warning | instant | A `PerconaXtraDBDatabase` object failed. | +| `KubeDBPerconaXtraDBSchemaExpired` | warning | instant | A `PerconaXtraDBDatabase` object expired. | + +--- + +## Customising Alerts + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + perconaxtradbTooManyConnections: + enabled: true + duration: "5m" + severity: warning + cluster: + enabled: "none" # disable if you don't run Galera +``` + +```bash +$ helm upgrade perconaxtradb-alert-demo oci://ghcr.io/appscode-charts/perconaxtradb-alerts \ + -n alert-perconaxtradb \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +```bash +$ helm uninstall perconaxtradb-alert-demo -n alert-perconaxtradb +$ kubectl delete perconaxtradb -n alert-perconaxtradb perconaxtradb-alert-demo +$ kubectl delete ns alert-perconaxtradb +``` + +## Next Steps + +- Monitor your PerconaXtraDB instance with KubeDB using [built-in Prometheus](/docs/guides/percona-xtradb/monitoring/builtin-prometheus/index.md). +- Monitor your PerconaXtraDB instance with KubeDB using [Prometheus operator](/docs/guides/percona-xtradb/monitoring/prometheus-operator/index.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/pgbouncer/monitoring/alerting.md b/docs/guides/pgbouncer/monitoring/alerting.md new file mode 100644 index 0000000000..be32fa4268 --- /dev/null +++ b/docs/guides/pgbouncer/monitoring/alerting.md @@ -0,0 +1,473 @@ +--- +title: PgBouncer Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: pb-monitoring-alerting + name: Alerting + parent: pb-monitoring-pgbouncer + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# PgBouncer Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed PgBouncer instance using the `pgbouncer-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-pgbouncer` namespace: + + ```bash + $ kubectl create ns alert-pgbouncer + namespace/alert-pgbouncer created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/pgbouncer/monitoring/overview.md). + +* PgBouncer is a connection pooler in front of a PostgreSQL backend, so this tutorial first deploys a single-node PostgreSQL instance, then a PgBouncer instance pointed at it. Both objects are deployed with monitoring enabled. + +> Note: YAML files used in this tutorial are stored in [docs/examples/pgbouncer](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/pgbouncer) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys PgBouncer with a dedicated `pgbouncer_exporter` sidecar (container name `exporter`) that exposes metrics on port `56790` — PgBouncer itself has no built-in Prometheus endpoint, so every pod runs **two containers**: `pgbouncer` and `exporter`. +- **ServiceMonitor** (named `{pgbouncer-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `pgbouncer-alerts` chart and contains PgBouncer alert definitions grouped by concern: database health and provisioner. +- **Grafana** dashboards for PgBouncer are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy the PostgreSQL Backend + +PgBouncer pools connections to a PostgreSQL instance, so deploy the backend first. + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-backend-alert + namespace: alert-pgbouncer +spec: + version: "16.13" + replicas: 1 + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/pgbouncer/monitoring/pg-backend-alert.yaml +postgres.kubedb.com/pg-backend-alert created +``` + +Wait for the PostgreSQL instance to go into `Ready` state. + +```bash +$ kubectl get postgres -n alert-pgbouncer pg-backend-alert +NAME VERSION STATUS AGE +pg-backend-alert 16.13 Ready 3m +``` + +## Deploy PgBouncer with Monitoring Enabled + +Now deploy PgBouncer, pointing `spec.database.databaseRef` at the PostgreSQL instance above. + +```yaml +apiVersion: kubedb.com/v1 +kind: PgBouncer +metadata: + name: pgbouncer-alert + namespace: alert-pgbouncer +spec: + version: "1.23.1" + replicas: 1 + database: + syncUsers: true + databaseName: "postgres" + databaseRef: + name: "pg-backend-alert" + namespace: alert-pgbouncer + connectionPool: + maxClientConnections: 20 + reservePoolSize: 5 + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.database.databaseRef` tells PgBouncer which KubeDB Postgres object to pool connections for, and `spec.database.syncUsers: true` keeps PgBouncer's user list in sync with the backend. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/pgbouncer/monitoring/pgbouncer-alert.yaml +pgbouncer.kubedb.com/pgbouncer-alert created +``` + +Now, wait for PgBouncer to go into `Ready` state. + +```bash +$ kubectl get pgbouncer -n alert-pgbouncer pgbouncer-alert +NAME VERSION STATUS AGE +pgbouncer-alert 1.23.1 Ready 40s +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-pgbouncer --selector="app.kubernetes.io/instance=pgbouncer-alert" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +pgbouncer-alert ClusterIP 10.43.40.193 5432/TCP 40s +pgbouncer-alert-pods ClusterIP None 5432/TCP 40s +pgbouncer-alert-stats ClusterIP 10.43.148.12 56790/TCP 40s +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-pgbouncer +NAME AGE +pgbouncer-alert-stats 40s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-pgbouncer pgbouncer-alert-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install pgbouncer-alerts + +The `pgbouncer-alerts` chart creates a `PrometheusRule` resource containing PgBouncer alert definitions grouped by concern. + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the PgBouncer object's name (`pgbouncer-alert`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i pgbouncer-alert oci://ghcr.io/appscode-charts/pgbouncer-alerts \ + -n alert-pgbouncer \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `pgbouncer-alert` (release name) | — | Scopes every PromQL expression to this instance (`job="pgbouncer-alert-stats"`, `app="pgbouncer-alert"`) | +| `-n alert-pgbouncer` | `alert-pgbouncer` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-pgbouncer +NAME AGE +pgbouncer-alert 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-pgbouncer pgbouncer-alert \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +> **Chart gap found:** the chart's `values.yaml` also declares an `opsManager` alert group (`opsRequestFailed`, `opsRequestOnProgress`, `opsRequestStatusProgressingToLong`), but only the `database` and `provisioner` groups are actually rendered into the live `PrometheusRule` at chart version `v2026.7.14` — the same missing-group gap seen in several other `*-alerts` charts. Cross-check `helm show values` against `kubectl get prometheusrule ... -o yaml` rather than assuming every documented group is loaded. + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `pgbouncer.database` and `pgbouncer.provisioner` groups. + +

+ Prometheus Rule Health +

+ +Both groups are visible with all 6 rules showing **OK**, confirming that Prometheus has loaded and is evaluating the PgBouncer alert definitions every 30 seconds. + +--- + +## Verify End-to-End + +### 1. Check the metrics endpoint + +The `exporter` sidecar container has no shell utilities (no `wget`/`curl`/`ps`), so rather than exec-ing into the pod, query the metric straight from Prometheus: + +```bash +$ curl -s 'http://localhost:9090/api/v1/query?query=pgbouncer_up%7Bnamespace%3D%22alert-pgbouncer%22%7D' | jq . +{ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "__name__": "pgbouncer_up", + "container": "exporter", + "job": "pgbouncer-alert-stats", + "namespace": "alert-pgbouncer", + "pod": "pgbouncer-alert-0" + }, + "value": [1784192719.41, "1"] + } + ] + } +} +``` + +`pgbouncer_up` reads `1`, confirming the exporter is successfully scraping the PgBouncer process. + +### 2. Check the Prometheus target is UP + +Prometheus discovers more than 20 scrape pools on a shared cluster, so instead of the Target health page, query `up` directly for a reliable view. + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-pgbouncer%22%7D&g0.tab=1`. + +

+ Prometheus up query — pgbouncer-alert-0 UP +

+ +The `pgbouncer-alert-0` pod reports `up == 1` via the `pgbouncer-alert-stats` service/job, confirming Prometheus is scraping it successfully. + +### 3. Confirm the PgBouncer alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — PgBouncer groups inactive +

+ +All rules across the `pgbouncer.database` and `pgbouncer.provisioner` groups show **INACTIVE**, confirming the instance is healthy and no alert thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +No alerts are firing for the `alert-pgbouncer` namespace. + +### 5. Explore the Grafana dashboard + +Grafana dashboards for PgBouncer are documented separately rather than duplicated in this alerting guide — see [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the PgBouncer dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.PgBouncer=true`). + +--- + +## Simulating a Firing Alert + +The previous section showed that all currently-loaded PgBouncer alerts are **INACTIVE** while the instance is healthy. This section deliberately triggers the `KubeDBpgbouncerPhaseNotReady` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +> **Chart bug found:** `PgBouncerDown` is defined as `pgbouncer_up{...} < 0`. Since `pgbouncer_up` only ever reads `0` or `1`, this condition can never be true — the alert can never fire regardless of how long PgBouncer is down. `KubeDBpgbouncerPhaseNotReady` (provisioner group) is used below instead, since it correctly reflects PgBouncer being down via the KubeDB operator's own phase tracking. + +PgBouncer runs under a `runit`-style process supervisor inside its container (`PID 1` is `runsvdir`, which watches the `pgbouncer` process via `runsv` and respawns it almost instantly after a kill). Killing it from *outside* the pod with repeated `kubectl exec ... kill` calls is not fast enough to out-pace the respawn — the network round-trip of each `exec` call is slower than `runsv`'s restart, so the process is back up before the next kill lands. Instead, run the crash loop **inside** the container with a single `exec` session so there is no round-trip delay between kills. + +> **Also watch for:** a plain `kill` (SIGTERM) puts PgBouncer into a graceful shutdown (`got SIGTERM, shutting down, waiting for all clients disconnect`) that can hang indefinitely if no client ever disconnects — the process stays alive but stops accepting new connections, and `runsv` won't respawn a process that hasn't actually exited. Always use `kill -9` (SIGKILL) for this simulation so the crash is immediate and clean. + +Because `KubeDBpgbouncerPhaseNotReady` requires the condition to persist for `for: 1m`, keep the process crashing for at least a couple of minutes so the KubeDB operator has time to mark the resource `NotReady` and hold it there past the one-minute window. + +### 1. Crash the PgBouncer process repeatedly + +```bash +$ kubectl exec -n alert-pgbouncer pgbouncer-alert-0 -c pgbouncer -- sh -c ' + end=$(( $(date +%s) + 150 )); + while [ $(date +%s) -lt $end ]; do + pkill -9 pgbouncer 2>/dev/null; + sleep 0.2; + done' +``` + +Let this run for a couple of minutes (it blocks in the foreground until the 150s window elapses — capture the screenshots below while it's running), then let it finish or interrupt it once you've captured the firing state. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — KubeDBpgbouncerPhaseNotReady Firing +

+ +`KubeDBpgbouncerPhaseNotReady` transitions from **INACTIVE** to **FIRING** once `kubedb_com_pgbouncer_status_phase{phase="NotReady"}` has read `1` continuously for the full `for: 1m` duration — this metric comes from the KubeDB operator's own view of the resource (exported via Panopticon), not from the PgBouncer metrics endpoint itself. Note that `PgBouncerExporterLastScrapeError` also fires as a side effect, since the exporter briefly fails to scrape a process that keeps getting killed mid-connection. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — KubeDBpgbouncerPhaseNotReady Firing +

+ +AlertManager shows the `KubeDBpgbouncerPhaseNotReady` alert. The alert card displays: + +- **Severity**: `critical` +- **pgbouncer**: `pgbouncer-alert` in the `alert-pgbouncer` namespace +- **phase**: `NotReady` +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore PgBouncer + +Once the crash loop stops, `runsv` respawns the `pgbouncer` process and the KubeDB operator marks the resource `Ready` again within a few reconcile cycles. + +```bash +$ kubectl get pgbouncer -n alert-pgbouncer pgbouncer-alert -w +NAME VERSION STATUS AGE +pgbouncer-alert 1.23.1 Ready 24m +``` + +> **Note:** if PgBouncer does not recover within a minute or two of stopping the loop (for example, if it was interrupted mid-shutdown rather than mid-crash), force a clean restart: +> +> ```bash +> $ kubectl delete pod -n alert-pgbouncer pgbouncer-alert-0 +> pod "pgbouncer-alert-0" deleted +> ``` +> +> The PetSet controller recreates the pod immediately. + +Once the phase returns to `Ready`, Prometheus marks the alert **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `pgbouncer-alert` instance in the `alert-pgbouncer` namespace via the PromQL label filters `job="pgbouncer-alert-stats"` / `namespace="alert-pgbouncer"` (database group), or `app="pgbouncer-alert"` / `namespace="alert-pgbouncer"` (provisioner group). + +### Database Group + +Fired based on live metrics from the `pgbouncer_exporter` sidecar. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `pgbouncerTooManyConnections` | warning | 1m | Current connections exceed 70% of `pgbouncer_databases_max_connections`. | +| `PgBouncerExporterLastScrapeError` | warning | instant | `pgbouncer_last_scrape_error` is non-zero — the exporter failed its last scrape of PgBouncer's admin console. | +| `PgBouncerDown` | critical | instant | **Chart bug**: expression is `pgbouncer_up < 0`, which can never be true since `pgbouncer_up` is only ever `0` or `1` — this alert never fires. Use `KubeDBpgbouncerPhaseNotReady` below to detect PgBouncer being down. | +| `PgBouncerLogPoolerErrorMessageCount` | critical | instant | More than 10 pooler error messages logged from the Postgres backend. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the PgBouncer resource phase (sourced from Panopticon, not the PgBouncer metrics endpoint). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBpgbouncerPhaseNotReady` | critical | 1m | KubeDB marked the PgBouncer resource `NotReady` — the pooler cannot reach/serve the backend. | +| `KubeDBPgBouncerPhaseCritical` | warning | 15m | PgBouncer is degraded but not fully unavailable. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + pgbouncerTooManyConnections: + enabled: true + duration: "5m" + val: 90 # fire at 90% instead of the default 70% + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade pgbouncer-alert oci://ghcr.io/appscode-charts/pgbouncer-alerts \ + -n alert-pgbouncer \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the pgbouncer-alerts release (PrometheusRule) +$ helm uninstall pgbouncer-alert -n alert-pgbouncer + +# Remove the PgBouncer instance +$ kubectl delete pgbouncer -n alert-pgbouncer pgbouncer-alert + +# Remove the PostgreSQL backend +$ kubectl delete postgres -n alert-pgbouncer pg-backend-alert + +# Delete namespace +$ kubectl delete ns alert-pgbouncer +``` + +## Next Steps + +- Monitor your PgBouncer instance with KubeDB using [built-in Prometheus](/docs/guides/pgbouncer/monitoring/using-builtin-prometheus.md). +- Monitor your PgBouncer instance with KubeDB using [Prometheus operator](/docs/guides/pgbouncer/monitoring/using-prometheus-operator.md). +- Use [private Docker registry](/docs/guides/pgbouncer/private-registry/using-private-registry.md) to deploy PgBouncer with KubeDB. +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/pgpool/monitoring/alerting.md b/docs/guides/pgpool/monitoring/alerting.md new file mode 100644 index 0000000000..0ec841ac95 --- /dev/null +++ b/docs/guides/pgpool/monitoring/alerting.md @@ -0,0 +1,478 @@ +--- +title: Pgpool Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: pp-monitoring-alerting + name: Alerting + parent: pp-monitoring-pgpool + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Pgpool Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Pgpool instance using the `pgpool-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-pgpool` namespace: + + ```bash + $ kubectl create ns alert-pgpool + namespace/alert-pgpool created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/pgpool/monitoring/overview.md). + +* Pgpool is a connection pooler/load balancer in front of a PostgreSQL backend, so this tutorial first deploys a single-node PostgreSQL instance, then a Pgpool instance pointed at it. Both objects are deployed with monitoring enabled. + +> Note: YAML files used in this tutorial are stored in [docs/examples/pgpool](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/pgpool) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Pgpool with a dedicated `pgpool2_exporter` sidecar (container name `exporter`) that exposes metrics on port `9719` — Pgpool itself has no built-in Prometheus endpoint, so every pod runs **two containers**: `pgpool` and `exporter`. +- **ServiceMonitor** (named `{pgpool-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `pgpool-alerts` chart and contains Pgpool alert definitions grouped by concern: database health and provisioner. +- **Grafana** dashboards for Pgpool are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy the PostgreSQL Backend + +Pgpool pools/load-balances connections to a PostgreSQL instance, so deploy the backend first. + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-backend-alert + namespace: alert-pgpool +spec: + version: "16.13" + replicas: 1 + storageType: Durable + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + configuration: + inline: + user.conf: | + max_connections=200 + deletionPolicy: WipeOut +``` + +Here, `spec.configuration.inline` raises `max_connections` from the default `100` to `200`. This isn't optional — see the note below. + +> **Why `max_connections` must be raised:** the `Pgpool` admission webhook enforces `2 * num_init_children * replicas * max_pool <= backend max_connections`, and separately requires `max_pool >= 15 * replicas` and `num_init_children >= 5`. Combining the two minimums (`num_init_children=5`, `max_pool=15`) already requires `2*5*1*15 = 150` backend connections — more than PostgreSQL's stock default of 100. Deploying Pgpool against an unmodified default-config Postgres backend fails validation outright with `total connection for pgpool exceed max backend connection`. Bumping the backend to `max_connections=200` (or higher) is the simplest fix. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/pgpool/monitoring/pg-backend-alert.yaml +postgres.kubedb.com/pg-backend-alert created +``` + +Wait for the PostgreSQL instance to go into `Ready` state. + +```bash +$ kubectl get postgres -n alert-pgpool pg-backend-alert +NAME VERSION STATUS AGE +pg-backend-alert 16.13 Ready 3m +``` + +## Deploy Pgpool with Monitoring Enabled + +Now deploy Pgpool, pointing `spec.postgresRef` at the PostgreSQL instance above. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Pgpool +metadata: + name: pgpool-alert + namespace: alert-pgpool +spec: + version: "4.5.3" + postgresRef: + name: pg-backend-alert + namespace: alert-pgpool + configuration: + inline: + pgpool.conf: | + num_init_children=5 + max_pool=15 + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.postgresRef` tells Pgpool which KubeDB Postgres object to pool/load-balance connections for. +- `spec.configuration.inline` sets `num_init_children`/`max_pool` to the minimum values the admission webhook accepts (see the note above) — matched against the backend's `max_connections=200`. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/pgpool/monitoring/pgpool-alert.yaml +pgpool.kubedb.com/pgpool-alert created +``` + +Now, wait for Pgpool to go into `Ready` state. + +```bash +$ kubectl get pgpool -n alert-pgpool pgpool-alert +NAME VERSION STATUS AGE +pgpool-alert 4.5.3 Ready 40s +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-pgpool --selector="app.kubernetes.io/instance=pgpool-alert" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +pgpool-alert ClusterIP 10.43.6.157 9999/TCP,9595/TCP 40s +pgpool-alert-pods ClusterIP None 9999/TCP 40s +pgpool-alert-stats ClusterIP 10.43.248.150 9719/TCP 40s +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-pgpool +NAME AGE +pgpool-alert-stats 40s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-pgpool pgpool-alert-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install pgpool-alerts + +The `pgpool-alerts` chart creates a `PrometheusRule` resource containing Pgpool alert definitions grouped by concern. + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the Pgpool object's name (`pgpool-alert`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i pgpool-alert oci://ghcr.io/appscode-charts/pgpool-alerts \ + -n alert-pgpool \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `pgpool-alert` (release name) | — | Scopes every PromQL expression to this instance (`job="pgpool-alert-stats"`, `app="pgpool-alert"`) | +| `-n alert-pgpool` | `alert-pgpool` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-pgpool +NAME AGE +pgpool-alert 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-pgpool pgpool-alert \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +> **Chart gap found:** like several other `*-alerts` charts, `values.yaml` also declares an `opsManager` alert group (`opsRequestFailed`, `opsRequestOnProgress`, `opsRequestStatusProgressingToLong`), but only the `database` and `provisioner` groups are actually rendered into the live `PrometheusRule` at chart version `v2026.7.14`. Cross-check `helm show values` against `kubectl get prometheusrule ... -o yaml` rather than assuming every documented group is loaded. + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `pgpool.database` and `pgpool.provisioner` groups. + +

+ Prometheus Rule Health +

+ +Both groups are visible with all 10 rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Pgpool alert definitions every 30 seconds. ("OK" here means the rule expression evaluates without error — it's independent of whether the alert condition itself is currently true; see the next section.) + +--- + +## Verify End-to-End + +### 1. Check the metrics endpoint + +```bash +$ curl -s 'http://localhost:9090/api/v1/query?query=pgpool2_up%7Bnamespace%3D%22alert-pgpool%22%7D' | jq . +{ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "__name__": "pgpool2_up", + "container": "exporter", + "job": "pgpool-alert-stats", + "namespace": "alert-pgpool", + "pod": "pgpool-alert-0" + }, + "value": [1784194103.883, "1"] + } + ] + } +} +``` + +`pgpool2_up` reads `1`, confirming the exporter is successfully querying Pgpool. + +### 2. Check the Prometheus target is UP + +Prometheus discovers more than 20 scrape pools on a shared cluster, so instead of the Target health page, query `up` directly for a reliable view. + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-pgpool%22%7D&g0.tab=1`. + +

+ Prometheus up query — pgpool-alert-0 UP +

+ +The `pgpool-alert-0` pod reports `up == 1` via the `pgpool-alert-stats` service/job, confirming Prometheus is scraping it successfully. + +### 3. Confirm the Pgpool alerts + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — two Pgpool alerts firing by design +

+ +6 of the 8 `database`-group rules show **INACTIVE** as expected, but **`PgpoolTooManyConnections` and `PgpoolLowCacheMemory` are FIRING even on this freshly-deployed, idle instance** — both are chart/threshold gaps, not real problems: + +- **`PgpoolTooManyConnections`** (`pgpool2_backend_by_process_total / pgpool2_backend_total > 0.1`) does not measure actual client-connection load at all. `pgpool2_backend_by_process_total` reads one child process's configured `max_pool`, and `pgpool2_backend_by_process_total`'s ratio to the fleet-wide total collapses to `1 / num_init_children` — a constant determined purely by config, not usage. With `num_init_children=5` (the webhook-enforced minimum from the note above), that ratio is a fixed `20%`, permanently over the chart's default `10%` threshold. Only `num_init_children >= 11` would bring the ratio under 10%, which in turn requires raising the backend's `max_connections` well past 200 to satisfy the other webhook check. +- **`PgpoolLowCacheMemory`** (`pgpool2_pool_cache_free_cache_entries_size / 1000000 < 100`) fires because Pgpool's in-memory query cache (`memory_cache_enabled`) is **off by default** — the metric simply reads `0` when the cache is disabled, which is always `< 100`. This alert is only meaningful once `memory_cache_enabled=on` is explicitly configured. + +Both are documented in the Alert Reference below; disable or raise their thresholds if you don't intend to run with a large `num_init_children` or the in-memory cache enabled. + +### 4. Check AlertManager + +Port-forward AlertManager to view currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager — PgpoolLowCacheMemory firing by design +

+ +Matches the previous step — `PgpoolLowCacheMemory` (and, depending on scrape timing, `PgpoolTooManyConnections`) is visible here for the reasons explained above, not because anything is actually broken. + +### 5. Explore the Grafana dashboard(s) + +Grafana dashboards for Pgpool are documented separately rather than duplicated in this alerting guide — see [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the Pgpool dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.Pgpool=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately breaks the PostgreSQL backend so you can observe `PgpoolDown` transition from **INACTIVE** to **FIRING**, through to the AlertManager dashboard, and then resolve it. + +> **Why the backend, not Pgpool itself:** Pgpool's own worker processes are supervised and respawn essentially instantly after a `kill`, so a simple repeated-kill loop against the `pgpool` container never leaves a large enough gap for Prometheus to observe an outage. Freezing every Pgpool process with `kill -STOP` also doesn't produce a clean "down" signal — the exporter's own query to Pgpool just hangs, so the whole scrape times out (`up=0`, no app metrics at all) rather than reporting `pgpool2_up=0`. Crashing the **backend Postgres** instead reliably drives `pgpool2_up` to `0`, because the exporter can still reach Pgpool itself; Pgpool just fails to service the exporter's probe query against a dead backend. + +> **Container caveat:** the `postgres` container's `PID 1` is `tini`, which does not notice if the `postgres` server process underneath it dies — `kubectl get pods` keeps reporting `1/1 Running` even though the database is gone for good. Recovery requires deleting the pod (see Step 4), not just waiting. + +### 1. Crash the PostgreSQL backend + +```bash +$ kubectl exec -n alert-pgpool pg-backend-alert-0 -- sh -c ' + end=$(( $(date +%s) + 60 )); + while [ $(date +%s) -lt $end ]; do + pid=$(pgrep -x postgres | head -1); + [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null; + sleep 1; + done' +``` + +A single `kill -9` on the postmaster is enough to bring the backend down permanently in this image — the wrapper script does not restart it. The loop above just ensures the very first attempt lands. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — PgpoolDown Firing +

+ +`PgpoolDown` (`pgpool2_up == 0`, `for: 0m`) transitions to **FIRING** as soon as the exporter's probe query against the dead backend fails. `PgpoolExporterLastScrapeError` fires alongside it for the same reason. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — PgpoolDown Firing +

+ +AlertManager shows the `PgpoolDown` alert. The alert card displays: + +- **Severity**: `critical` +- **pgpool**: `pgpool-alert` in the `alert-pgpool` namespace +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore the PostgreSQL backend + +Because of the `tini`/dead-postmaster caveat above, the pod will not recover on its own — delete it to force the PetSet to recreate it cleanly. + +```bash +$ kubectl delete pod -n alert-pgpool pg-backend-alert-0 +pod "pg-backend-alert-0" deleted + +$ kubectl get postgres -n alert-pgpool pg-backend-alert -w +NAME VERSION STATUS AGE +pg-backend-alert 16.13 Ready 40m +``` + +Once both the Postgres backend and Pgpool report `Ready`, Prometheus marks `PgpoolDown` **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `pgpool-alert` instance in the `alert-pgpool` namespace via the PromQL label filters `job="pgpool-alert-stats"` / `namespace="alert-pgpool"` (database group), or `app="pgpool-alert"` / `namespace="alert-pgpool"` (provisioner group). + +### Database Group + +Fired based on live metrics from the `pgpool2_exporter` sidecar. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `PgpoolTooManyConnections` | warning | 1m | **Fires by design at the webhook-minimum pool config** — see the explanation above. Not usage-based; only meaningful once `num_init_children` is raised well above the minimum. | +| `PgpoolPostgresHealthCheckFailure` | critical | instant | More than 10 recorded health-check failures against the backend Postgres. | +| `PgpoolExporterLastScrapeError` | warning | instant | The exporter's last scrape/probe against Pgpool failed. | +| `PgpoolBackendPanicMessageCount` | critical | instant | More than 10 `PANIC`-level messages returned from the backend. | +| `PgpoolBackendFatalMessageCount` | critical | instant | More than 10 `FATAL`-level messages returned from the backend. | +| `PgpoolBackendErrorMessageCount` | critical | instant | More than 10 `ERROR`-level messages returned from the backend. | +| `PgpoolLowCacheMemory` | warning | 1m | **Fires by design when `memory_cache_enabled` is off** (the default) — see the explanation above. | +| `PgpoolDown` | critical | instant | `pgpool2_up == 0` — the exporter's probe query against Pgpool failed, typically because the backend Postgres is unreachable. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Pgpool resource phase (sourced from Panopticon, not the Pgpool metrics endpoint). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBPgpoolPhaseNotReady` | critical | 1m | KubeDB marked the Pgpool resource `NotReady`. | +| `KubeDBPgpoolPhaseCritical` | warning | 15m | Pgpool is degraded but not fully unavailable. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + pgpoolLowCacheMemory: + enabled: false # disable, since memory_cache is off in this setup + pgpoolTooManyConnections: + enabled: true + duration: "5m" + val: 0.5 # only fire above 50%, since the ratio is config-derived here + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade pgpool-alert oci://ghcr.io/appscode-charts/pgpool-alerts \ + -n alert-pgpool \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the pgpool-alerts release (PrometheusRule) +$ helm uninstall pgpool-alert -n alert-pgpool + +# Remove the Pgpool instance +$ kubectl delete pgpool -n alert-pgpool pgpool-alert + +# Remove the PostgreSQL backend +$ kubectl delete postgres -n alert-pgpool pg-backend-alert + +# Delete namespace +$ kubectl delete ns alert-pgpool +``` + +## Next Steps + +- Monitor your Pgpool instance with KubeDB using [built-in Prometheus](/docs/guides/pgpool/monitoring/using-builtin-prometheus.md). +- Monitor your Pgpool instance with KubeDB using [Prometheus operator](/docs/guides/pgpool/monitoring/using-prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/postgres/monitoring/alerting.md b/docs/guides/postgres/monitoring/alerting.md new file mode 100644 index 0000000000..8713f5162d --- /dev/null +++ b/docs/guides/postgres/monitoring/alerting.md @@ -0,0 +1,584 @@ +--- +title: PostgreSQL Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: pg-monitoring-alerting + name: Alerting + parent: pg-monitoring-postgres + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# PostgreSQL Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed PostgreSQL instance using the `postgres-alerts` Helm chart, and how to visualise live metrics using the `kubedb-grafana-dashboards` chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes your Prometheus instance is configured with both `serviceMonitorSelector` and `ruleSelector` matching the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/postgres/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/postgres](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/postgres) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +The diagram below shows the full alerting architecture — from PostgreSQL metric export through to alert delivery and Grafana visualisation. + +

+ PostgreSQL Alerting Architecture +

+ +- **KubeDB** deploys PostgreSQL with a built-in `postgres_exporter` sidecar that exposes metrics on port `56790`. +- **ServiceMonitor** (named `{postgres-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `postgres-alerts` chart and contains all PostgreSQL alert definitions. Prometheus loads it because it carries the `release: prometheus` label matching the `ruleSelector`. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). +- **Grafana** visualises metrics through pre-built dashboards provisioned by the `kubedb-grafana-dashboards` chart. + +--- + +## Deploy PostgreSQL with Monitoring Enabled + +At first, let's deploy a PostgreSQL database with monitoring enabled. Below is the PostgreSQL object we are going to create. + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-grafana-demo + namespace: demo +spec: + version: "13.13" + deletionPolicy: WipeOut + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + exporter: + port: 56790 + serviceMonitor: + interval: 10s + labels: + release: prometheus +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the PostgreSQL resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/postgres/monitoring/pg-grafana-demo.yaml +postgres.kubedb.com/pg-grafana-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get postgres -n demo pg-grafana-demo +NAME VERSION STATUS AGE +pg-grafana-demo 13.13 Ready 2m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n demo --selector="app.kubernetes.io/instance=pg-grafana-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +pg-grafana-demo ClusterIP 10.43.6.170 5432/TCP 2m +pg-grafana-demo-pods ClusterIP None 5432/TCP 2m +pg-grafana-demo-stats ClusterIP 10.43.181.56 56790/TCP 2m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n demo +NAME AGE +pg-grafana-demo-stats 2m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n demo pg-grafana-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install postgres-alerts + +The `postgres-alerts` chart creates a `PrometheusRule` resource containing all PostgreSQL alert definitions grouped by concern: database health, provisioner, ops-manager, backup, and schema manager. + +### Why the `release: prometheus` label matters + +The Prometheus `ruleSelector` only loads `PrometheusRule` resources that carry `release: prometheus`. The chart default label is `release: kube-prometheus-stack`, so we must override it at install time. + +### Install + +```bash +$ helm upgrade -i postgres-alerts oci://ghcr.io/appscode-charts/postgres-alerts \ + -n demo --create-namespace --version=v2026.2.24 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `metadata.release.name` | `pg-grafana-demo` | Scopes every PromQL expression to this instance | +| `metadata.release.namespace` | `demo` | Scopes every PromQL expression to this namespace | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n demo +NAME AGE +postgres-alerts 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n demo postgres-alerts \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and search for **postgres**. + +

+ Prometheus Rule Health +

+ +The `postgres.database.demo.postgres-alerts.rules` group is visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the PostgreSQL alert definitions every 30 seconds. + +--- + +## Step 2 — Install kubedb-grafana-dashboards + +The `kubedb-grafana-dashboards` chart creates `GrafanaDashboard` CRDs containing pre-built PostgreSQL dashboard JSON. These are automatically provisioned into Grafana. + +### Create a Grafana service account token + +The chart needs a Grafana API key to push dashboards. + +```bash +# Port-forward Grafana +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 + +# Create a service account with Admin role +$ curl -s -X POST -H "Content-Type: application/json" \ + -u admin: \ + http://localhost:3000/api/serviceaccounts \ + -d '{"name":"kubedb-dashboards","role":"Admin"}' +# Note the returned "id" + +# Create a token for the service account (replace ) +$ curl -s -X POST -H "Content-Type: application/json" \ + -u admin: \ + http://localhost:3000/api/serviceaccounts//tokens \ + -d '{"name":"kubedb-token","secondsToLive":0}' +# Note the returned "key" + +$ kill %1 +``` + +> **Tip:** Retrieve the Grafana admin password from its secret: +> ```bash +> $ kubectl get secret -n monitoring prometheus-grafana \ +> -o jsonpath='{.data.admin-password}' | base64 -d && echo +> ``` + +### Install + +The `kubedb-grafana-dashboards` chart bundles many large Grafana dashboard JSON files. Even with a single `featureGate` enabled, the rendered manifests can exceed Kubernetes' hard 1 MB Secret limit that Helm uses to store release state. To work around this, render the chart locally with `helm template` and apply the output directly with `kubectl apply`, which bypasses Helm's Secret storage entirely. + +```bash +$ helm repo add appscode https://charts.appscode.com/stable/ +$ helm repo update appscode + +# Create the namespace first (idempotent) +$ kubectl create namespace kubeops --dry-run=client -o yaml | kubectl apply -f - + +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.6.19 \ + --set featureGates.Postgres=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl apply -n kubeops -f - +``` + +> **Note:** Because `helm template | kubectl apply` does not create a Helm release object, `helm uninstall` will not work for cleanup. Use `kubectl delete` directly (see [Cleaning up](#cleaning-up)). + +### Verify dashboards are created + +```bash +$ kubectl get grafanadashboards -n kubeops +NAME TITLE STATUS AGE +kubedb-postgres-database KubeDB / Postgres / Database 2m +kubedb-postgres-pod KubeDB / Postgres / Pod 2m +kubedb-postgres-summary KubeDB / Postgres / Summary 2m +``` + +--- + +## Verify End-to-End + +### 1. Check the exporter is running + +The `exporter` sidecar inside each PostgreSQL pod serves metrics at `:56790/metrics`. A value of `pg_up 1` confirms the exporter can reach PostgreSQL. + +```bash +$ kubectl exec -n demo pg-grafana-demo-0 -c exporter -- \ + wget -qO- localhost:56790/metrics | grep pg_up +# HELP pg_up Whether the last scrape of metrics from PostgreSQL was able to connect to the server (1 for yes, 0 for no). +# TYPE pg_up gauge +pg_up 1 +``` + +### 2. Check the Prometheus target is UP + +Port-forward Prometheus and open the **Status → Target health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/targets?search=pg-grafana-demo`. + +

+ Prometheus Target UP +

+ +The target `serviceMonitor/demo/pg-grafana-demo-stats/0` shows **UP** with labels confirming metrics come from `pg-grafana-demo-0` in the `demo` namespace, scraped 8 seconds ago in 25ms. + +### 3. Confirm all PostgreSQL alerts are inactive + +Open `http://localhost:9090/alerts?search=postgres` to see the PostgreSQL alert groups. + +

+ Prometheus Alerts — All Inactive +

+ +All 11 rules in the `postgres.database` group show **INACTIVE (11)**, meaning the database is healthy and no thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +> **Note for k3s users:** You may see `KubeProxyDown`, `KubeControllerManagerDown`, and `KubeSchedulerDown` alerts. These are k3s-specific — the control plane components do not expose Prometheus scrape endpoints by default. They are **not** related to PostgreSQL. To silence them, add a silence rule in AlertManager or set `kubeProxy.enabled: false`, `kubeControllerManager.enabled: false`, and `kubeScheduler.enabled: false` in your kube-prometheus-stack Helm values. + +### 5. Explore Grafana dashboards + +Port-forward Grafana and log in. + +```bash +$ kubectl port-forward -n monitoring svc/prometheus-grafana 3000:80 +``` + +Open `http://localhost:3000` (username: `admin`). Search for **postgres** in the Dashboards section. + +

+ Grafana — PostgreSQL Dashboard List +

+ +Three pre-built dashboards are available. The `Namespace` and `postgres` drop-downs at the top of each dashboard let you switch between instances. + +**KubeDB / Postgres / Summary** — high-level health: version, uptime, replica count, database phase, connection count, and replication lag. + +

+ Grafana — KubeDB Postgres Summary +

+ +**KubeDB / Postgres / Database** — query rates (QPS), transactions (commits vs rollbacks), active sessions, lock tables, and fetch/insert data throughput. + +

+ Grafana — KubeDB Postgres Database +

+ +**KubeDB / Postgres / Pod** — pod-level CPU usage, memory usage (resident vs virtual), open file descriptors, and PostgreSQL runtime settings (shared buffers, work_mem, max_connections). + +

+ Grafana — KubeDB Postgres Pod +

+ +--- + +## Simulating a Firing Alert + +The previous section confirmed that all alerts are **INACTIVE** while the database is healthy. This section walks through deliberately triggering the `PostgresqlDown` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +### 1. Stop the PostgreSQL pod + +Scale the PostgreSQL StatefulSet to zero replicas. The `postgres_exporter` sidecar will stop responding and `pg_up` will drop to `0` on the next scrape. + +```bash +$ kubectl scale statefulset -n demo pg-grafana-demo --replicas=0 +statefulset.apps/pg-grafana-demo scaled +``` + +Wait 30–60 seconds for the next Prometheus scrape cycle (configured at 10 s) and rule-evaluation cycle (30 s) to register the failure. + +### 2. Watch the alert fire in Prometheus + +Port-forward Prometheus if it is not already running. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/alerts?search=postgres`. + +

+ Prometheus Alerts — PostgresqlDown Firing +

+ +The `PostgresqlDown` alert transitions through three states: + +| State | Colour | Meaning | +|-------|--------|---------| +| **INACTIVE** | grey | Expression is false — database is up | +| **PENDING** | yellow | Expression is true but the `for` window has not elapsed | +| **FIRING** | red | Expression has been true for the full `for` duration — alert is sent to AlertManager | + +Because `PostgresqlDown` has `for: 0m` (instant), it moves directly from **INACTIVE** to **FIRING** within one evaluation cycle. + +### 3. Check the AlertManager dashboard + +Port-forward AlertManager if it is not already running. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager — PostgresqlDown Firing +

+ +AlertManager shows the `PostgresqlDown` alert grouped by `namespace` and `job`. The alert card displays: + +- **Severity**: `critical` +- **Instance**: `pg-grafana-demo-0` in the `demo` namespace +- **Source**: link back to the Prometheus expression that fired the alert +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore the PostgreSQL pod + +Scale the StatefulSet back to `1` to resolve the alert. + +```bash +$ kubectl scale statefulset -n demo pg-grafana-demo --replicas=1 +statefulset.apps/pg-grafana-demo scaled +``` + +Wait for the pod to become `Running` and for the next scrape cycle to register `pg_up 1`. + +```bash +$ kubectl get pods -n demo -w +NAME READY STATUS RESTARTS AGE +pg-grafana-demo-0 2/2 Running 0 45s +``` + +Once `pg_up` returns to `1`, Prometheus marks the alert **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. The AlertManager dashboard will show no active alerts for the instance. + +--- + +## Alert Reference + +All alerts are scoped to the `pg-grafana-demo` instance in the `demo` namespace via the PromQL label filters `job="pg-grafana-demo-stats"` and `namespace="demo"`. + +### Database Group + +Fired based on live metrics from `postgres_exporter`. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `PostgresqlDown` | critical | instant | Exporter cannot reach PostgreSQL — instance is down or exporter crashed. | +| `PostgresqlSplitBrain` | critical | instant | More than one node reports as primary — data divergence risk in a replica set. | +| `PostgresqlTooManyLocksAcquired` | critical | 2m | Lock table nearly full; transactions may fail with "out of shared memory". | +| `PostgresReplicationSlotLagHigh` | warning | 1m | Replication slot consumer is falling behind; WAL is accumulating (>800 MB). | +| `PostgresReplicationSlotLagCritical` | critical | 1m | Slot lag is critical — disk exhaustion or slot invalidation imminent (>1.2 GB). | +| `PostgresqlRestarted` | critical | instant | PostgreSQL restarted within the last minute. | +| `PostgresqlExporterError` | warning | 5m | Exporter has errors — metrics may be missing or stale. | +| `PostgresqlHighRollbackRate` | warning | instant | High proportion of transactions are rolling back — application errors or lock contention. | +| `PostgresTooManyConnections` | warning | 2m | Connection pool nearly exhausted (>95% of `max_connections`). | +| `DiskUsageHigh` | warning | 1m | PVC used space exceeds 80% — plan for expansion. | +| `DiskAlmostFull` | critical | 1m | PVC almost full (>95%) — PostgreSQL may become read-only or crash. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Postgres resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBPostgreSQLPhaseNotReady` | critical | 1m | KubeDB marked the Postgres resource `NotReady` — operator cannot reach the database. | +| `KubeDBPostgreSQLPhaseCritical` | warning | 15m | One or more replicas are down; cluster is degraded but primary is still up. | + +### OpsManager Group + +Tracks `PostgresOpsRequest` lifecycle during upgrades, scaling, reconfiguration, and certificate rotations. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBPostgreSQLOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes — likely stuck. | +| `KubeDBPostgreSQLOpsRequestFailed` | critical | instant | An ops request failed — check the `OpsRequest` object for the error. | + +### Backup & Restore Groups (Stash / KubeStash) + +Two parallel sets of alerts cover both the legacy Stash and current KubeStash backup frameworks. + +| Alert | Severity | What It Means | +|-------|----------|---------------| +| `PostgreSQL*BackupSessionFailed` | critical | A scheduled backup run failed — check `BackupSession` logs. | +| `PostgreSQL*RestoreSessionFailed` | critical | A restore operation failed. | +| `PostgreSQL*NoBackupSessionForTooLong` | critical | No successful backup in the expected window — backup may be misconfigured or stuck. | +| `PostgreSQL*RepositoryCorrupted` | critical | Backup repository integrity check failed — backups may be unrestorable. | +| `PostgreSQL*RepositoryStorageRunningLow` | warning | Backup storage has less than 10 GB free. | +| `PostgreSQL*BackupSessionPeriodTooLong` | warning | Backup took longer than 30 minutes. | +| `PostgreSQL*RestoreSessionPeriodTooLong` | warning | Restore took longer than 30 minutes. | + +### SchemaManager Group + +Monitors `PostgresDatabase` schema lifecycle objects managed by KubeDB Schema Manager. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBPostgreSQLSchemaPendingForTooLong` | warning | 30m | Schema object stuck in `Pending` — may be waiting on a dependency. | +| `KubeDBPostgreSQLSchemaInProgressForTooLong` | warning | 30m | Schema migration running for 30+ minutes — may be stuck. | +| `KubeDBPostgreSQLSchemaTerminatingForTooLong` | warning | 30m | Schema deletion stuck — a finalizer may be blocking it. | +| `KubeDBPostgreSQLSchemaFailed` | warning | instant | Schema operation failed. | +| `KubeDBPostgreSQLSchemaExpired` | warning | instant | A schema with a TTL has expired and been revoked. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + postgresqlTooManyConnections: + enabled: true + duration: "5m" + val: 90 # fire at 90% instead of the default 95% + severity: warning + stash: + enabled: "none" # disable all stash backup alerts +``` + +```bash +$ helm upgrade postgres-alerts oci://ghcr.io/appscode-charts/postgres-alerts \ + -n demo \ + --version=v2026.2.24 \ + --set metadata.release.name=pg-grafana-demo \ + --set metadata.release.namespace=demo \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the Grafana dashboards (installed via helm template | kubectl apply, not helm install) +$ helm template kubedb-grafana-dashboards appscode/kubedb-grafana-dashboards \ + -n kubeops \ + --version=v2026.6.19 \ + --set featureGates.Postgres=true \ + --set grafana.url="http://prometheus-grafana.monitoring.svc:80" \ + --set grafana.apikey="" \ + | kubectl delete -n kubeops -f - --ignore-not-found + +# Remove the postgres-alerts +$ helm uninstall postgres-alerts -n demo + +# Remove the PostgreSQL instance +$ kubectl delete postgres -n demo pg-grafana-demo + +# Delete namespaces +$ kubectl delete ns demo +$ kubectl delete ns kubeops +``` + +## Next Steps + +- Monitor your PostgreSQL database with KubeDB using [builtin Prometheus](/docs/guides/postgres/monitoring/using-builtin-prometheus.md). +- Monitor your PostgreSQL database with KubeDB using [Prometheus operator](/docs/guides/postgres/monitoring/using-prometheus-operator.md). +- Learn about [backup and restore](/docs/guides/postgres/backup/stash/overview/index.md) PostgreSQL databases using Stash. +- Use [private Docker registry](/docs/guides/postgres/private-registry/using-private-registry.md) to deploy PostgreSQL with KubeDB. +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/proxysql/monitoring/alerting.md b/docs/guides/proxysql/monitoring/alerting.md new file mode 100644 index 0000000000..0f1db12f29 --- /dev/null +++ b/docs/guides/proxysql/monitoring/alerting.md @@ -0,0 +1,489 @@ +--- +title: ProxySQL Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: guides-proxysql-monitoring-alerting + name: Alerting + parent: guides-proxysql-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# ProxySQL Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed ProxySQL instance using the `proxysql-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-proxysql` namespace: + + ```bash + $ kubectl create ns alert-proxysql + namespace/alert-proxysql created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/proxysql/monitoring/overview/index.md). + +* ProxySQL is a proxy layer in front of a MySQL backend, so this tutorial first deploys a 3-member MySQL Group Replication cluster, then a ProxySQL instance pointed at it. Both objects are deployed with monitoring enabled. + +> Note: YAML files used in this tutorial are stored in [docs/examples/proxysql](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/proxysql) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys ProxySQL with metrics exposed directly by the `proxysql` container itself on port `6070` (an embedded `proxysql_exporter`-style endpoint) — there is no separate exporter sidecar. +- **ServiceMonitor** (named `{proxysql-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the metrics endpoint every 10 seconds. +- **PrometheusRule** is created by the `proxysql-alerts` chart and contains ProxySQL alert definitions grouped by concern: database health, cluster sync, provisioner, and ops-manager. +- **Grafana** dashboards for ProxySQL are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy the MySQL Backend + +ProxySQL routes traffic to a MySQL cluster, so deploy the backend first. Below is the MySQL object we are going to create — a 3-member Group Replication cluster on the `longhorn` StorageClass. + +```yaml +apiVersion: kubedb.com/v1 +kind: MySQL +metadata: + name: my-group-alert + namespace: alert-proxysql +spec: + version: "9.1.0" + replicas: 3 + topology: + mode: GroupReplication + group: + name: "dc002fc3-c412-4d18-b1d4-66c1fbfbbc9b" + storageType: Durable + storage: + storageClassName: "longhorn" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/proxysql/monitoring/my-group-alert.yaml +mysql.kubedb.com/my-group-alert created +``` + +Wait for the MySQL cluster to go into `Ready` state, and confirm all three PVCs bind on `longhorn`. + +```bash +$ kubectl get mysql -n alert-proxysql my-group-alert +NAME VERSION STATUS AGE +my-group-alert 9.1.0 Ready 5m + +$ kubectl get pvc -n alert-proxysql +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +data-my-group-alert-0 Bound pvc-9d47405f-06ec-4c79-ab93-b1588451896a 1Gi RWO longhorn 5m +data-my-group-alert-1 Bound pvc-a3fec212-fff1-489d-b9d0-c8daf143f8fc 1Gi RWO longhorn 2m +data-my-group-alert-2 Bound pvc-a3fd6857-70bf-4a1b-a20a-2997379d8678 1Gi RWO longhorn 2m +``` + +> **Note:** `storageClassName` is immutable on a PVC. If you need to move an existing MySQL/ProxySQL instance from one StorageClass to another (e.g. `local-path` → `longhorn`), you cannot edit the field in place — delete the database object (and its PVCs, if `deletionPolicy` is not `WipeOut`) and recreate it pointing at the new StorageClass. + +## Deploy ProxySQL with Monitoring Enabled + +Now deploy ProxySQL, pointing `spec.backend.name` at the MySQL cluster above. + +```yaml +apiVersion: kubedb.com/v1 +kind: ProxySQL +metadata: + name: proxysql-alert + namespace: alert-proxysql +spec: + version: "2.3.2-debian" + replicas: 1 + backend: + name: my-group-alert + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + exporter: + port: 42004 + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.backend.name: my-group-alert` tells ProxySQL which KubeDB MySQL object to load-balance traffic to. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +> ProxySQL itself does not provision its own PVC — it has no `spec.storage` field, so there is nothing to migrate to `longhorn` beyond the MySQL backend's PVCs above. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/proxysql/monitoring/proxysql-alert.yaml +proxysql.kubedb.com/proxysql-alert created +``` + +Now, wait for ProxySQL to go into `Ready` state. + +```bash +$ kubectl get proxysql -n alert-proxysql proxysql-alert +NAME VERSION STATUS AGE +proxysql-alert 2.3.2-debian Ready 40s +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-proxysql --selector="app.kubernetes.io/instance=proxysql-alert" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +proxysql-alert ClusterIP 10.43.180.207 6033/TCP 40s +proxysql-alert-pods ClusterIP None 6032/TCP,6033/TCP 40s +proxysql-alert-stats ClusterIP 10.43.61.242 6070/TCP 40s +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-proxysql +NAME AGE +proxysql-alert-stats 40s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-proxysql proxysql-alert-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install proxysql-alerts + +The `proxysql-alerts` chart creates a `PrometheusRule` resource containing ProxySQL alert definitions grouped by concern. + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the ProxySQL object's name (`proxysql-alert`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i proxysql-alert oci://ghcr.io/appscode-charts/proxysql-alerts \ + -n alert-proxysql \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `proxysql-alert` (release name) | — | Scopes every PromQL expression to this instance (`job="proxysql-alert-stats"`, `app="proxysql-alert"`) | +| `-n alert-proxysql` | `alert-proxysql` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-proxysql +NAME AGE +proxysql-alert 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-proxysql proxysql-alert \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and filter by **proxysql**. + +

+ Prometheus Rule Health +

+ +Three rule groups are visible — `proxysql.database`, `proxysql.opsManager`, and `proxysql.provisioner` — all showing **OK**, confirming Prometheus has loaded and is evaluating the ProxySQL alert definitions every 30 seconds. + +> **Known chart bug (v2026.7.14):** The chart's own `values.yaml` declares nine alerts under the `database` group (`ProxySQLInstanceDown`, `ProxySQLServiceDown`, `ProxySQLTooManyConnections`, `ProxySQLHighThreadsRunning`, `ProxySQLSlowQueries`, `ProxySQLRestarted`, `ProxySQLHighQPS`, `ProxySQLHighIncomingBytes`, `ProxySQLHighOutgoingBytes`) plus a separate `cluster` group holding `ProxySQLCLusterSyncFailure`. Running `helm template`/`helm get manifest` against this same chart version now renders all of that correctly. However, the `PrometheusRule` actually installed live in this cluster contains only **one** rule under the group named `proxysql.database...` — and it is in fact the `cluster` group's `ProxySQLCLusterSyncFailure` rule content, mislabeled under the `database` group name, with the real database-health rules (instance/service down, connections, QPS, threads, slow queries, bytes) entirely absent, and no independent `proxysql.cluster...` group at all. Since the Helm release has never been upgraded (`generation: 1`, single revision) and `managedFields` shows only `helm` ever wrote the object, this indicates the chart's OCI artifact at tag `v2026.7.14` changed content after this release was first installed. If you need the full rule set, re-run the `helm upgrade` command above to reconcile the live object with the current chart content — the value of doing so is that you gain the missing `database`-group health alerts, at the cost of a brief `PrometheusRule` re-apply (no database downtime). + +--- + +## Verify End-to-End + +### 1. Check the metrics endpoint + +The `proxysql` container serves its own Prometheus metrics at `:6070/metrics` — no exporter sidecar is involved. + +```bash +$ kubectl exec -n alert-proxysql proxysql-alert-0 -c proxysql -- \ + wget -qO- localhost:6070/metrics | grep proxysql_servers_table_version_total +# HELP proxysql_servers_table_version_total Number of times the "servers_table" have been modified. +# TYPE proxysql_servers_table_version_total counter +proxysql_servers_table_version_total 16.000000 +``` + +### 2. Check the Prometheus target is UP + +Prometheus discovers more than 20 scrape pools on a shared cluster, so instead of the Target health page, query `up` directly for a reliable view. + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-proxysql%22%7D&g0.tab=1`. + +

+ Prometheus up query — proxysql-alert-0 UP +

+ +The `proxysql-alert-0` pod reports `up == 1` via the `proxysql-alert-stats` service/job, confirming Prometheus is scraping it successfully. + +### 3. Confirm all ProxySQL alerts are inactive + +Open `http://localhost:9090/alerts` and filter by **proxysql**. + +

+ Prometheus Alerts — ProxySQL groups inactive +

+ +All 5 currently-loaded rules across the three groups show **INACTIVE**, confirming the cluster is healthy and no thresholds are breached (see the chart-bug callout above for why only 5 of the chart's defined rules are loaded). + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +No alerts are firing for the `alert-proxysql` namespace. + +### 5. Explore the Grafana Dashboard + +Grafana dashboards for ProxySQL are documented separately rather than duplicated in this alerting guide — see [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the ProxySQL dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.ProxySQL=true`). + +--- + +## Simulating a Firing Alert + +The previous section showed that all currently-loaded ProxySQL alerts are **INACTIVE** while the instance is healthy. This section deliberately triggers the `KubeDBProxySQLPhaseNotReady` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +ProxySQL runs as a **single container per pod** — there is no separate exporter sidecar. The container has neither `ps` nor `pgrep`, only `bash`/`sh`, so identify the actual `proxysql` process via `/proc` and kill it directly (rather than the container's PID 1, which is `tini`). Because `KubeDBProxySQLPhaseNotReady` requires the condition to persist for `for: 1m`, a single `kill` is not enough — keep the process crashing long enough for the KubeDB operator to mark the resource `NotReady` and hold it there past the one-minute window. + +### 1. Crash the ProxySQL process repeatedly + +```bash +$ while true; do + kubectl exec -n alert-proxysql proxysql-alert-0 -c proxysql -- bash -c ' + for p in /proc/[0-9]*; do + pid=$(basename "$p") + cmd=$(tr "\0" " " < "$p/cmdline" 2>/dev/null) + case "$cmd" in + proxysql\ -c*) kill -9 "$pid" ;; + esac + done + ' >/dev/null 2>&1 + sleep 3 + done +``` + +Let this loop run for a couple of minutes (leave it running while you check the next steps), then stop it once you've captured the firing state. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts` filtered by **proxysql**. + +

+ Prometheus Alerts — KubeDBProxySQLPhaseNotReady Firing +

+ +`KubeDBProxySQLPhaseNotReady` transitions from **INACTIVE** to **FIRING** once `kubedb_com_proxysql_status_phase{phase="NotReady"}` has read `1` continuously for the full `for: 1m` duration — this metric comes from the KubeDB operator's own view of the resource (exported via Panopticon), not from the ProxySQL metrics endpoint itself. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — KubeDBProxySQLPhaseNotReady Firing +

+ +AlertManager shows the `KubeDBProxySQLPhaseNotReady` alert. The alert card displays: + +- **Severity**: `critical` +- **proxysql**: `proxysql-alert` in the `alert-proxysql` namespace +- **phase**: `NotReady` +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore ProxySQL + +Stop the loop from step 1. + +> **Note:** Unlike some other KubeDB databases, the ProxySQL image's entrypoint script does **not** automatically respawn the `proxysql` process after it is repeatedly `kill -9`'d — the wrapper script exits instead of retrying, and no liveness probe recovers it, so the pod can remain `Running` (`READY 1/1`) while the daemon inside is actually dead. If ProxySQL does not return to `Ready` on its own within a minute or two of stopping the loop, force a clean restart: +> +> ```bash +> $ kubectl delete pod -n alert-proxysql proxysql-alert-0 +> pod "proxysql-alert-0" deleted +> ``` +> +> The PetSet controller recreates the pod immediately. + +```bash +$ kubectl get proxysql -n alert-proxysql proxysql-alert -w +NAME VERSION STATUS AGE +proxysql-alert 2.3.2-debian Ready 24m +``` + +Once the phase returns to `Ready`, Prometheus marks the alert **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `proxysql-alert` instance in the `alert-proxysql` namespace via the PromQL label filters `job="proxysql-alert-stats"` / `namespace="alert-proxysql"` (database/cluster groups), or `app="proxysql-alert"` / `namespace="alert-proxysql"` (provisioner/opsManager groups). + +The tables below list every alert **defined by the chart's `values.yaml`**. As documented in the chart-bug callout above, only `ProxySQLCLusterSyncFailure`, `KubeDBProxySQLPhaseNotReady`, `KubeDBProxySQLPhaseCritical`, `KubeDBProxySQLOpsRequestStatusProgressingToLong`, and `KubeDBProxySQLOpsRequestFailed` are actually loaded in this cluster's live `PrometheusRule`; the rest of the `Database Group` table describes what `helm template` renders for this chart version but is **not currently active** until the release is upgraded/reconciled. + +### Database Group + +Fired based on live metrics from the ProxySQL container's built-in metrics endpoint. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ProxySQLInstanceDown` | critical | instant | `proxysql_uptime_seconds_total` reads `0` — the ProxySQL process is down. | +| `ProxySQLServiceDown` | critical | instant | The summed uptime across the service is `0` — no ProxySQL replica is answering. | +| `ProxySQLTooManyConnections` | warning | 2m | Client connections exceed 80% of `proxysql_mysql_max_connections`. | +| `ProxySQLHighThreadsRunning` | warning | 2m | More than 60 worker threads are running — the proxy may be saturated. | +| `ProxySQLSlowQueries` | warning | 2m | `proxysql_slow_queries_total` increased in the last minute. | +| `ProxySQLRestarted` | warning | instant | `proxysql_uptime_seconds_total` is below 60s — the process restarted recently. | +| `ProxySQLHighQPS` | critical | instant | Query rate exceeds 1000 QPS. | +| `ProxySQLHighIncomingBytes` | critical | instant | Frontend-received byte rate exceeds 1 MB/s. | +| `ProxySQLHighOutgoingBytes` | critical | instant | Frontend-sent byte rate exceeds 1 MB/s. | + +### Cluster Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ProxySQLCLusterSyncFailure` | warning | 5m | `proxysql_cluster_syn_conflict_total` rate exceeds `0.1/s` — ProxySQL cluster nodes are failing to sync config. **This is the only rule currently loaded live in this cluster, under a group mislabeled `proxysql.database...` — see the chart-bug callout above.** | + +### Provisioner Group + +Monitors the KubeDB operator's view of the ProxySQL resource phase (sourced from Panopticon, not the ProxySQL metrics endpoint). **Loaded live.** + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBProxySQLPhaseNotReady` | critical | 1m | KubeDB marked the ProxySQL resource `NotReady` — operator cannot reach a healthy instance. | +| `KubeDBProxySQLPhaseCritical` | warning | 15m | ProxySQL is degraded but not fully unavailable. | + +### OpsManager Group + +Tracks `ProxySQLOpsRequest` lifecycle during upgrades, scaling, and reconfiguration. **Loaded live** (except `opsRequestOnProgress`, whose `info` severity is filtered out by the chart's `enabled: warning` group gate). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBProxySQLOpsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes — likely stuck. | +| `KubeDBProxySQLOpsRequestFailed` | critical | instant | An ops request failed — check the `ProxySQLOpsRequest` object for the error. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + proxysqlTooManyConnections: + enabled: true + duration: "5m" + val: 90 # fire at 90% instead of the default 80% + severity: warning + cluster: + enabled: "none" # disable the cluster-sync alert +``` + +```bash +$ helm upgrade proxysql-alert oci://ghcr.io/appscode-charts/proxysql-alerts \ + -n alert-proxysql \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +> Since the currently-installed release predates the chart's present content (see the chart-bug callout above), any `helm upgrade` — customised or not — will also pull in the full, correctly-split `database`/`cluster` rule groups. + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the proxysql-alerts release (PrometheusRule) +$ helm uninstall proxysql-alert -n alert-proxysql + +# Remove the ProxySQL instance +$ kubectl delete proxysql -n alert-proxysql proxysql-alert + +# Remove the MySQL backend +$ kubectl delete mysql -n alert-proxysql my-group-alert + +# Delete namespace +$ kubectl delete ns alert-proxysql +``` + +## Next Steps + +- Monitor your ProxySQL instance with KubeDB using [built-in Prometheus](/docs/guides/proxysql/monitoring/builtin-prometheus/index.md). +- Monitor your ProxySQL instance with KubeDB using [Prometheus operator](/docs/guides/proxysql/monitoring/prometheus-operator/index.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/qdrant/monitoring/alerting.md b/docs/guides/qdrant/monitoring/alerting.md new file mode 100644 index 0000000000..9b4ac5b162 --- /dev/null +++ b/docs/guides/qdrant/monitoring/alerting.md @@ -0,0 +1,419 @@ +--- +title: Qdrant Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: qd-monitoring-alerting + name: Alerting + parent: qdrant-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Qdrant Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Qdrant instance using the `qdrant-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. See the [Grafana Dashboard](grafana-dashboard.md#configuration) guide for how to deploy kube-prometheus-stack if you don't have it yet. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/qdrant/monitoring/overview.md). + +* For dashboards and visualisation, see [Grafana Dashboard](grafana-dashboard.md) for Qdrant. Note that, at the time of writing, the `kubedb-grafana-dashboards` chart does not yet ship dedicated Qdrant dashboards — the link is provided for consistency with other KubeDB database guides and will render dashboards once they are added upstream. + +> Note: YAML files used in this tutorial are stored in [docs/examples/qdrant](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/qdrant) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Qdrant with metrics served natively by Qdrant itself on its API port (`6333`) — no separate exporter sidecar is required. +- **ServiceMonitor** (named `{qdrant-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the metrics endpoint every 10 seconds. It is pre-configured to send a Bearer token (read from the `{qdrant-name}-auth` Secret's `api-key` key) with every scrape request, since Qdrant's `/metrics` endpoint requires authentication. +- **PrometheusRule** is created by the `qdrant-alerts` chart and contains all Qdrant alert definitions grouped by concern: database health and provisioner. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy Qdrant with Monitoring Enabled + +At first, let's deploy a Qdrant database with monitoring enabled. Below is the Qdrant object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Qdrant +metadata: + name: qd-alert-demo + namespace: demo +spec: + version: "1.17.0" + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + exporter: + port: 6333 + serviceMonitor: + interval: 10s + labels: + release: prometheus + deletionPolicy: WipeOut +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.exporter.port: 6333` tells KubeDB which port to scrape metrics from — Qdrant's own API port. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the Qdrant resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/qdrant/monitoring/qdrant-monitoring.yaml +qdrant.kubedb.com/qd-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get qdrant -n demo qd-alert-demo +NAME VERSION STATUS AGE +qd-alert-demo 1.17.0 Ready 37s +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n demo --selector="app.kubernetes.io/instance=qd-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +qd-alert-demo ClusterIP 10.43.182.24 6333/TCP,6334/TCP 43s +qd-alert-demo-pods ClusterIP None 6335/TCP 43s +qd-alert-demo-stats ClusterIP 10.43.199.143 6333/TCP 43s +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n demo +NAME AGE +qd-alert-demo-stats 40s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n demo qd-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +Because Qdrant's `/metrics` endpoint requires authentication, KubeDB pre-wires the `ServiceMonitor`'s scrape endpoint with a Bearer token sourced from the `{qdrant-name}-auth` Secret: + +```bash +$ kubectl get servicemonitor -n demo qd-alert-demo-stats -o jsonpath='{.spec.endpoints[0].authorization}' +{"credentials":{"key":"api-key","name":"qd-alert-demo-auth"},"type":"Bearer"} +``` + +No manual credential wiring is required — Prometheus authenticates automatically. + +--- + +## Step 1 — Install qdrant-alerts + +The `qdrant-alerts` chart creates a `PrometheusRule` resource containing all Qdrant alert definitions grouped by concern: database health and provisioner. + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the Qdrant object's name (`qd-alert-demo`) for the rules to be correctly scoped to this instance. Do **not** use `--set metadata.release.name=...` to try to achieve this — that values field is a no-op for query scoping; only the actual Helm release name (the first positional argument to `helm upgrade -i`) is used by the chart's templates. + +The chart's default label is `release: prometheus` already, matching the Prometheus `ruleSelector` on this cluster, but we set it explicitly below for clarity and portability. + +### Install + +```bash +$ helm upgrade -i qd-alert-demo oci://ghcr.io/appscode-charts/qdrant-alerts \ + -n demo \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `qd-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`job="qd-alert-demo-stats"`) | +| `-n demo` | `demo` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n demo +NAME AGE +qd-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n demo qd-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=qdrant`. + +

+ Prometheus Rule Health +

+ +Both the `qdrant.database.demo.qd-alert-demo.rules` and `qdrant.provisioner.demo.qd-alert-demo.rules` groups are visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Qdrant alert definitions every 30 seconds. + +--- + +## Verify End-to-End + +### 1. Check the metrics endpoint + +Qdrant serves its own metrics; there is no separate exporter sidecar to check. Because the endpoint requires a Bearer token, fetch the API key from the auth Secret first. + +```bash +$ kubectl get secret -n demo qd-alert-demo-auth -o jsonpath='{.data.api-key}' | base64 -d +38XLVOctmGr5lQzT +``` + +```bash +$ kubectl port-forward -n demo svc/qd-alert-demo-stats 6333:6333 & +$ curl -s -H "Authorization: Bearer 38XLVOctmGr5lQzT" localhost:6333/metrics | head -5 +# HELP app_info information about qdrant server +# TYPE app_info gauge +app_info{name="qdrant",version="1.17.0"} 1 +# HELP collections_total number of collections +# TYPE collections_total gauge +``` + +A successful `200` response with Qdrant metrics confirms Prometheus can scrape this endpoint using the same Bearer token wired into the `ServiceMonitor`. + +### 2. Check the Prometheus target is UP + +Open `http://localhost:9090/targets?search=qd-alert-demo`. + +

+ Prometheus Target UP +

+ +The target `serviceMonitor/demo/qd-alert-demo-stats/0` shows **UP**, confirming metrics are being scraped from `qd-alert-demo-0` in the `demo` namespace. + +### 3. Confirm Qdrant alerts + +Open `http://localhost:9090/alerts?search=qdrant` to see the Qdrant alert groups. + +

+ Prometheus Alerts +

+ +Most rules in the `qdrant.database` group show **INACTIVE**, meaning the corresponding thresholds are not breached. On the live cluster used to capture this screenshot, `DiskUsageHigh` was genuinely **FIRING** — the `local-path` StorageClass reports the *underlying node filesystem's* usage/capacity rather than the PVC's logical 1Gi request, and the shared demo node happened to be above the 80% threshold at the time. This is a real, useful illustration of the alert doing its job: it is not specific to Qdrant, but to any workload using `local-path` on a node with high disk utilization. In production, prefer a StorageClass that enforces real capacity accounting (e.g. most CSI drivers) if you rely on `DiskUsageHigh`/`DiskAlmostFull` for capacity planning. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy Qdrant instance, only pre-existing environmental alerts (such as `DiskUsageHigh`, if applicable to your node) will be listed here — no Qdrant-availability alert should be present. + +--- + +## Simulating a Firing Alert + +This section walks through deliberately triggering the `QdrantInstanceDown` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +### 1. Stop the Qdrant process + +The Qdrant pod runs a single container (no exporter sidecar), and its main process runs as PID 1. A plain `kill 1` isn't available since the image ships no `kill` binary, so use the `bash` built-in instead. A single `SIGTERM` restarts fast enough (well under one 10s scrape interval) that Prometheus may never observe a failed scrape, so repeat the signal a few times to keep the container down for longer than the alert's `for: 30s` window: + +```bash +$ for i in $(seq 1 15); do + kubectl exec -n demo qd-alert-demo-0 -c qdrant -- bash -c 'kill -TERM 1' + sleep 3 + done +``` + +This repeatedly restarts the container faster than it can finish starting up, eventually pushing it into `CrashLoopBackOff`, which keeps the metrics endpoint unreachable for well over 30 seconds. + +```bash +$ kubectl get pod -n demo qd-alert-demo-0 +NAME READY STATUS RESTARTS AGE +qd-alert-demo-0 0/1 Completed 3 (30s ago) 56m +``` + +Wait 30–60 seconds for the next Prometheus scrape cycle (configured at 10s) and rule-evaluation cycle (30s) to register the failure. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=qdrant`. + +

+ Prometheus Alerts — QdrantInstanceDown Firing +

+ +Both `QdrantInstanceDown` (the scrape-target-down alert, `for: 30s`) and `QdrantPhaseCritical` (driven by the KubeDB operator's own view of the resource phase, `for: 1m`) move to **FIRING** once their respective durations elapse. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — QdrantInstanceDown Firing +

+ +AlertManager shows the `QdrantInstanceDown` alert. The alert card displays: + +- **Severity**: `critical` +- **Instance**: `qd-alert-demo-0` in the `alert-qdrant` namespace +- **job**: `qd-alert-demo-stats` +- **Started**: timestamp when the alert first fired + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore Qdrant + +Delete the pod so KubeDB recreates it cleanly. + +```bash +$ kubectl delete pod -n demo qd-alert-demo-0 +``` + +Once the metrics target returns to `up`, Prometheus marks the alerts **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `qd-alert-demo` instance in the `demo` namespace via PromQL label filters such as `job="qd-alert-demo-stats"`, `app="qd-alert-demo"`, and `namespace="demo"`. + +### Database Group + +Fired based on live metrics from Qdrant itself, the KubeDB operator's status reporting, and node-level cAdvisor/kubelet metrics. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `QdrantInstanceDown` | critical | 30s | The database is in NotReady phase — not accepting connections; read/write operations are failing. | +| `QdrantPhaseCritical` | warning | 1m | The database is in Critical phase — one or more nodes are experiencing issues but the database is still operational. | +| `QdrantRestarted` | warning | 1m | The database service restarted recently (uptime under 180s). | +| `QdrantHighCPUUsage` | warning | 1m | Database CPU usage (ratio of usage to limits) exceeds 80%. | +| `QdrantHighMemoryUsage` | warning | 1m | Database memory usage (ratio of working set to limits) exceeds 80%. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80% — storage is running low and may need expansion soon. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95% — storage is critically low; immediate action is required to avoid data loss. | +| `QdrantHighPendingOperations` | critical | 5m | The number of pending operations on a pod exceeds 10 — a backlog is building up. | +| `QdrantGrpcResponsesFailHigh` | critical | 5m | More than 5 failed gRPC responses in the last 5 minutes on a pod. | +| `QdrantRestResponsesFailHigh` | critical | 5m | More than 5 failed REST responses in the last 5 minutes on a pod. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Qdrant resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `appPhaseNotReady` (alert name `KubeDBQdrantPhaseNotReady`) | critical | 1m | KubeDB marked the Qdrant resource `NotReady` — the database is not accepting connections and requires attention. | +| `appPhaseCritical` (alert name `KubeDBQdrantPhaseCritical`) | warning | 15m | The instance is in a degraded/critical phase — one or more nodes are experiencing issues but the database is still operational. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + qdrantHighCPUUsage: + enabled: true + duration: "5m" + val: 90 # fire at 90% CPU instead of the default 80% + severity: warning + diskUsageHigh: + enabled: true + val: 90 # raise the disk-usage warning threshold to 90% + duration: "1m" + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade qd-alert-demo oci://ghcr.io/appscode-charts/qdrant-alerts \ + -n demo \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the qdrant-alerts release +$ helm uninstall qd-alert-demo -n demo + +# Remove the Qdrant instance +$ kubectl delete qdrant -n demo qd-alert-demo + +# Delete namespace +$ kubectl delete ns demo +``` + +## Next Steps + +- Monitor your Qdrant database with KubeDB using [Prometheus operator](/docs/guides/qdrant/monitoring/using-prometheus-operator.md). +- Visualise Qdrant metrics with [Grafana Dashboard](grafana-dashboard.md). +- Detail concepts of [Qdrant object](/docs/guides/qdrant/concepts/qdrant.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/rabbitmq/monitoring/alerting.md b/docs/guides/rabbitmq/monitoring/alerting.md new file mode 100644 index 0000000000..9f2f49c6cd --- /dev/null +++ b/docs/guides/rabbitmq/monitoring/alerting.md @@ -0,0 +1,433 @@ +--- +title: RabbitMQ Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: rm-monitoring-alerting + name: Alerting + parent: rm-monitoring-guides + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# RabbitMQ Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed RabbitMQ instance using the `rabbitmq-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-rabbitmq` namespace: + + ```bash + $ kubectl create ns alert-rabbitmq + namespace/alert-rabbitmq created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. See the [Grafana Dashboard](grafana-dashboard.md) guide for how to deploy kube-prometheus-stack if you don't have it yet. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/rabbitmq/monitoring/overview.md). + +* For dashboards and visualisation, see [Grafana Dashboard](grafana-dashboard.md) for RabbitMQ. + +> Note: YAML files used in this tutorial are stored in [docs/examples/rabbitmq](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/rabbitmq) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys RabbitMQ with the built-in `rabbitmq_prometheus` plugin enabled, which serves metrics directly from the `rabbitmq` container on port `15692` — unlike some other databases, RabbitMQ needs no separate exporter sidecar. +- **ServiceMonitor** (named `{rabbitmq-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the metrics endpoint every 10 seconds. +- **KubeDB operator (panopticon)** also exposes the CR's own status as a metric, `kubedb_com_rabbitmq_status_phase`. The `RabbitMQDown` and provisioner-group alerts key off this metric instead of the database's own stats endpoint, so they fire purely based on what KubeDB itself observes about the resource — even if the metrics scrape target is otherwise healthy. +- **PrometheusRule** is created by the `rabbitmq-alerts` chart and contains RabbitMQ alert definitions grouped by concern: database health and provisioner. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy RabbitMQ with Monitoring Enabled + +At first, let's deploy a RabbitMQ database with monitoring enabled. Below is the RabbitMQ object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: RabbitMQ +metadata: + name: rmq-alert-demo + namespace: alert-rabbitmq +spec: + version: "4.2.4" + replicas: 3 + deletionPolicy: WipeOut + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the RabbitMQ resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/rabbitmq/monitoring/rmq-alert-demo.yaml +rabbitmq.kubedb.com/rmq-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get rabbitmq -n alert-rabbitmq rmq-alert-demo +NAME VERSION STATUS AGE +rmq-alert-demo 4.2.4 Ready 5m +``` + +KubeDB brings up 3 pods, one per RabbitMQ node: + +```bash +$ kubectl get pods -n alert-rabbitmq +NAME READY STATUS RESTARTS AGE +rmq-alert-demo-0 1/1 Running 0 5m +rmq-alert-demo-1 1/1 Running 0 4m +rmq-alert-demo-2 1/1 Running 0 4m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-rabbitmq --selector="app.kubernetes.io/instance=rmq-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rmq-alert-demo ClusterIP 10.43.174.187 5672/TCP,1883/TCP,61613/TCP,15675/TCP,15674/TCP 59s +rmq-alert-demo-dashboard ClusterIP 10.43.236.58 15672/TCP 59s +rmq-alert-demo-pods ClusterIP None 4369/TCP,25672/TCP 59s +rmq-alert-demo-stats ClusterIP 10.43.128.229 15692/TCP 59s +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-rabbitmq +NAME AGE +rmq-alert-demo-stats 55s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-rabbitmq rmq-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install rabbitmq-alerts + +The `rabbitmq-alerts` chart creates a `PrometheusRule` resource containing RabbitMQ alert definitions grouped by concern: database health and provisioner. + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the RabbitMQ object's name (`rmq-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i rmq-alert-demo appscode/rabbitmq-alerts \ + -n alert-rabbitmq \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `rmq-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`job="rmq-alert-demo-stats"`, `app="rmq-alert-demo"`) | +| `-n alert-rabbitmq` | `alert-rabbitmq` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-rabbitmq +NAME AGE +rmq-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-rabbitmq rmq-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=rabbitmq`. + +

+ Prometheus Rule Health +

+ +The `rabbitmq.database.alert-rabbitmq.rmq-alert-demo.rules` group is visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the RabbitMQ alert definitions every 30 seconds. + +--- + +## Verify End-to-End + +### 1. Check the metrics endpoint + +Because RabbitMQ's Prometheus plugin runs inside the `rabbitmq` container itself, there is no separate `exporter` container to check — query the plugin's own endpoint directly. + +```bash +$ kubectl exec -n alert-rabbitmq rmq-alert-demo-0 -c rabbitmq -- \ + + wget -qO- http://127.0.0.1:15692/metrics | grep rabbitmq_identity_info +# TYPE rabbitmq_identity_info untyped +# HELP rabbitmq_identity_info RabbitMQ node & cluster identity info +rabbitmq_identity_info{rabbitmq_node="rabbit@rmq-alert-demo-0.rmq-alert-demo-pods.alert-rabbitmq",rabbitmq_cluster="rmq-alert-demo",rabbitmq_cluster_permanent_id="rabbitmq-cluster-id-lcFtUYEzj3-MLTpL-uEbcg",rabbitmq_endpoint="aggregated"} 1 +``` + +### 2. Check the Prometheus target is UP + +Open `http://localhost:9090/targets`. + +

+ Prometheus Target UP +

+ +The target `serviceMonitor/alert-rabbitmq/rmq-alert-demo-stats/0` shows **3 / 3 up**, confirming metrics are being scraped from all three pods — `rmq-alert-demo-0`, `rmq-alert-demo-1`, and `rmq-alert-demo-2` — in the `alert-rabbitmq` namespace. + +### 3. Confirm the RabbitMQ alerts are inactive + +Open `http://localhost:9090/alerts?search=rabbitmq` to see the RabbitMQ alert groups. + +

+ Prometheus Alerts +

+ +All 11 rules in the `rabbitmq.database` group and both rules in the `rabbitmq.provisioner` group show **INACTIVE**, meaning the 3-node cluster is healthy and no thresholds are breached. + +> Note: this cluster uses the `local-path` storage class, whose PVCs are just directories on the node's root filesystem rather than isolated volumes — so `DiskUsageHigh`/`DiskAlmostFull` (`kubelet_volume_stats_used_bytes`) can occasionally read the **node's actual disk usage** instead of the small demo volume's own usage, showing PENDING/FIRING even though the RabbitMQ data volume itself is nearly empty. Not observed in this run, but worth knowing if your own result differs from the screenshot above. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. **PENDING** rules have not yet fired — only alerts that cross into **FIRING** are forwarded to AlertManager — so with a healthy RabbitMQ instance no alerts for `rmq-alert-demo` are listed here yet. + +--- + +## Simulating a Firing Alert + +The previous section confirmed that the RabbitMQ alerts are healthy. This section walks through deliberately triggering the `RabbitMQPhaseCritical` alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +On a 3-node cluster, crashing a single pod degrades the cluster rather than taking it fully down — the KubeDB operator moves the resource's `status.phase` to `Critical` (one or more nodes unhealthy, but the remaining nodes keep serving) rather than `NotReady` (which needs a majority/all of the nodes down). `RabbitMQPhaseCritical` is therefore the realistic alert to demonstrate here; `RabbitMQDown`/`KubeDBRabbitMQPhaseNotReady` would need all 3 nodes crashed simultaneously and held down. + +### 1. Crash one RabbitMQ node repeatedly + +Kill the RabbitMQ process inside `rmq-alert-demo-0`. A lone `kill 1` restarts fast enough that the container becomes `Ready` again before KubeDB's health check can even observe the outage, and `RabbitMQPhaseCritical` needs the phase held at `Critical` for a full `for: 3m` — so keep the pod crash-looping for several minutes, not just a handful of kills. + +```bash +$ end=$(( $(date +%s) + 240 )) + while [ $(date +%s) -lt $end ]; do + kubectl exec -n alert-rabbitmq rmq-alert-demo-0 -c rabbitmq -- kill 1 >/dev/null 2>&1 + sleep 5 + done +``` + +Watch the CR phase move from `Ready` to `Critical`: + +```bash +$ kubectl get rabbitmq -n alert-rabbitmq rmq-alert-demo -o jsonpath='{.status.phase}' +Critical +``` + +`RabbitMQPhaseCritical` keys off `kubedb_com_rabbitmq_status_phase` (a metric emitted by the KubeDB operator itself), so what matters is the CR's `status.phase` staying at `Critical` continuously, not the exporter's own scrape health — if the pod recovers even briefly between kills, the `for: 3m` timer resets and you'll see the rule flip back to **PENDING**. Let the loop run for the full 4 minutes above before checking. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=rabbitmq`. + +

+ Prometheus Alerts — RabbitMQPhaseCritical Firing +

+ +`RabbitMQPhaseCritical` moves from **INACTIVE** to **FIRING** once its `for: 3m` window elapses with the phase held at `Critical`, while the rest of the `rabbitmq.database` group stays **INACTIVE**. The provisioner-group `KubeDBRabbitMQPhaseNotReady` alert only reaches **PENDING** in this scenario — it needs the operator to view the resource as fully `NotReady`, which a single crashed node in a 3-node cluster doesn't trigger. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter=%7Bnamespace%3D%22alert-rabbitmq%22%7D`. + +

+ AlertManager — RabbitMQPhaseCritical Firing +

+ +AlertManager shows the `RabbitMQPhaseCritical` alert. The alert card displays labels including: + +- **alertname**: `RabbitMQPhaseCritical` +- **severity**: `warning` +- **app**: `rmq-alert-demo`, **app_namespace**: `alert-rabbitmq` +- **phase**: `Critical` +- **k8s_kind**: `RabbitMQ` + +Note that the `instance`/`pod`/`job` labels on this alert point at the KubeDB operator's **panopticon** component (`job="panopticon"`), not at the RabbitMQ pod itself — because this alert is derived from the operator's own status metric rather than from the database's stats endpoint. + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore RabbitMQ + +Delete the pod so KubeDB recreates it cleanly. + +```bash +$ kubectl delete pod -n alert-rabbitmq rmq-alert-demo-0 +pod "rmq-alert-demo-0" deleted +``` + +Once `status.phase` returns to `Ready`, Prometheus marks both alerts **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `rmq-alert-demo` instance in the `alert-rabbitmq` namespace via the PromQL label filters `job="rmq-alert-demo-stats"` / `app="rmq-alert-demo"` and `namespace="alert-rabbitmq"`. + +### Database Group + +Fired based on live metrics from RabbitMQ's `rabbitmq_prometheus` plugin, plus the two persistent-volume rules and the two KubeDB status-phase rules. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `RabbitmqFileDescriptorsNearLimit` | warning | 30s | More than 80% of the node's file descriptor limit is in use — at 100%, new connections will be refused and disk writes may fail. | +| `RabbitmqQueueIsGrowing` | warning | 30s | A queue's message count has been steadily increasing over the last 10 minutes — consumers may not be keeping up with publishers. | +| `RabbitmqUnroutableMessages` | warning | 30s | Messages published to an exchange could not be routed to any queue in the last 5 minutes — check your exchange/queue bindings. | +| `RabbitmqTCPSocketsNearLimit` | warning | 30s | More than 80% of the node's TCP socket limit is in use — at 100%, new connections will be refused. | +| `RabbitmqLowDiskWatermarkPredicted` | warning | 30s | Based on the last 24h trend, free disk space is predicted to drop below the configured watermark within 24 hours, which would block all publishers cluster-wide. | +| `RabbitmqInsufficientEstablishedErlangDistributionLinks` | warning | 30s | Fewer Erlang distribution links than expected for a full-mesh cluster are established — indicates partial inter-node connectivity issues. | +| `RabbitmqHighConnectionChurn` | warning | 30s | More than 10% of total connections were opened/closed per second over the last 5 minutes — client connections are short-lived instead of long-lived. | +| `RabbitMQPhaseCritical` | warning | 3m | KubeDB reports the database in Critical phase — one or more nodes are down, but read/write is not yet hampered. | +| `RabbitMQDown` | critical | 30s | KubeDB reports the database in NotReady phase — the cluster is not accepting connections and read/write is failing. | +| `DiskUsageHigh` | warning | 1m | The RabbitMQ data volume (PVC) is more than 80% full. | +| `DiskAlmostFull` | critical | 1m | The RabbitMQ data volume (PVC) is more than 95% full. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the RabbitMQ resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBRabbitMQPhaseNotReady` | critical | 1m | KubeDB marked the RabbitMQ resource `NotReady` — operator cannot reach the database. | +| `KubeDBRabbitMQPhaseCritical` | warning | 15m | The instance is in a degraded/critical phase. | + +### OpsManager Group + +The chart's `values.yaml` also ships an `opsManager` group (`opsRequestOnProgress`, `opsRequestStatusProgressingToLong`, `opsRequestFailed`), intended to track `RabbitMQOpsRequest` lifecycle during upgrades, scaling, and reconfiguration — the same pattern used by other `*-alerts` charts. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `opsRequestOnProgress` | info | instant | An ops request is currently in progress. | +| `opsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes — likely stuck. | +| `opsRequestFailed` | critical | instant | An ops request failed — check the `RabbitMQOpsRequest` object for the error. | + +> **Verified note:** in chart version `v2026.7.14`, `templates/alert.yaml` only renders the `database` and `provisioner` groups into the `PrometheusRule` — `kubectl get prometheusrule -n alert-rabbitmq rmq-alert-demo -o yaml` shows no `opsManager` rules, even though `opsManager.enabled` is `warning` by default in `values.yaml`. If you rely on ops-request alerting, verify with `kubectl get prometheusrule -n -o yaml` after installing, and check for a newer chart version if you need these rules. + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + rabbitmqHighConnectionChurn: + enabled: true + duration: "2m" + severity: warning + diskUsageHigh: + enabled: true + val: 90 # fire at 90% disk usage instead of the default 80% + duration: "5m" + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade rmq-alert-demo appscode/rabbitmq-alerts \ + -n alert-rabbitmq \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the rabbitmq-alerts release +$ helm uninstall rmq-alert-demo -n alert-rabbitmq + +# Remove the RabbitMQ instance +$ kubectl delete rabbitmq -n alert-rabbitmq rmq-alert-demo + +# Delete namespace +$ kubectl delete ns alert-rabbitmq +``` + +## Next Steps + +- Monitor your RabbitMQ database with KubeDB using [builtin Prometheus](/docs/guides/rabbitmq/monitoring/using-builtin-prometheus.md). +- Monitor your RabbitMQ database with KubeDB using [Prometheus operator](/docs/guides/rabbitmq/monitoring/using-prometheus-operator.md). +- Visualise RabbitMQ metrics with [Grafana Dashboard](grafana-dashboard.md). +- Detail concepts of [RabbitMQ object](/docs/guides/rabbitmq/concepts/rabbitmq.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/redis/monitoring/alerting.md b/docs/guides/redis/monitoring/alerting.md new file mode 100644 index 0000000000..ed376e92ae --- /dev/null +++ b/docs/guides/redis/monitoring/alerting.md @@ -0,0 +1,442 @@ +--- +title: Redis Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: rd-monitoring-alerting + name: Alerting + parent: rd-monitoring-redis + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Redis Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Redis instance using the `redis-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-redis` namespace: + + ```bash + $ kubectl create ns alert-redis + namespace/alert-redis created + ``` + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. See the [Grafana Dashboard](grafana-dashboard.md#configuration) guide for how to deploy kube-prometheus-stack if you don't have it yet. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/redis/monitoring/overview.md). + +* For dashboards and visualisation, see [Grafana Dashboard](grafana-dashboard.md) for Redis. + +> Note: YAML files used in this tutorial are stored in [docs/examples/redis](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/redis) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Redis with a built-in `redis_exporter` sidecar that exposes metrics on port `56790`. +- **ServiceMonitor** (named `{redis-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `redis-alerts` chart and contains all Redis alert definitions grouped by concern: database health, provisioner, ops-manager, and backups (Stash and KubeStash). +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy Redis with Monitoring Enabled + +At first, let's deploy a Redis database with monitoring enabled. Below is the Redis object we are going to create. + +```yaml +apiVersion: kubedb.com/v1 +kind: Redis +metadata: + name: rd-alert-demo + namespace: alert-redis +spec: + version: "6.0.20" + deletionPolicy: WipeOut + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus +``` + +Here, + +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the Redis resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/redis/monitoring/rd-alert-demo.yaml +redis.kubedb.com/rd-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get redis -n alert-redis rd-alert-demo +NAME VERSION STATUS AGE +rd-alert-demo 6.0.20 Ready 62m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-redis --selector="app.kubernetes.io/instance=rd-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +rd-alert-demo ClusterIP 10.43.236.232 6379/TCP 62m +rd-alert-demo-pods ClusterIP None 6379/TCP,16379/TCP 62m +rd-alert-demo-stats ClusterIP 10.43.191.236 56790/TCP 62m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-redis +NAME AGE +rd-alert-demo-stats 62m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-redis rd-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install redis-alerts + +The `redis-alerts` chart creates a `PrometheusRule` resource containing all Redis alert definitions grouped by concern: database health, provisioner, ops-manager, and backups (Stash / KubeStash). + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the Redis object's name (`rd-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i rd-alert-demo oci://ghcr.io/appscode-charts/redis-alerts \ + -n alert-redis \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `rd-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`job="rd-alert-demo-stats"`) | +| `-n alert-redis` | `alert-redis` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-redis +NAME AGE +rd-alert-demo 60m +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n alert-redis rd-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=redis`. + +

+ Prometheus Rule Health +

+ +The `redis.database.alert-redis.rd-alert-demo.rules` group (and the accompanying `redis.kubeStash.alert-redis.rd-alert-demo.rules` group) is visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the Redis alert definitions every 30 seconds. + +--- + +## Verify End-to-End + +### 1. Check the exporter is running + +The `exporter` sidecar inside the Redis pod serves metrics at `:56790/metrics`. A value of `redis_up 1` confirms the exporter can reach Redis. + +```bash +$ kubectl exec -n alert-redis rd-alert-demo-0 -c exporter -- \ + wget -qO- localhost:56790/metrics | grep redis_up +# HELP redis_up Information about the Redis instance +# TYPE redis_up gauge +redis_up 1 +# HELP redis_uptime_in_seconds uptime_in_seconds metric +# TYPE redis_uptime_in_seconds gauge +redis_uptime_in_seconds 2591 + +``` + +### 2. Check the Prometheus target is UP + +Open `http://localhost:9090/targets?search=rd-alert-demo-stats`. Prometheus discovers more than 20 scrape pools on this cluster, so it will ask you to pick one from the dropdown — select `serviceMonitor/alert-redis/rd-alert-demo-stats/0`. + +

+ Prometheus Target UP +

+ +The target `serviceMonitor/alert-redis/rd-alert-demo-stats/0` shows **UP**, confirming metrics are being scraped from `rd-alert-demo-0` in the `alert-redis` namespace. + +### 3. Confirm all Redis alerts are inactive + +Open `http://localhost:9090/alerts?search=redis` to see the Redis alert groups. + +

+ Prometheus Alerts — All Inactive +

+ +All 7 rules in the `redis.database` group show **INACTIVE (7)**, meaning the database is healthy and no thresholds are breached. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy Redis instance, no alerts for `rd-alert-demo` will be listed here. + +

+ AlertManager — No Active Alerts +

+ +--- + +## Simulating a Firing Alert + +The previous section confirmed that all alerts are **INACTIVE** while the database is healthy. This section walks through deliberately triggering the `RedisDown` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +Unlike some other database exporters, the Redis exporter runs as a **separate sidecar container** that keeps running fine even if the main `redis` container crashes and gets restarted by Kubernetes. Because `RedisDown` requires the outage to persist for `for: 2m` (not instant), and Kubernetes tends to restart a crashed container within a few seconds, a single `redis-cli shutdown` is not reliable — confirmed live: one isolated shutdown attempt produced zero pod restarts over what should have been a full window (the container came back before the next scrape landed), while a **sustained** loop reliably kept it down and fired the alert on the very next try. Don't trust a one-shot kill for this alert; always use the loop below for the full duration. + +### 1. Stop the Redis process repeatedly + +Shut down the `redis` process inside the pod. This crashes the main container so the `exporter` sidecar can no longer reach it and reports `redis_up 0` on the next scrape, while Kubernetes restarts the crashed container in the background. + +```bash +$ kubectl exec -n alert-redis rd-alert-demo-0 -c redis -- redis-cli shutdown nosave +``` + +Because Kubernetes restarts the container quickly, repeat this command every second for at least two and a half minutes to keep Redis down continuously long enough for the `for: 2m` window on `RedisDown` to be satisfied — confirmed live at exactly this cadence: + +```bash +$ end=$(( $(date +%s) + 150 )) +while [ $(date +%s) -lt $end ]; do + kubectl exec -n alert-redis rd-alert-demo-0 -c redis -- redis-cli shutdown nosave >/dev/null 2>&1 + sleep 1 +done +``` + +Run this in the background (or a separate terminal) and let it run for the full 150 seconds before moving to the next step — stopping early risks landing back in the same "restarted before the next scrape" gap described above. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=redis`. + +

+ Prometheus Alerts — RedisDown Firing +

+ +`RedisDown` moves from **INACTIVE** to **FIRING** once the `redis_up == 0` condition has held continuously for the configured `for: 2m` duration. `RedisMissingMaster` fires alongside it, since with no master node visible the cluster is also missing a master. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter=%7Bapp_namespace%3D%22alert-redis%22%7D`. + +> Note: like several other `*-alerts` charts in this project (e.g. Cassandra), Redis's operator-sourced alerts carry `app_namespace` rather than a plain `namespace` label — filtering on `namespace="alert-redis"` silently returns nothing (no error, just an empty "No alert groups found" that looks identical to "healthy") even while alerts are actively firing. Always filter/group on `app_namespace` here. + +

+ AlertManager — RedisDown Firing +

+ +AlertManager shows `RedisDown` and `RedisMissingMaster` together (confirmed live — dropping to zero visible master nodes trips both at once), alongside `KubeDBRedisPhaseNotReady` (the KubeDB operator's own view that the pod isn't ready) and, if your cluster happens to have real disk pressure at the time, possibly `DiskUsageHigh` too — that one is unrelated to this simulation and reflects genuine node-disk usage, not a side effect of the kill loop. Each alert card displays: + +- **Severity**: `critical` (`warning` for `DiskUsageHigh`) +- **pod**: `rd-alert-demo-0` in the `alert-redis` namespace +- **job**: `rd-alert-demo-stats` +- **Started**: timestamp when the alert first fired + +AlertManager routes these alerts to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alerts are visible here but silently dropped. + +### 4. Restore Redis + +Let the loop from step 1 finish (or stop it early) — the container's own restart policy brings `redis` back up on its own; no manual pod deletion is needed. + +```bash +$ kubectl get pod -n alert-redis rd-alert-demo-0 +NAME READY STATUS RESTARTS AGE +rd-alert-demo-0 2/2 Running 17 2d16h +``` + +If it's still cycling through `CrashLoopBackOff` a while after the loop has stopped, Kubernetes' exponential backoff is just catching up — wait a bit longer before concluding something's wrong. Once `redis_up` returns to `1` continuously, Prometheus marks the alert **INACTIVE** again and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `rd-alert-demo` instance in the `alert-redis` namespace via the PromQL label filters `job="rd-alert-demo-stats"` and `namespace="alert-redis"`. + +### Database Group + +Fired based on live metrics from the Redis exporter. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `RedisDown` | critical | 2m | Exporter cannot reach Redis — instance is down or crashed. | +| `RedisMissingMaster` | critical | 2m | Redis cluster has fewer nodes marked as master than expected. | +| `RedisTooManyMasters` | critical | 2m | Redis cluster has more nodes marked as master than expected — possible split-brain. | +| `RedisDisconnectedSlaves` | warning | 2m | Redis is not replicating for all slaves — review the replication status. | +| `RedisTooManyConnections` | warning | 2m | More than 80% of `maxclients` are in use — client pool nearing exhaustion. | +| `DiskUsageHigh` | warning | 5m | Persistent volume usage is between 80% and 95% — plan for capacity. | +| `DiskAlmostFull` | critical | 5m | Persistent volume usage has exceeded 95% — the volume is nearly full. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Redis resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBRedisPhaseNotReady` | critical | 1m | KubeDB marked the Redis resource `NotReady` — operator cannot reach the database. | +| `KubeDBRedisPhaseCritical` | warning | 15m | The instance is in a degraded/critical phase. | + +### OpsManager Group + +Tracks `RedisOpsRequest` lifecycle during upgrades, scaling, and reconfiguration. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBRedisOpsRequestOnProgress` | info | instant | A `RedisOpsRequest` is currently in progress. | +| `KubeDBRedisOpsRequestStatusProgressingToLong` | critical | 30m | A `RedisOpsRequest` has been in progress for 30+ minutes — likely stuck. | +| `KubeDBRedisOpsRequestFailed` | critical | instant | A `RedisOpsRequest` failed — check the `RedisOpsRequest` object for the error. | + +### Stash Group + +Tracks Stash-driven backup/restore health for this instance. (You do not need to configure backups to see these rules; they are included in the `PrometheusRule` regardless.) + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `RedisStashBackupSessionFailed` | critical | instant | A Stash backup session failed. | +| `RedisStashRestoreSessionFailed` | critical | instant | A Stash restore session failed. | +| `RedisStashNoBackupSessionForTooLong` | warning | instant | No successful backup session for more than 18000s (5 hours). | +| `RedisStashRepositoryCorrupted` | critical | 5m | The Stash backup repository integrity check failed — repository is corrupted. | +| `RedisStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage size has exceeded 10GB. | +| `RedisStashBackupSessionPeriodTooLong` | warning | instant | A backup session took more than 1800s (30 minutes) to complete. | +| `RedisStashRestoreSessionPeriodTooLong` | warning | instant | A restore session took more than 1800s (30 minutes) to complete. | + +### KubeStash Group + +Tracks KubeStash-driven backup/restore health for this instance. Same semantics as the Stash group above, sourced from KubeStash metrics instead. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `RedisKubeStashBackupSessionFailed` | critical | instant | A KubeStash backup session failed. | +| `RedisKubeStashRestoreSessionFailed` | critical | instant | A KubeStash restore session failed. | +| `RedisKubeStashNoBackupSessionForTooLong` | warning | instant | No successful backup session for more than 18000s (5 hours). | +| `RedisKubeStashRepositoryCorrupted` | critical | 5m | The KubeStash backup repository integrity check failed — repository is corrupted. | +| `RedisKubeStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage size has exceeded 10GB. | +| `RedisKubeStashBackupSessionPeriodTooLong` | warning | instant | A backup session took more than 1800s (30 minutes) to complete. | +| `RedisKubeStashRestoreSessionPeriodTooLong` | warning | instant | A restore session took more than 1800s (30 minutes) to complete. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + redisTooManyConnections: + enabled: true + duration: "5m" + val: 90 # fire at 90% of maxclients instead of the default 80% + severity: warning + opsManager: + enabled: "none" # disable all ops-manager alerts +``` + +```bash +$ helm upgrade rd-alert-demo oci://ghcr.io/appscode-charts/redis-alerts \ + -n alert-redis \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the redis-alerts release +$ helm uninstall rd-alert-demo -n alert-redis + +# Remove the Redis instance +$ kubectl delete redis -n alert-redis rd-alert-demo + +# Delete namespace +$ kubectl delete ns alert-redis +``` + +## Next Steps + +- Monitor your Redis database with KubeDB using [builtin Prometheus](/docs/guides/redis/monitoring/using-builtin-prometheus.md). +- Monitor your Redis database with KubeDB using [Prometheus operator](/docs/guides/redis/monitoring/using-prometheus-operator.md). +- Visualise Redis metrics with [Grafana Dashboard](grafana-dashboard.md). +- Use [private Docker registry](/docs/guides/redis/private-registry/using-private-registry.md) to deploy Redis with KubeDB. +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/singlestore/monitoring/alerting.md b/docs/guides/singlestore/monitoring/alerting.md new file mode 100644 index 0000000000..e46b3e52b0 --- /dev/null +++ b/docs/guides/singlestore/monitoring/alerting.md @@ -0,0 +1,411 @@ +--- +title: SingleStore Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: guides-sdb-monitoring-alerting + name: Alerting + parent: guides-sdb-monitoring + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# SingleStore Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed SingleStore cluster using the `singlestore-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-singlestore` namespace: + + ```bash + $ kubectl create ns alert-singlestore + namespace/alert-singlestore created + ``` + +* SingleStore requires a license. Create a secret with your license before deploying: + + ```bash + $ kubectl create secret generic -n alert-singlestore license-secret \ + --from-literal=username=license \ + --from-literal=password='your-license-key-here' + secret/license-secret created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/singlestore/monitoring/overview/index.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/singlestore](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/singlestore) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys SingleStore without a separate exporter image — metrics are obtained via the `memsql-admin` binary built into the SingleStore container itself, which KubeDB's operator configures automatically when `spec.monitor` is set. +- **ServiceMonitor** (named `{singlestore-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape metrics every 10 seconds. +- **PrometheusRule** is created by the `singlestore-alerts` chart and contains alert definitions grouped by concern: database health, provisioner, and KubeStash backup/restore. +- **Grafana** dashboards for SingleStore are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy SingleStore with Monitoring Enabled + +SingleStore is always deployed as a cluster of `aggregator` and `leaf` nodes. Below is the topology used in this tutorial — one aggregator, two leaves. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Singlestore +metadata: + name: singlestore-alert-demo + namespace: alert-singlestore +spec: + version: "8.9.3" + storageType: Durable + topology: + aggregator: + replicas: 1 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + leaf: + replicas: 2 + storage: + storageClassName: "local-path" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + licenseSecret: + name: license-secret + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, + +- `spec.topology.aggregator` / `spec.topology.leaf` define the aggregator and leaf node groups that make up the SingleStore cluster. The aggregator is a single point of coordination for the cluster — crashing it takes the whole cluster `NotReady`, while a single leaf going down only degrades the cluster (more leaves survive). +- `spec.licenseSecret.name` references the license secret created in [Before You Begin](#before-you-begin). +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the SingleStore resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/singlestore/monitoring/singlestore-alert-demo.yaml +singlestore.kubedb.com/singlestore-alert-demo created +``` + +Wait for the cluster to go into `Ready` state. + +```bash +$ kubectl get singlestore -n alert-singlestore singlestore-alert-demo +NAME VERSION STATUS AGE +singlestore-alert-demo 8.9.3 Ready 5m +``` + +KubeDB brings up one aggregator pod and two leaf pods: + +```bash +$ kubectl get pods -n alert-singlestore +NAME READY STATUS RESTARTS AGE +singlestore-alert-demo-aggregator-0 2/2 Running 0 5m +singlestore-alert-demo-leaf-0 2/2 Running 0 5m +singlestore-alert-demo-leaf-1 2/2 Running 0 4m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-singlestore --selector="app.kubernetes.io/instance=singlestore-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +singlestore-alert-demo ClusterIP 10.43.10.20 3306/TCP 5m +singlestore-alert-demo-pods ClusterIP None 3306/TCP 5m +singlestore-alert-demo-stats ClusterIP 10.43.10.21 56790/TCP 5m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-singlestore +NAME AGE +singlestore-alert-demo-stats 5m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-singlestore singlestore-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install singlestore-alerts + +The `singlestore-alerts` chart creates a `PrometheusRule` resource containing all SingleStore alert definitions. + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression (via `job="{release-name}-stats"` / `app="{release-name}"`) from the **Helm release name** — so the release name must match the SingleStore object's name (`singlestore-alert-demo`). + +### Install + +```bash +$ helm upgrade -i singlestore-alert-demo appscode/singlestore-alerts \ + -n alert-singlestore \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-singlestore +NAME AGE +singlestore-alert-demo 30s + +$ kubectl get prometheusrule -n alert-singlestore singlestore-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `singlestore.database`, `singlestore.provisioner`, and `singlestore.kubeStash` groups. + +

+ Prometheus Rule Health +

+ +All groups should show **OK**. Unlike several other `*-alerts` charts, `singlestore-alerts` v2026.7.14 has no `opsManager` group at all — its `values.yaml` only declares `database`, `provisioner`, and `kubeStash`. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-singlestore%22%7D&g0.tab=1`. + +

+ Prometheus up query — singlestore-alert-demo nodes UP +

+ +Only **one** series is returned, scoped to `pod="singlestore-alert-demo-aggregator-0"`. This is not a query mistake — the `singlestore-alert-demo-stats` Service selects all three pods, but its Endpoints only ever resolve to the aggregator (`kubectl get endpoints -n alert-singlestore singlestore-alert-demo-stats` shows a single address). In this chart, **only the aggregator node exposes the Prometheus metrics endpoint** — the leaf pods don't. Every database-group alert (`SinglestoreInstanceDown`, `SinglestoreHighQPS`, etc.) is therefore effectively scoped to the aggregator only, not the leaves, regardless of how many leaf replicas you run. + +### 2. Confirm the SingleStore alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — SingleStore groups inactive +

+ +All 10 rules in the `singlestore.database` group and all 7 rules in the `singlestore.kubeStash` group show **INACTIVE**. `singlestore.kubeStash` rules stay INACTIVE with no data unless KubeStash backups are configured. + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +### 4. Explore the Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the SingleStore dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.Singlestore=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `KubeDBSinglestorePhaseNotReady` by crashing the **aggregator** node. With `spec.topology.aggregator.replicas: 1`, the aggregator is a single point of coordination for the whole cluster — crashing it repeatedly holds the CR's `status.phase` at `NotReady` long enough for the provisioner-group alert to fire, unlike crashing a single leaf (which only degrades the cluster, since a second leaf survives). + +### 1. Crash the aggregator process repeatedly + +```bash +$ end=$(( $(date +%s) + 120 )) + while [ $(date +%s) -lt $end ]; do + kubectl exec -n alert-singlestore singlestore-alert-demo-aggregator-0 -c singlestore -- sh -c 'pid=$(pgrep -x memsqld | head -1); [ -n "$pid" ] && kill -9 "$pid"' >/dev/null 2>&1 + sleep 5 + done +``` + +Watch the CR phase move to `NotReady`: + +```bash +$ kubectl get singlestore -n alert-singlestore singlestore-alert-demo -o jsonpath='{.status.phase}' +NotReady +``` + +`KubeDBSinglestorePhaseNotReady` keys off `kubedb_com_singlestore_status_phase` (a metric emitted by the KubeDB operator itself, via panopticon), so what matters is the CR's `status.phase` staying at `NotReady` for the full `for: 1m` window, not the aggregator's own scrape health. + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — KubeDBSinglestorePhaseNotReady Firing +

+ +`KubeDBSinglestorePhaseNotReady` (in the `singlestore.provisioner` group) transitions from **INACTIVE** to **FIRING**, while `singlestore.database` and `singlestore.kubeStash` stay **INACTIVE** — crashing the aggregator's `memsqld` process doesn't take the metrics scrape target down (the pod's `singlestore` container restarts quickly), so `SinglestoreInstanceDown` never fires here; it's specifically the operator's own phase tracking that reacts. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — KubeDBSinglestorePhaseNotReady Firing +

+ +AlertManager shows the `KubeDBSinglestorePhaseNotReady` alert. The alert card displays labels including: + +- **alertname**: `KubeDBSinglestorePhaseNotReady` +- **severity**: `critical` +- **app**: `singlestore-alert-demo`, **app_namespace**: `alert-singlestore` +- **phase**: `NotReady` +- **k8s_kind**: `Singlestore` + +Note that the `instance`/`pod`/`job` labels point at the KubeDB operator's **panopticon** component (`job="panopticon"`), not at the SingleStore pod itself — because this alert is derived from the operator's own status metric. + +### 4. Restore SingleStore + +Stop the loop from step 1. + +```bash +$ kubectl get singlestore -n alert-singlestore singlestore-alert-demo -w +NAME VERSION STATUS AGE +singlestore-alert-demo 8.9.3 Ready 24m +``` + +If the aggregator does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-singlestore singlestore-alert-demo-aggregator-0`. + +--- + +## Alert Reference + +All database-group alerts are scoped to the `singlestore-alert-demo` instance via the PromQL label filters `job="singlestore-alert-demo-stats"` / `namespace="alert-singlestore"`; provisioner/kubeStash alerts use `app="singlestore-alert-demo"` / `namespace="alert-singlestore"`. + +### Database Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `SinglestoreInstanceDown` | critical | instant | `memsql_up == 0` on a node. | +| `SinglestoreServiceDown` | critical | instant | No replica behind the service is answering. | +| `SinglestoreTooManyConnections` | warning | 2m | Connection count is high. | +| `SinglestoreHighThreadsRunning` | warning | 2m | Too many threads actively running. | +| `SinglestoreRestarted` | warning | instant | Uptime indicates a recent restart. | +| `SinglestoreHighQPS` | critical | instant | Query rate is unusually high. | +| `SinglestoreHighIncomingBytes` | critical | instant | Inbound network traffic is unusually high. | +| `SinglestoreHighOutgoingBytes` | critical | instant | Outbound network traffic is unusually high. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Provisioner Group + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBSinglestorePhaseNotReady` | critical | 1m | KubeDB marked the SingleStore resource `NotReady`. | +| `KubeDBSinglestorePhaseCritical` | warning | 15m | SingleStore is degraded but not fully unavailable. | + +### KubeStash Group + +Only meaningful once KubeStash backup/restore is configured. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `SinglestoreKubeStashBackupSessionFailed` | critical | instant | Most recent backup session failed. | +| `SinglestoreKubeStashRestoreSessionFailed` | critical | instant | Most recent restore session failed. | +| `SinglestoreKubeStashNoBackupSessionForTooLong` | warning | instant | No recent successful backup. | +| `SinglestoreKubeStashRepositoryCorrupted` | critical | 5m | Backup repository integrity check failed. | +| `SinglestoreKubeStashRepositoryStorageRunningLow` | warning | 5m | Backup repository storage usage is high. | +| `SinglestoreKubeStashBackupSessionPeriodTooLong` | warning | instant | A backup session is taking unusually long. | +| `SinglestoreKubeStashRestoreSessionPeriodTooLong` | warning | instant | A restore session is taking unusually long. | + +--- + +## Customising Alerts + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + singlestoreTooManyConnections: + enabled: true + duration: "5m" + severity: warning + kubeStash: + enabled: "none" # disable if you don't use KubeStash +``` + +```bash +$ helm upgrade singlestore-alert-demo appscode/singlestore-alerts \ + -n alert-singlestore \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +```bash +$ helm uninstall singlestore-alert-demo -n alert-singlestore +$ kubectl delete singlestore -n alert-singlestore singlestore-alert-demo +$ kubectl delete secret -n alert-singlestore license-secret +$ kubectl delete ns alert-singlestore +``` + +## Next Steps + +- Monitor your SingleStore cluster with KubeDB using [built-in Prometheus](/docs/guides/singlestore/monitoring/builtin-prometheus/index.md). +- Monitor your SingleStore cluster with KubeDB using [Prometheus operator](/docs/guides/singlestore/monitoring/prometheus-operator/index.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/solr/monitoring/alerting.md b/docs/guides/solr/monitoring/alerting.md new file mode 100644 index 0000000000..90534131a5 --- /dev/null +++ b/docs/guides/solr/monitoring/alerting.md @@ -0,0 +1,408 @@ +--- +title: Solr Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: sl-monitoring-alerting + name: Alerting + parent: sl-monitoring-solr + weight: 60 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Solr Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed Solr instance using the `solr-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `alert-solr` namespace: + + ```bash + $ kubectl create ns alert-solr + namespace/alert-solr created + ``` + +* Solr requires a reference to a KubeDB `ZooKeeper` cluster for coordination — deploy one first (see below). + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack** and **Panopticon**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/solr/monitoring/overview.md). + +> Note: YAML files used in this tutorial are stored in [docs/examples/solr](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/solr) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **KubeDB** deploys Solr with a metrics-exporter sidecar (container `exporter`) that exposes Solr's own metrics (`solr_metrics_*`, `solr_collections_*`). +- **ServiceMonitor** (named `{solr-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape the exporter every 10 seconds. +- **PrometheusRule** is created by the `solr-alerts` chart and contains alert definitions grouped by concern: database health and provisioner. +- **Prometheus Operator** evaluates every rule expression every 30 seconds and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). +- **Grafana** dashboards for Solr are covered separately — see [Grafana Dashboard](grafana-dashboard.md) rather than duplicated here. + +--- + +## Deploy the ZooKeeper Coordinator + +Solr coordinates via a KubeDB `ZooKeeper` cluster, so deploy that first. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: ZooKeeper +metadata: + name: zoo + namespace: alert-solr +spec: + version: 3.8.3 + replicas: 3 + deletionPolicy: WipeOut + adminServerPort: 8080 + storage: + resources: + requests: + storage: "100Mi" + storageClassName: local-path + accessModes: + - ReadWriteOnce +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/solr/monitoring/zookeeper-alert-demo.yaml +zookeeper.kubedb.com/zoo created + +$ kubectl get zookeeper -n alert-solr zoo +NAME VERSION STATUS AGE +zoo 3.8.3 Ready 3m +``` + +## Deploy Solr with Monitoring Enabled + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Solr +metadata: + name: solr-alert + namespace: alert-solr +spec: + version: 9.8.0 + replicas: 3 + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s + solrModules: + - s3-repository + - gcs-repository + - prometheus-exporter + zookeeperRef: + name: zoo + namespace: alert-solr + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: local-path + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/solr/monitoring/solr-alert-demo.yaml +solr.kubedb.com/solr-alert created +``` + +Wait for the database to go into `Ready` state. + +```bash +$ kubectl get solr -n alert-solr solr-alert +NAME VERSION STATUS AGE +solr-alert 9.8.0 Ready 5m +``` + +KubeDB brings up 3 pods, one per Solr node: + +```bash +$ kubectl get pods -n alert-solr +NAME READY STATUS RESTARTS AGE +solr-alert-0 1/1 Running 0 5m +solr-alert-1 1/1 Running 0 5m +solr-alert-2 1/1 Running 0 5m +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n alert-solr --selector="app.kubernetes.io/instance=solr-alert" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +solr-alert ClusterIP 10.43.219.25 8983/TCP 5m +solr-alert-pods ClusterIP None 8983/TCP 5m +solr-alert-stats ClusterIP 10.43.163.252 9854/TCP 5m +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n alert-solr +NAME AGE +solr-alert-stats 5m +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n alert-solr solr-alert-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install solr-alerts + +### Why the Helm release name matters + +The chart derives the `PrometheusRule` name and scopes every PromQL expression from the **Helm release name** — so the release name must match the Solr object's name (`solr-alert`). + +### Install + +```bash +$ helm upgrade -i solr-alert appscode/solr-alerts \ + -n alert-solr \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n alert-solr +NAME AGE +solr-alert 30s + +$ kubectl get prometheusrule -n alert-solr solr-alert \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules` and locate the `solr.database` and `solr.provisioner` groups. + +

+ Prometheus Rule Health +

+ +Both groups should show **OK**. `solr-alerts` v2026.7.14 has no `opsManager`/`stash`/`kubeStash` groups — only `database` and `provisioner`. Note there is no plain `SolrDown` alert; the closest equivalent is `SolrDownShards` (shard-level) and the provisioner group's `KubeDBSolrPhaseNotReady`. + +--- + +## Verify End-to-End + +### 1. Check the Prometheus target is UP + +Open `http://localhost:9090/query?g0.expr=up%7Bnamespace%3D%22alert-solr%22%7D&g0.tab=1`. + +

+ Prometheus up query — solr-alert-0 UP +

+ +All three pods — `solr-alert-0`, `solr-alert-1`, `solr-alert-2` — report `up == 1`, confirming Prometheus is scraping every Solr node in the `alert-solr` namespace. + +### 2. Confirm the Solr alerts are inactive + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — Solr groups inactive +

+ +All 9 rules in the `solr.database` group and both rules in the `solr.provisioner` group show **INACTIVE**, confirming the cluster is healthy and no thresholds are breached. + +### 3. Check AlertManager + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. + +

+ AlertManager +

+ +### 4. Explore the Grafana dashboard + +See [Grafana Dashboard](grafana-dashboard.md) for how to provision and explore the Solr dashboards (via the `kubedb-grafana-dashboards` chart, `--set featureGates.Solr=true`). + +--- + +## Simulating a Firing Alert + +This section deliberately triggers `KubeDBSolrPhaseNotReady` by repeatedly crashing the main Solr JVM process on one node. A single `kill` restarts fast enough that the container becomes `Ready` again before KubeDB's health check can observe the outage, so the crash needs to be sustained over a longer window than the `for` duration of the alert. + +### 1. Crash the Solr process repeatedly + +```bash +$ end=$(( $(date +%s) + 150 )) + while [ $(date +%s) -lt $end ]; do + kubectl exec -n alert-solr solr-alert-0 -c solr -- sh -c 'pid=$(pgrep -f "org.apache.solr" | head -1); [ -n "$pid" ] && kill -9 "$pid"' >/dev/null 2>&1 + sleep 5 + done +``` + +Watch the CR phase move to `NotReady`: + +```bash +$ kubectl get solr -n alert-solr solr-alert -o jsonpath='{.status.phase}' +NotReady +``` + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts`. + +

+ Prometheus Alerts — SolrDownShards Firing, KubeDBSolrPhaseNotReady Pending +

+ +`SolrDownShards` (database group, `for: 30s`) reaches **FIRING** first, since the crashed node's shard replica goes unreachable almost immediately. The provisioner-group `KubeDBSolrPhaseNotReady` (`for: 1m`) takes longer — in this screenshot it's still **PENDING**, and transitions to **FIRING** once the KubeDB operator holds the resource at `NotReady` past the full one-minute window (confirmed via `kubectl get solr ... -o jsonpath='{.status.phase}'` above, and in the AlertManager screenshot next). + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093`. + +

+ AlertManager — KubeDBSolrPhaseNotReady Firing +

+ +AlertManager shows the `KubeDBSolrPhaseNotReady` alert, confirming it did reach **FIRING** shortly after the previous screenshot. The alert card displays labels including: + +- **alertname**: `KubeDBSolrPhaseNotReady` +- **severity**: `critical` +- **app**: `solr-alert`, **app_namespace**: `alert-solr` +- **phase**: `NotReady` +- **k8s_kind**: `Solr` + +Note that the `instance`/`pod`/`job` labels point at the KubeDB operator's **panopticon** component (`job="panopticon"`), not at the Solr pod itself — because this alert is derived from the operator's own status metric rather than from the Solr exporter. + +AlertManager routes this alert to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alert is visible here but silently dropped. + +### 4. Restore Solr + +Stop the loop from step 1. + +```bash +$ kubectl get solr -n alert-solr solr-alert -w +NAME VERSION STATUS AGE +solr-alert 9.8.0 Ready 24m +``` + +If Solr does not recover on its own within a minute or two, force a clean restart: `kubectl delete pod -n alert-solr solr-alert-0`. + +--- + +## Alert Reference + +All alerts are scoped to the `solr-alert` instance in the `alert-solr` namespace via `job="solr-alert-stats"` / `namespace="alert-solr"` (database group), or `app="solr-alert"` / `namespace="alert-solr"` (provisioner group). + +### Database Group + +Fired based on live metrics from the Solr exporter sidecar and node/kubelet metrics. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `SolrDownShards` | critical | 30s | One or more collection shards have no active replica. | +| `SolrRecoveryFailedShards` | critical | 30s | A shard replica is stuck in recovery-failed state. | +| `SolrHighThreadRunning` | warning | 30s | JVM thread count is high. | +| `SolrHighPoolSize` | warning | 30s | JVM memory pool usage is high. | +| `SolrHighQPS` | warning | 30s | Query rate is unusually high for a collection. | +| `SolrHighHeapSize` | warning | 30s | JVM heap usage is high. | +| `SolrHighBufferSize` | warning | 30s | JVM direct buffer usage is high. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage exceeds 95%. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the Solr resource phase (sourced from Panopticon, not the Solr metrics endpoint). + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBSolrPhaseNotReady` | critical | 1m | KubeDB marked the Solr resource `NotReady`. | +| `KubeDBSolrPhaseCritical` | warning | 1m | Solr is degraded but not fully unavailable. | + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + solrHighQPS: + enabled: true + duration: "2m" + severity: warning +``` + +```bash +$ helm upgrade solr-alert appscode/solr-alerts \ + -n alert-solr \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +$ helm uninstall solr-alert -n alert-solr +$ kubectl delete solr -n alert-solr solr-alert +$ kubectl delete zookeeper -n alert-solr zoo +$ kubectl delete ns alert-solr +``` + +## Next Steps + +- Monitor your Solr instance with KubeDB using [built-in Prometheus](/docs/guides/solr/monitoring/prometheus-builtin.md). +- Monitor your Solr instance with KubeDB using [Prometheus operator](/docs/guides/solr/monitoring/prometheus-operator.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/guides/zookeeper/monitoring/alerting.md b/docs/guides/zookeeper/monitoring/alerting.md new file mode 100644 index 0000000000..e28c5e3a6f --- /dev/null +++ b/docs/guides/zookeeper/monitoring/alerting.md @@ -0,0 +1,445 @@ +--- +title: ZooKeeper Alerting with Prometheus +menu: + docs_{{ .version }}: + identifier: zk-monitoring-alerting + name: Alerting + parent: zk-monitoring-guides + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# ZooKeeper Alerting with Prometheus + +This tutorial shows you how to configure Prometheus-based alerting for a KubeDB-managed ZooKeeper instance using the `zookeeper-alerts` Helm chart. + +## Before You Begin + +* Ensure you have a Kubernetes cluster and that `kubectl` is configured to communicate with it. If you do not already have a cluster, you can create one using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +* Install the KubeDB operator by following the steps [here](/docs/setup/README.md). + +* Deploy the database in the `demo` namespace: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +* Before proceeding, complete the [Configuration](grafana-dashboard.md#configuration) steps to deploy **kube-prometheus-stack**. + +* This tutorial assumes you already have a **kube-prometheus-stack** running in your cluster, with `Prometheus` configured so that both `serviceMonitorSelector` and `ruleSelector` match the label `release: prometheus`. + + To verify the selectors: + + ```bash + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.ruleSelector}' + {"matchLabels":{"release":"prometheus"}} + + $ kubectl get prometheus -n monitoring -o jsonpath='{.items[0].spec.serviceMonitorSelector}' + {"matchLabels":{"release":"prometheus"}} + ``` + +* To learn more about how Prometheus monitoring works with KubeDB, see the overview [here](/docs/guides/zookeeper/monitoring/overview.md). + +* For dashboards and visualisation, see [Grafana Dashboard](grafana-dashboard.md) for ZooKeeper. + +> Note: YAML files used in this tutorial are stored in [docs/examples/zookeeper](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/zookeeper) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +- **ZooKeeper** (3.6+) ships with a built-in Prometheus `MetricsProvider` that exposes metrics natively on an admin/metrics port — KubeDB does not need to inject a separate exporter sidecar for ZooKeeper, unlike some other databases. +- **ServiceMonitor** (named `{zookeeper-name}-stats`) is created automatically by KubeDB and tells Prometheus to scrape each ZooKeeper pod's metrics endpoint every 10 seconds. +- **PrometheusRule** is created by the `zookeeper-alerts` chart and contains all ZooKeeper alert definitions grouped by concern: database health and provisioner. +- **Prometheus Operator** evaluates every rule expression on its configured evaluation interval and fires matching alerts to AlertManager. +- **AlertManager** groups, inhibits, and silences alerts, then routes them to configured receivers (Slack, email, PagerDuty, webhook, etc.). + +--- + +## Deploy ZooKeeper with Monitoring Enabled + +At first, let's deploy a ZooKeeper database with monitoring enabled. Below is the ZooKeeper object we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: ZooKeeper +metadata: + name: zk-alert-demo + namespace: demo +spec: + version: "3.8.3" + replicas: 3 + storage: + resources: + requests: + storage: "1Gi" + storageClassName: "standard" + accessModes: + - ReadWriteOnce + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus +``` + +Here, + +- `spec.replicas: 3` is the documented minimum ZooKeeper ensemble size for quorum — keep it at 3 (or more, in odd numbers) for a production-like setup. +- `spec.monitor.agent: prometheus.io/operator` tells KubeDB to create a `ServiceMonitor` resource managed by the Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels.release: prometheus` adds the `release: prometheus` label to the created `ServiceMonitor`, matching the Prometheus `serviceMonitorSelector` so the target is discovered automatically. + +Let's create the ZooKeeper resource. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/zookeeper/monitoring/zk-alert-demo.yaml +zookeeper.kubedb.com/zk-alert-demo created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get zookeeper -n demo zk-alert-demo +NAME VERSION STATUS AGE +zk-alert-demo 3.8.3 Ready 90s +``` + +KubeDB creates a dedicated stats service with the `-stats` suffix for monitoring. + +```bash +$ kubectl get svc -n demo --selector="app.kubernetes.io/instance=zk-alert-demo" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +zk-alert-demo ClusterIP 10.43.78.149 2181/TCP 97s +zk-alert-demo-admin-server ClusterIP 10.43.214.188 8080/TCP 97s +zk-alert-demo-pods ClusterIP None 2181/TCP,2888/TCP,3888/TCP 97s +zk-alert-demo-stats ClusterIP 10.43.250.162 7000/TCP 97s +``` + +KubeDB also creates a `ServiceMonitor` that tells Prometheus where to scrape. + +```bash +$ kubectl get servicemonitor -n demo +NAME AGE +zk-alert-demo-stats 97s +``` + +Verify that the `ServiceMonitor` carries the `release: prometheus` label so Prometheus discovers it. + +```bash +$ kubectl get servicemonitor -n demo zk-alert-demo-stats \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +--- + +## Step 1 — Install zookeeper-alerts + +The `zookeeper-alerts` chart creates a `PrometheusRule` resource containing all ZooKeeper alert definitions grouped by concern: database health and provisioner. + +### Why the Helm release name matters + +The chart derives the PromQL `job`/instance scoping (and the `PrometheusRule` name) from the **Helm release name**, not from a values field — so the release name must match the ZooKeeper object's name (`zk-alert-demo`) for the rules to be correctly scoped to this instance. + +The chart's default label is `release: kube-prometheus-stack`, so we must also override it at install time to match the Prometheus `ruleSelector`. + +### Install + +```bash +$ helm upgrade -i zk-alert-demo oci://ghcr.io/appscode-charts/zookeeper-alerts \ + -n demo \ + --create-namespace \ + --version=v2026.7.14 \ + --set form.alert.labels.release=prometheus +``` + +| Flag | Value | Purpose | +|------|-------|---------| +| `zk-alert-demo` (release name) | — | Scopes every PromQL expression to this instance (`job="zk-alert-demo-stats"`) | +| `-n demo` | `demo` | Installs the `PrometheusRule` in the same namespace as the database | +| `form.alert.labels.release` | `prometheus` | Matches the Prometheus `ruleSelector` so the rules are loaded | + +### Verify the PrometheusRule is created + +```bash +$ kubectl get prometheusrule -n demo +NAME AGE +zk-alert-demo 30s +``` + +Confirm the `release: prometheus` label is present. + +```bash +$ kubectl get prometheusrule -n demo zk-alert-demo \ + -o jsonpath='{.metadata.labels.release}' +prometheus +``` + +### Confirm Prometheus loaded the rules + +Port-forward the Prometheus UI and open the **Status → Rule health** page. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-prometheus 9090:9090 +``` + +Open `http://localhost:9090/rules?search=zookeeper`. + +

+ Prometheus Rule Health +

+ +The `zookeeper.database.demo.zk-alert-demo.rules` group is visible with all rules showing **OK**, confirming that Prometheus has loaded and is evaluating the ZooKeeper alert definitions. + +--- + +## Verify End-to-End + +### 1. Check the metrics endpoint is serving + +ZooKeeper's built-in metrics provider serves Prometheus metrics at `:7000/metrics` directly from the ZooKeeper JVM — there is no separate exporter sidecar container to check for this database. + +```bash +$ kubectl exec -n demo zk-alert-demo-0 -c zookeeper -- \ + wget -qO- localhost:7000/metrics | grep -m1 "^# HELP" +# HELP write_commit_proc_issued write_commit_proc_issued +``` + +A non-empty response confirms the JVM's Prometheus servlet is up and exporting metrics. + +### 2. Check the Prometheus target is UP + +Open `http://localhost:9090/targets?search=zk-alert-demo`. + +

+ Prometheus Target UP +

+ +All three targets under `serviceMonitor/demo/zk-alert-demo-stats/0` show **UP** — one per ensemble member (`zk-alert-demo-0`, `zk-alert-demo-1`, `zk-alert-demo-2`). + +> Note: with more than 20 scrape pools on a busy cluster, the Prometheus React UI's target-search box can sometimes default to showing an unrelated pool first. If that happens, use the pool dropdown (or the `pool=` URL parameter) to explicitly pick `serviceMonitor//-stats/0`. + +### 3. Confirm the ZooKeeper alerts are inactive + +Open `http://localhost:9090/alerts?search=zookeeper` to see the ZooKeeper alert groups. + +

+ Prometheus Alerts +

+ +All 11 metric-driven rules in the `zookeeper.database` group (`ZooKeeperDown`, `ZooKeeperTooManyNodes`, `ZooKeeperTooManyConnections`, etc.) show **INACTIVE**, meaning the ensemble is healthy and no thresholds are breached. + +> Note: the `DiskUsageHigh` / `DiskAlmostFull` rules in the same group compare `kubelet_volume_stats_used_bytes` against the PVC's usage on the underlying node. On a demo cluster where the node's disk is already heavily utilized by other workloads, these two rules can legitimately show **PENDING**/**FIRING** independent of ZooKeeper itself — that reflects real node-level disk pressure, not a problem with the ZooKeeper instance. Give your PVC's underlying disk enough free space if you want to keep these two alerts quiet in your own cluster. + +### 4. Check AlertManager + +Port-forward AlertManager to view any currently firing alerts. + +```bash +$ kubectl port-forward -n monitoring \ + svc/prometheus-kube-prometheus-alertmanager 9093:9093 +``` + +Open `http://localhost:9093`. With a healthy ZooKeeper ensemble, no `ZooKeeperDown` (or other database-group) alert for `zk-alert-demo` will be listed here. + +--- + +## Simulating a Firing Alert + +The previous section confirmed that the ZooKeeper-specific alerts are **INACTIVE** while the ensemble is healthy. This section walks through deliberately triggering the `ZooKeeperDown` critical alert so you can observe the full alert lifecycle — from firing in Prometheus through to the AlertManager dashboard — and then resolve it. + +### 1. Stop the ZooKeeper process + +`ZooKeeperDown` fires on the standard Prometheus `up{job="zk-alert-demo-stats"} == 0` scrape-health metric, with `for: 1m` — the target has to stay unreachable for a full minute before the alert transitions from **PENDING** to **FIRING**. + +ZooKeeper ships as a single container per pod (no exporter sidecar), so killing the main JVM process brings the whole container down; the container then restarts automatically. Find the ZooKeeper JVM's PID inside the pod and kill it: + +```bash +$ kubectl exec -n demo zk-alert-demo-0 -c zookeeper -- pgrep -f QuorumPeerMain +38 + +$ kubectl exec -n demo zk-alert-demo-0 -c zookeeper -- kill -9 38 +``` + +Because the container typically restarts within a few seconds — faster than the 1-minute `for` window — a single kill is usually not enough to observe a **FIRING** state. Repeat the kill every few seconds until the container is caught in `CrashLoopBackOff`, which keeps it down long enough to satisfy the 1-minute condition: + +```bash +$ for i in $(seq 1 20); do + PID=$(kubectl exec -n demo zk-alert-demo-0 -c zookeeper -- pgrep -f QuorumPeerMain 2>/dev/null | head -1) + [ -n "$PID" ] && kubectl exec -n demo zk-alert-demo-0 -c zookeeper -- kill -9 "$PID" + sleep 3 + done + +$ kubectl get pod -n demo zk-alert-demo-0 +NAME READY STATUS RESTARTS AGE +zk-alert-demo-0 0/1 Error 6 (93s ago) 56m +``` + +Poll the Prometheus API until the rule state flips from `pending` to `firing` (roughly 60–90 seconds after the pod first goes down): + +```bash +$ curl -s "http://localhost:9090/api/v1/rules?type=alert" \ + | jq -r '.data.groups[] | select(.name | contains("zookeeper.database")) | .rules[] | select(.name=="ZooKeeperDown") | .state' +firing +``` + +### 2. Watch the alert fire in Prometheus + +Open `http://localhost:9090/alerts?search=zookeeper`. + +

+ Prometheus Alerts — ZooKeeperDown Firing +

+ +`ZooKeeperDown` now shows **FIRING (1)** — for the single pod (`zk-alert-demo-0`) that was taken down; the other two ensemble members remain healthy. + +### 3. Check the AlertManager dashboard + +Open `http://localhost:9093/#/alerts?filter=%7Bnamespace%3D%22demo%22%7D`. + +

+ AlertManager — ZooKeeperDown Firing +

+ +AlertManager shows the `ZooKeeperDown` alert. The alert card displays: + +- **Severity**: `critical` +- **Instance**: `zk-alert-demo-0` in the `demo` namespace +- **job**: `zk-alert-demo-stats` +- **Started**: timestamp when the alert first fired + +In this demo run, AlertManager also showed a `KubeDBZooKeeperPhaseNotReady` alert (from the `provisioner` group) firing at the same time — the repeated `CrashLoopBackOff` was severe enough that the KubeDB operator itself marked the ZooKeeper object `NotReady`, which is a realistic example of two independent alert groups correctly firing together from a single underlying failure. + +AlertManager routes these alerts to every receiver configured in your `alertmanagerConfig` (Slack, email, PagerDuty, webhook, etc.) based on your routing tree. If no receiver is configured, the alerts are visible here but silently dropped. + +### 4. Restore ZooKeeper + +Delete the pod so KubeDB recreates it cleanly instead of leaving it in `CrashLoopBackOff`. + +```bash +$ kubectl delete pod -n demo zk-alert-demo-0 +``` + +Once the pod is back to `Running` and its metrics endpoint is reachable again, Prometheus marks `ZooKeeperDown` **INACTIVE** and AlertManager sends a **resolved** notification to all receivers. + +--- + +## Alert Reference + +All alerts are scoped to the `zk-alert-demo` instance in the `demo` namespace via the PromQL label filters `job="zk-alert-demo-stats"` and `namespace="demo"`. + +### Database Group + +Fired based on live metrics from ZooKeeper's built-in Prometheus metrics provider. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `ZooKeeperDown` | critical | 1m | ZooKeeper instance is down — the scrape target has been unreachable for a full minute. | +| `ZooKeeperTooManyNodes` | warning | 1m | ZooKeeper ensemble has too many znodes (more than 1,000,000) — consider scaling up. | +| `ZooKeeperTooBigMemory` | warning | 1m | ZooKeeper's znode total occupied memory (`approximate_data_size`) is too big (more than 1 GB). | +| `ZooKeeperTooManyWatch` | warning | 1m | Too many watches are set (more than 10,000) on this instance. | +| `ZooKeeperTooManyConnections` | warning | 1m | More than 60 client connections are in use on this instance. | +| `ZooKeeperLeaderElection` | warning | 1m | A leader election happened in the last 5 minutes — indicates ensemble instability. | +| `ZooKeeperTooManyOpenFiles` | warning | 1m | More than 300 open file descriptors on this instance. | +| `ZooKeeperTooLongFsyncTime` | warning | 1m | fsync operations are taking too long (rate over 1m exceeds 100). | +| `ZooKeeperTooLongSnapshotTime` | warning | 1m | Snapshotting is taking too long (rate over 5m exceeds 100). | +| `ZooKeeperTooHighAvgLatency` | warning | 1m | Average request latency (`avg_latency`) exceeds 100ms. | +| `ZooKeeperJvmMemoryFilingUp` | warning | 1m | JVM heap usage exceeds 80% of the max heap size. | +| `DiskUsageHigh` | warning | 1m | Persistent volume usage for this instance's data directory exceeds 80%. | +| `DiskAlmostFull` | critical | 1m | Persistent volume usage for this instance's data directory exceeds 95%. | + +### Provisioner Group + +Monitors the KubeDB operator's view of the ZooKeeper resource phase. + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `KubeDBZooKeeperPhaseNotReady` | critical | 1m | KubeDB marked the ZooKeeper resource `NotReady` — operator cannot reach the ensemble. | +| `KubeDBZooKeeperPhaseCritical` | warning | 15m | The instance is in a degraded/critical phase. | + +### OpsManager Group + +The chart's `values.yaml` declares an `opsManager` group, intended to track `ZooKeeperOpsRequest` lifecycle during upgrades, scaling, and reconfiguration — the same pattern used by other `*-alerts` charts: + +| Alert | Severity | For | What It Means | +|-------|----------|-----|---------------| +| `opsRequestOnProgress` | info | instant | An ops request is currently in progress. | +| `opsRequestStatusProgressingToLong` | critical | 30m | An ops request has been running for 30+ minutes — likely stuck. | +| `opsRequestFailed` | critical | instant | An ops request failed — check the `OpsRequest` object for the error. | + +> Verified against chart `v2026.7.14`: unlike the `database` and `provisioner` groups, this `opsManager` group is **not** currently rendered into the `PrometheusRule` template (`kubectl get prometheusrule -n demo zk-alert-demo -o jsonpath='{.spec.groups[*].name}'` only returns `zookeeper.database...` and `zookeeper.provisioner...`). The values above document the intended configuration surface; confirm with `helm template` against the chart version you install whether opsManager rules are actually emitted before relying on them for `ZooKeeperOpsRequest` alerting: +> +> ```bash +> $ helm template zk-alert-demo oci://ghcr.io/appscode-charts/zookeeper-alerts \ +> --version=v2026.7.14 \ +> --set form.alert.labels.release=prometheus \ +> | grep "alert: " +> ``` + +--- + +## Customising Alerts + +To override thresholds or disable specific alert groups, create a custom values file and upgrade the chart. + +```yaml +# custom-alerts.yaml +form: + alert: + labels: + release: prometheus + groups: + database: + enabled: warning + rules: + zookeeperTooManyConnections: + enabled: true + duration: "5m" + val: 100 # fire at 100 connections instead of the default 60 + severity: warning + diskUsageHigh: + enabled: true + val: 90 # raise the disk-usage-high threshold to 90% + duration: "5m" + severity: warning + provisioner: + enabled: "none" # disable all provisioner alerts +``` + +```bash +$ helm upgrade zk-alert-demo oci://ghcr.io/appscode-charts/zookeeper-alerts \ + -n demo \ + --version=v2026.7.14 \ + -f custom-alerts.yaml +``` + +--- + +## Cleaning up + +To remove all resources created in this tutorial, run the following commands. + +```bash +# Remove the zookeeper-alerts release +$ helm uninstall zk-alert-demo -n demo + +# Remove the ZooKeeper instance +$ kubectl delete zookeeper -n demo zk-alert-demo + +# Delete namespace +$ kubectl delete ns demo +``` + +## Next Steps + +- Monitor your ZooKeeper database with KubeDB using [builtin Prometheus](/docs/guides/zookeeper/monitoring/using-builtin-prometheus.md). +- Monitor your ZooKeeper database with KubeDB using [Prometheus operator](/docs/guides/zookeeper/monitoring/using-prometheus-operator.md). +- Visualise ZooKeeper metrics with [Grafana Dashboard](grafana-dashboard.md). +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). diff --git a/docs/images/cassandra/monitoring/cas-alerting-alertmanager-firing.png b/docs/images/cassandra/monitoring/cas-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..dce2a98da8 Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-alertmanager-firing.png differ diff --git a/docs/images/cassandra/monitoring/cas-alerting-alertmanager.png b/docs/images/cassandra/monitoring/cas-alerting-alertmanager.png new file mode 100644 index 0000000000..27b0385c4d Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-alertmanager.png differ diff --git a/docs/images/cassandra/monitoring/cas-alerting-grafana-dashboard.png b/docs/images/cassandra/monitoring/cas-alerting-grafana-dashboard.png new file mode 100644 index 0000000000..a3402a5c0d Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-grafana-dashboard.png differ diff --git a/docs/images/cassandra/monitoring/cas-alerting-prom-alerts-firing.png b/docs/images/cassandra/monitoring/cas-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..82f71d40df Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-prom-alerts-firing.png differ diff --git a/docs/images/cassandra/monitoring/cas-alerting-prom-alerts.png b/docs/images/cassandra/monitoring/cas-alerting-prom-alerts.png new file mode 100644 index 0000000000..ad14bb7659 Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-prom-alerts.png differ diff --git a/docs/images/cassandra/monitoring/cas-alerting-prom-rules..png b/docs/images/cassandra/monitoring/cas-alerting-prom-rules..png new file mode 100644 index 0000000000..957a367a1f Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-prom-rules..png differ diff --git a/docs/images/cassandra/monitoring/cas-alerting-prom-rules.png b/docs/images/cassandra/monitoring/cas-alerting-prom-rules.png new file mode 100644 index 0000000000..ea12e0645c Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-prom-rules.png differ diff --git a/docs/images/cassandra/monitoring/cas-alerting-prom-target.png b/docs/images/cassandra/monitoring/cas-alerting-prom-target.png new file mode 100644 index 0000000000..d13eddaef2 Binary files /dev/null and b/docs/images/cassandra/monitoring/cas-alerting-prom-target.png differ diff --git a/docs/images/clickhouse/monitoring/clickhouse-alerting-alertmanager-firing.png b/docs/images/clickhouse/monitoring/clickhouse-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..c01de693af Binary files /dev/null and b/docs/images/clickhouse/monitoring/clickhouse-alerting-alertmanager-firing.png differ diff --git a/docs/images/clickhouse/monitoring/clickhouse-alerting-alertmanager.png b/docs/images/clickhouse/monitoring/clickhouse-alerting-alertmanager.png new file mode 100644 index 0000000000..c0c8b78149 Binary files /dev/null and b/docs/images/clickhouse/monitoring/clickhouse-alerting-alertmanager.png differ diff --git a/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-alerts-firing.png b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..9de84e51fd Binary files /dev/null and b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-alerts-firing.png differ diff --git a/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-alerts.png b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-alerts.png new file mode 100644 index 0000000000..be8d3661f1 Binary files /dev/null and b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-alerts.png differ diff --git a/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-rules.png b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-rules.png new file mode 100644 index 0000000000..bf1905f3a0 Binary files /dev/null and b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-rules.png differ diff --git a/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-target.png b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-target.png new file mode 100644 index 0000000000..2d06d4919e Binary files /dev/null and b/docs/images/clickhouse/monitoring/clickhouse-alerting-prom-target.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-alertmanager-firing.png b/docs/images/elasticsearch/monitoring/es-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..5cb1e20a76 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-alertmanager-firing.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-alertmanager.png b/docs/images/elasticsearch/monitoring/es-alerting-alertmanager.png new file mode 100644 index 0000000000..4e692f8730 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-alertmanager.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-grafana-dashboards.png b/docs/images/elasticsearch/monitoring/es-alerting-grafana-dashboards.png new file mode 100644 index 0000000000..e59c627b04 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-grafana-dashboards.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-grafana-database.png b/docs/images/elasticsearch/monitoring/es-alerting-grafana-database.png new file mode 100644 index 0000000000..5b6d7c79e0 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-grafana-database.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-grafana-pod.png b/docs/images/elasticsearch/monitoring/es-alerting-grafana-pod.png new file mode 100644 index 0000000000..8f2ced8437 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-grafana-pod.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-grafana-summary.png b/docs/images/elasticsearch/monitoring/es-alerting-grafana-summary.png new file mode 100644 index 0000000000..0f9f014110 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-grafana-summary.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-overview.svg b/docs/images/elasticsearch/monitoring/es-alerting-overview.svg new file mode 100644 index 0000000000..fca099d758 --- /dev/null +++ b/docs/images/elasticsearch/monitoring/es-alerting-overview.svg @@ -0,0 +1,98 @@ + + + + + + + + Elasticsearch Topology + master · data · ingest pods + + + + elasticsearch + + + + elasticsearch + exporter (sidecar) + + + metrics on :56790/metrics + one target per pod (7 for a 2m/3d/2i cluster) + + + + ServiceMonitor + es-alert-stats + + + + scrape + + + + Prometheus + kube-prometheus-stack + evaluates rules every 30s + ruleSelector: release=prometheus + + + + discovers + + + + PrometheusRule + elasticsearch-alerts chart + release name = es-alert + + + + loads + + + + AlertManager + groups · inhibits · silences + routes to receivers + + + + fires alerts + + + + Slack + + + Email + + + PagerDuty / Webhook + + + + notify + + + + Grafana + kubedb-grafana-dashboards + Summary · Database · Pod + + + + datasource + + + + + + + + + + Elasticsearch Alerting Architecture with KubeDB + + diff --git a/docs/images/elasticsearch/monitoring/es-alerting-prom-alerts-firing.png b/docs/images/elasticsearch/monitoring/es-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..e2449b6299 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-prom-alerts-firing.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-prom-alerts.png b/docs/images/elasticsearch/monitoring/es-alerting-prom-alerts.png new file mode 100644 index 0000000000..7e01f104f8 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-prom-alerts.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-prom-rules.png b/docs/images/elasticsearch/monitoring/es-alerting-prom-rules.png new file mode 100644 index 0000000000..03f8cd906c Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-prom-rules.png differ diff --git a/docs/images/elasticsearch/monitoring/es-alerting-prom-target.png b/docs/images/elasticsearch/monitoring/es-alerting-prom-target.png new file mode 100644 index 0000000000..d5778c5df4 Binary files /dev/null and b/docs/images/elasticsearch/monitoring/es-alerting-prom-target.png differ diff --git a/docs/images/kafka/monitoring/kf-alerting-alertmanager-firing.png b/docs/images/kafka/monitoring/kf-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..a51e1200fd Binary files /dev/null and b/docs/images/kafka/monitoring/kf-alerting-alertmanager-firing.png differ diff --git a/docs/images/kafka/monitoring/kf-alerting-prom-alerts-firing.png b/docs/images/kafka/monitoring/kf-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..98a0fcc702 Binary files /dev/null and b/docs/images/kafka/monitoring/kf-alerting-prom-alerts-firing.png differ diff --git a/docs/images/kafka/monitoring/kf-alerting-prom-alerts.png b/docs/images/kafka/monitoring/kf-alerting-prom-alerts.png new file mode 100644 index 0000000000..0214731531 Binary files /dev/null and b/docs/images/kafka/monitoring/kf-alerting-prom-alerts.png differ diff --git a/docs/images/kafka/monitoring/kf-alerting-prom-rules.png b/docs/images/kafka/monitoring/kf-alerting-prom-rules.png new file mode 100644 index 0000000000..831693447d Binary files /dev/null and b/docs/images/kafka/monitoring/kf-alerting-prom-rules.png differ diff --git a/docs/images/kafka/monitoring/kf-alerting-prom-target.png b/docs/images/kafka/monitoring/kf-alerting-prom-target.png new file mode 100644 index 0000000000..ab9e9be75c Binary files /dev/null and b/docs/images/kafka/monitoring/kf-alerting-prom-target.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-alertmanager-firing.png b/docs/images/mariadb/monitoring/mariadb-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..8bb3b35961 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-alertmanager-firing.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-alertmanager.png b/docs/images/mariadb/monitoring/mariadb-alerting-alertmanager.png new file mode 100644 index 0000000000..37178a7237 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-alertmanager.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-grafana-dashboards.png b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-dashboards.png new file mode 100644 index 0000000000..75b8e8f915 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-dashboards.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-grafana-database.png b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-database.png new file mode 100644 index 0000000000..491427803d Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-database.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-grafana-galera.png b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-galera.png new file mode 100644 index 0000000000..1dfc73119b Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-galera.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-grafana-pod.png b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-pod.png new file mode 100644 index 0000000000..27efcbcf02 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-pod.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-grafana-summary.png b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-summary.png new file mode 100644 index 0000000000..720becaa52 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-grafana-summary.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-overview.svg b/docs/images/mariadb/monitoring/mariadb-alerting-overview.svg new file mode 100644 index 0000000000..58dace9b97 --- /dev/null +++ b/docs/images/mariadb/monitoring/mariadb-alerting-overview.svg @@ -0,0 +1,98 @@ + + + + + + + + MariaDB Galera Cluster + 3 Primary-Primary pods + + + + mariadb + + + + mysqld + exporter (sidecar) + + + metrics on :56790/metrics + one target per pod (3 for this cluster) + + + + ServiceMonitor + mariadb-alert-demo-stats + + + + scrape + + + + Prometheus + kube-prometheus-stack + evaluates rules every 30s + ruleSelector: release=prometheus + + + + discovers + + + + PrometheusRule + mariadb-alerts chart + release name = mariadb-alert-demo + + + + loads + + + + AlertManager + groups · inhibits · silences + routes to receivers + + + + fires alerts + + + + Slack + + + Email + + + PagerDuty / Webhook + + + + notify + + + + Grafana + kubedb-grafana-dashboards + Summary · Database · Pod · Galera-Cluster + + + + datasource + + + + + + + + + + MariaDB Alerting Architecture with KubeDB + + diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-prom-alerts-firing.png b/docs/images/mariadb/monitoring/mariadb-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..0904b22fd1 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-prom-alerts-firing.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-prom-alerts.png b/docs/images/mariadb/monitoring/mariadb-alerting-prom-alerts.png new file mode 100644 index 0000000000..e16bbf0de2 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-prom-alerts.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-prom-rules.png b/docs/images/mariadb/monitoring/mariadb-alerting-prom-rules.png new file mode 100644 index 0000000000..e19399a699 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-prom-rules.png differ diff --git a/docs/images/mariadb/monitoring/mariadb-alerting-prom-target.png b/docs/images/mariadb/monitoring/mariadb-alerting-prom-target.png new file mode 100644 index 0000000000..a725c1a967 Binary files /dev/null and b/docs/images/mariadb/monitoring/mariadb-alerting-prom-target.png differ diff --git a/docs/images/memcached/monitoring/mc-alerting-alertmanager-firing.png b/docs/images/memcached/monitoring/mc-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..589bd1edba Binary files /dev/null and b/docs/images/memcached/monitoring/mc-alerting-alertmanager-firing.png differ diff --git a/docs/images/memcached/monitoring/mc-alerting-prom-alerts-firing.png b/docs/images/memcached/monitoring/mc-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..265694b593 Binary files /dev/null and b/docs/images/memcached/monitoring/mc-alerting-prom-alerts-firing.png differ diff --git a/docs/images/memcached/monitoring/mc-alerting-prom-alerts.png b/docs/images/memcached/monitoring/mc-alerting-prom-alerts.png new file mode 100644 index 0000000000..84234baeb9 Binary files /dev/null and b/docs/images/memcached/monitoring/mc-alerting-prom-alerts.png differ diff --git a/docs/images/memcached/monitoring/mc-alerting-prom-rules.png b/docs/images/memcached/monitoring/mc-alerting-prom-rules.png new file mode 100644 index 0000000000..0d05f05837 Binary files /dev/null and b/docs/images/memcached/monitoring/mc-alerting-prom-rules.png differ diff --git a/docs/images/memcached/monitoring/mc-alerting-prom-target.png b/docs/images/memcached/monitoring/mc-alerting-prom-target.png new file mode 100644 index 0000000000..febc2b0e70 Binary files /dev/null and b/docs/images/memcached/monitoring/mc-alerting-prom-target.png differ diff --git a/docs/images/mongodb/monitoring/mongodb-alerting-alertmanager-firing.png b/docs/images/mongodb/monitoring/mongodb-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..8b44a7459b Binary files /dev/null and b/docs/images/mongodb/monitoring/mongodb-alerting-alertmanager-firing.png differ diff --git a/docs/images/mongodb/monitoring/mongodb-alerting-alertmanager.png b/docs/images/mongodb/monitoring/mongodb-alerting-alertmanager.png new file mode 100644 index 0000000000..90d0dbe2c6 Binary files /dev/null and b/docs/images/mongodb/monitoring/mongodb-alerting-alertmanager.png differ diff --git a/docs/images/mongodb/monitoring/mongodb-alerting-prom-alerts-firing.png b/docs/images/mongodb/monitoring/mongodb-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..2cc1fc04ac Binary files /dev/null and b/docs/images/mongodb/monitoring/mongodb-alerting-prom-alerts-firing.png differ diff --git a/docs/images/mongodb/monitoring/mongodb-alerting-prom-alerts.png b/docs/images/mongodb/monitoring/mongodb-alerting-prom-alerts.png new file mode 100644 index 0000000000..06e12abc81 Binary files /dev/null and b/docs/images/mongodb/monitoring/mongodb-alerting-prom-alerts.png differ diff --git a/docs/images/mongodb/monitoring/mongodb-alerting-prom-rules.png b/docs/images/mongodb/monitoring/mongodb-alerting-prom-rules.png new file mode 100644 index 0000000000..b602152f8f Binary files /dev/null and b/docs/images/mongodb/monitoring/mongodb-alerting-prom-rules.png differ diff --git a/docs/images/mongodb/monitoring/mongodb-alerting-prom-target.png b/docs/images/mongodb/monitoring/mongodb-alerting-prom-target.png new file mode 100644 index 0000000000..9088ffd7a0 Binary files /dev/null and b/docs/images/mongodb/monitoring/mongodb-alerting-prom-target.png differ diff --git a/docs/images/mssqlserver/monitoring/mssqlserver-alerting-alertmanager-firing.png b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..7fe0df3fa9 Binary files /dev/null and b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-alertmanager-firing.png differ diff --git a/docs/images/mssqlserver/monitoring/mssqlserver-alerting-alertmanager.png b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-alertmanager.png new file mode 100644 index 0000000000..e082c2b2f0 Binary files /dev/null and b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-alertmanager.png differ diff --git a/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-alerts-firing.png b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..41476470f6 Binary files /dev/null and b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-alerts-firing.png differ diff --git a/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-alerts.png b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-alerts.png new file mode 100644 index 0000000000..eb29fb6524 Binary files /dev/null and b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-alerts.png differ diff --git a/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-rules.png b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-rules.png new file mode 100644 index 0000000000..3bd014a5dc Binary files /dev/null and b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-rules.png differ diff --git a/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-target.png b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-target.png new file mode 100644 index 0000000000..225fb5e4a4 Binary files /dev/null and b/docs/images/mssqlserver/monitoring/mssqlserver-alerting-prom-target.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-alertmanager-firing.png b/docs/images/mysql/monitoring/mysql-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..b9f89f81eb Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-alertmanager-firing.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-alertmanager.png b/docs/images/mysql/monitoring/mysql-alerting-alertmanager.png new file mode 100644 index 0000000000..b96cef1d19 Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-alertmanager.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-grafana-dashboards.png b/docs/images/mysql/monitoring/mysql-alerting-grafana-dashboards.png new file mode 100644 index 0000000000..739047f083 Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-grafana-dashboards.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-grafana-database.png b/docs/images/mysql/monitoring/mysql-alerting-grafana-database.png new file mode 100644 index 0000000000..f7736ac4fd Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-grafana-database.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-grafana-group-replication.png b/docs/images/mysql/monitoring/mysql-alerting-grafana-group-replication.png new file mode 100644 index 0000000000..6067e3f51a Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-grafana-group-replication.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-grafana-pod.png b/docs/images/mysql/monitoring/mysql-alerting-grafana-pod.png new file mode 100644 index 0000000000..e407a35268 Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-grafana-pod.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-grafana-summary.png b/docs/images/mysql/monitoring/mysql-alerting-grafana-summary.png new file mode 100644 index 0000000000..7467cec3e8 Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-grafana-summary.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-overview.svg b/docs/images/mysql/monitoring/mysql-alerting-overview.svg new file mode 100644 index 0000000000..88cc31e24c --- /dev/null +++ b/docs/images/mysql/monitoring/mysql-alerting-overview.svg @@ -0,0 +1,98 @@ + + + + + + + + MySQL Group Replication + 3 pods, Single-Primary mode + + + + mysql + + + + mysqld + exporter (sidecar) + + + metrics on :56790/metrics + one target per pod (3 for this cluster) + + + + ServiceMonitor + mysql-alert-stats + + + + scrape + + + + Prometheus + kube-prometheus-stack + evaluates rules every 30s + ruleSelector: release=prometheus + + + + discovers + + + + PrometheusRule + mysql-alerts chart + release name = mysql-alert + + + + loads + + + + AlertManager + groups · inhibits · silences + routes to receivers + + + + fires alerts + + + + Slack + + + Email + + + PagerDuty / Webhook + + + + notify + + + + Grafana + kubedb-grafana-dashboards + Summary · Database · Pod · Group-Replication + + + + datasource + + + + + + + + + + MySQL Alerting Architecture with KubeDB + + diff --git a/docs/images/mysql/monitoring/mysql-alerting-prom-alerts-firing.png b/docs/images/mysql/monitoring/mysql-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..5d401ec297 Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-prom-alerts-firing.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-prom-alerts.png b/docs/images/mysql/monitoring/mysql-alerting-prom-alerts.png new file mode 100644 index 0000000000..f70ffd8422 Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-prom-alerts.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-prom-rules.png b/docs/images/mysql/monitoring/mysql-alerting-prom-rules.png new file mode 100644 index 0000000000..2566fa24fa Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-prom-rules.png differ diff --git a/docs/images/mysql/monitoring/mysql-alerting-prom-target.png b/docs/images/mysql/monitoring/mysql-alerting-prom-target.png new file mode 100644 index 0000000000..7c9c185f9f Binary files /dev/null and b/docs/images/mysql/monitoring/mysql-alerting-prom-target.png differ diff --git a/docs/images/neo4j/monitoring/neo4j-alerting-alertmanager-firing.png b/docs/images/neo4j/monitoring/neo4j-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..91dc49d614 Binary files /dev/null and b/docs/images/neo4j/monitoring/neo4j-alerting-alertmanager-firing.png differ diff --git a/docs/images/neo4j/monitoring/neo4j-alerting-alertmanager.png b/docs/images/neo4j/monitoring/neo4j-alerting-alertmanager.png new file mode 100644 index 0000000000..ef86afa81f Binary files /dev/null and b/docs/images/neo4j/monitoring/neo4j-alerting-alertmanager.png differ diff --git a/docs/images/neo4j/monitoring/neo4j-alerting-grafana-dashboard.png b/docs/images/neo4j/monitoring/neo4j-alerting-grafana-dashboard.png new file mode 100644 index 0000000000..53d610365d Binary files /dev/null and b/docs/images/neo4j/monitoring/neo4j-alerting-grafana-dashboard.png differ diff --git a/docs/images/neo4j/monitoring/neo4j-alerting-prom-alerts-firing.png b/docs/images/neo4j/monitoring/neo4j-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..92b7827fee Binary files /dev/null and b/docs/images/neo4j/monitoring/neo4j-alerting-prom-alerts-firing.png differ diff --git a/docs/images/neo4j/monitoring/neo4j-alerting-prom-alerts.png b/docs/images/neo4j/monitoring/neo4j-alerting-prom-alerts.png new file mode 100644 index 0000000000..7b27789f19 Binary files /dev/null and b/docs/images/neo4j/monitoring/neo4j-alerting-prom-alerts.png differ diff --git a/docs/images/neo4j/monitoring/neo4j-alerting-prom-rules.png b/docs/images/neo4j/monitoring/neo4j-alerting-prom-rules.png new file mode 100644 index 0000000000..8d75383a13 Binary files /dev/null and b/docs/images/neo4j/monitoring/neo4j-alerting-prom-rules.png differ diff --git a/docs/images/neo4j/monitoring/neo4j-alerting-prom-target.png b/docs/images/neo4j/monitoring/neo4j-alerting-prom-target.png new file mode 100644 index 0000000000..da14654664 Binary files /dev/null and b/docs/images/neo4j/monitoring/neo4j-alerting-prom-target.png differ diff --git a/docs/images/pgbouncer/monitoring/pgbouncer-alerting-alertmanager-firing.png b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..28913b6e3b Binary files /dev/null and b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-alertmanager-firing.png differ diff --git a/docs/images/pgbouncer/monitoring/pgbouncer-alerting-alertmanager.png b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-alertmanager.png new file mode 100644 index 0000000000..696e7afc2b Binary files /dev/null and b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-alertmanager.png differ diff --git a/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-alerts-firing.png b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..e5f581d631 Binary files /dev/null and b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-alerts-firing.png differ diff --git a/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-alerts.png b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-alerts.png new file mode 100644 index 0000000000..49e2e2c2ea Binary files /dev/null and b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-alerts.png differ diff --git a/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-rules.png b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-rules.png new file mode 100644 index 0000000000..9a46527ab5 Binary files /dev/null and b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-rules.png differ diff --git a/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-target.png b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-target.png new file mode 100644 index 0000000000..a714f241bd Binary files /dev/null and b/docs/images/pgbouncer/monitoring/pgbouncer-alerting-prom-target.png differ diff --git a/docs/images/pgpool/monitoring/pgpool-alerting-alertmanager-firing.png b/docs/images/pgpool/monitoring/pgpool-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..ddddd2dfb5 Binary files /dev/null and b/docs/images/pgpool/monitoring/pgpool-alerting-alertmanager-firing.png differ diff --git a/docs/images/pgpool/monitoring/pgpool-alerting-alertmanager.png b/docs/images/pgpool/monitoring/pgpool-alerting-alertmanager.png new file mode 100644 index 0000000000..3787bdb7fe Binary files /dev/null and b/docs/images/pgpool/monitoring/pgpool-alerting-alertmanager.png differ diff --git a/docs/images/pgpool/monitoring/pgpool-alerting-prom-alerts-firing.png b/docs/images/pgpool/monitoring/pgpool-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..5541bff935 Binary files /dev/null and b/docs/images/pgpool/monitoring/pgpool-alerting-prom-alerts-firing.png differ diff --git a/docs/images/pgpool/monitoring/pgpool-alerting-prom-alerts.png b/docs/images/pgpool/monitoring/pgpool-alerting-prom-alerts.png new file mode 100644 index 0000000000..e702e1560e Binary files /dev/null and b/docs/images/pgpool/monitoring/pgpool-alerting-prom-alerts.png differ diff --git a/docs/images/pgpool/monitoring/pgpool-alerting-prom-rules.png b/docs/images/pgpool/monitoring/pgpool-alerting-prom-rules.png new file mode 100644 index 0000000000..60ab75eeab Binary files /dev/null and b/docs/images/pgpool/monitoring/pgpool-alerting-prom-rules.png differ diff --git a/docs/images/pgpool/monitoring/pgpool-alerting-prom-target.png b/docs/images/pgpool/monitoring/pgpool-alerting-prom-target.png new file mode 100644 index 0000000000..d3f923ba97 Binary files /dev/null and b/docs/images/pgpool/monitoring/pgpool-alerting-prom-target.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-alertmanager-firing.png b/docs/images/postgres/monitoring/pg-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..7c05dce1bc Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-alertmanager-firing.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-alertmanager.png b/docs/images/postgres/monitoring/pg-alerting-alertmanager.png new file mode 100644 index 0000000000..804c730727 Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-alertmanager.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-grafana-dashboards.png b/docs/images/postgres/monitoring/pg-alerting-grafana-dashboards.png new file mode 100644 index 0000000000..2b8bf1590c Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-grafana-dashboards.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-grafana-database.png b/docs/images/postgres/monitoring/pg-alerting-grafana-database.png new file mode 100644 index 0000000000..1e90945690 Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-grafana-database.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-grafana-pod.png b/docs/images/postgres/monitoring/pg-alerting-grafana-pod.png new file mode 100644 index 0000000000..4976441dfc Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-grafana-pod.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-grafana-summary.png b/docs/images/postgres/monitoring/pg-alerting-grafana-summary.png new file mode 100644 index 0000000000..64970aa2c3 Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-grafana-summary.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-overview.svg b/docs/images/postgres/monitoring/pg-alerting-overview.svg new file mode 100644 index 0000000000..160357bdc5 --- /dev/null +++ b/docs/images/postgres/monitoring/pg-alerting-overview.svg @@ -0,0 +1,96 @@ + + + + + + + + PostgreSQL Pod + + + + postgres + + + + postgres + exporter + + + metrics on :56790/metrics + + + + ServiceMonitor + pg-grafana-demo-stats + + + + scrape + + + + Prometheus + kube-prometheus-stack + evaluates rules every 30s + ruleSelector: release=prometheus + + + + discovers + + + + PrometheusRule + postgres-alerts chart + label: release=prometheus + + + + loads + + + + AlertManager + groups · inhibits · silences + routes to receivers + + + + fires alerts + + + + Slack + + + Email + + + PagerDuty / Webhook + + + + notify + + + + Grafana + kubedb-grafana-dashboards + Summary · Database · Pod + + + + datasource + + + + + + + + + + PostgreSQL Alerting Architecture with KubeDB + + diff --git a/docs/images/postgres/monitoring/pg-alerting-prom-alerts-firing.png b/docs/images/postgres/monitoring/pg-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..55ff72dbdb Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-prom-alerts-firing.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-prom-alerts.png b/docs/images/postgres/monitoring/pg-alerting-prom-alerts.png new file mode 100644 index 0000000000..c37ec5fd7e Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-prom-alerts.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-prom-rules.png b/docs/images/postgres/monitoring/pg-alerting-prom-rules.png new file mode 100644 index 0000000000..971139e9e9 Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-prom-rules.png differ diff --git a/docs/images/postgres/monitoring/pg-alerting-prom-target.png b/docs/images/postgres/monitoring/pg-alerting-prom-target.png new file mode 100644 index 0000000000..b348a7c2ff Binary files /dev/null and b/docs/images/postgres/monitoring/pg-alerting-prom-target.png differ diff --git a/docs/images/proxysql/monitoring/proxysql-alerting-alertmanager-firing.png b/docs/images/proxysql/monitoring/proxysql-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..edfdd183b8 Binary files /dev/null and b/docs/images/proxysql/monitoring/proxysql-alerting-alertmanager-firing.png differ diff --git a/docs/images/proxysql/monitoring/proxysql-alerting-alertmanager.png b/docs/images/proxysql/monitoring/proxysql-alerting-alertmanager.png new file mode 100644 index 0000000000..0a26d72085 Binary files /dev/null and b/docs/images/proxysql/monitoring/proxysql-alerting-alertmanager.png differ diff --git a/docs/images/proxysql/monitoring/proxysql-alerting-prom-alerts-firing.png b/docs/images/proxysql/monitoring/proxysql-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..ba3252f994 Binary files /dev/null and b/docs/images/proxysql/monitoring/proxysql-alerting-prom-alerts-firing.png differ diff --git a/docs/images/proxysql/monitoring/proxysql-alerting-prom-alerts.png b/docs/images/proxysql/monitoring/proxysql-alerting-prom-alerts.png new file mode 100644 index 0000000000..07f7c436a4 Binary files /dev/null and b/docs/images/proxysql/monitoring/proxysql-alerting-prom-alerts.png differ diff --git a/docs/images/proxysql/monitoring/proxysql-alerting-prom-rules.png b/docs/images/proxysql/monitoring/proxysql-alerting-prom-rules.png new file mode 100644 index 0000000000..8f1f0c6a66 Binary files /dev/null and b/docs/images/proxysql/monitoring/proxysql-alerting-prom-rules.png differ diff --git a/docs/images/proxysql/monitoring/proxysql-alerting-prom-target.png b/docs/images/proxysql/monitoring/proxysql-alerting-prom-target.png new file mode 100644 index 0000000000..9394993330 Binary files /dev/null and b/docs/images/proxysql/monitoring/proxysql-alerting-prom-target.png differ diff --git a/docs/images/qdrant/monitoring/qd-alerting-alertmanager-firing.png b/docs/images/qdrant/monitoring/qd-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..932a9f493a Binary files /dev/null and b/docs/images/qdrant/monitoring/qd-alerting-alertmanager-firing.png differ diff --git a/docs/images/qdrant/monitoring/qd-alerting-prom-alerts-firing.png b/docs/images/qdrant/monitoring/qd-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..59d330d963 Binary files /dev/null and b/docs/images/qdrant/monitoring/qd-alerting-prom-alerts-firing.png differ diff --git a/docs/images/qdrant/monitoring/qd-alerting-prom-alerts.png b/docs/images/qdrant/monitoring/qd-alerting-prom-alerts.png new file mode 100644 index 0000000000..46bdbfd81f Binary files /dev/null and b/docs/images/qdrant/monitoring/qd-alerting-prom-alerts.png differ diff --git a/docs/images/qdrant/monitoring/qd-alerting-prom-rules.png b/docs/images/qdrant/monitoring/qd-alerting-prom-rules.png new file mode 100644 index 0000000000..25ac86e8a6 Binary files /dev/null and b/docs/images/qdrant/monitoring/qd-alerting-prom-rules.png differ diff --git a/docs/images/qdrant/monitoring/qd-alerting-prom-target.png b/docs/images/qdrant/monitoring/qd-alerting-prom-target.png new file mode 100644 index 0000000000..12518e7b05 Binary files /dev/null and b/docs/images/qdrant/monitoring/qd-alerting-prom-target.png differ diff --git a/docs/images/rabbitmq/monitoring/rmq-alerting-alertmanager-firing.png b/docs/images/rabbitmq/monitoring/rmq-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..b202ff6d5a Binary files /dev/null and b/docs/images/rabbitmq/monitoring/rmq-alerting-alertmanager-firing.png differ diff --git a/docs/images/rabbitmq/monitoring/rmq-alerting-prom-alerts-firing.png b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..0c98864e92 Binary files /dev/null and b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-alerts-firing.png differ diff --git a/docs/images/rabbitmq/monitoring/rmq-alerting-prom-alerts.png b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-alerts.png new file mode 100644 index 0000000000..27b1412d3f Binary files /dev/null and b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-alerts.png differ diff --git a/docs/images/rabbitmq/monitoring/rmq-alerting-prom-rules.png b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-rules.png new file mode 100644 index 0000000000..674d8c8035 Binary files /dev/null and b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-rules.png differ diff --git a/docs/images/rabbitmq/monitoring/rmq-alerting-prom-target.png b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-target.png new file mode 100644 index 0000000000..f425907642 Binary files /dev/null and b/docs/images/rabbitmq/monitoring/rmq-alerting-prom-target.png differ diff --git a/docs/images/redis/monitoring/rd-alerting-alertmanager-firing.png b/docs/images/redis/monitoring/rd-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..943deb6f45 Binary files /dev/null and b/docs/images/redis/monitoring/rd-alerting-alertmanager-firing.png differ diff --git a/docs/images/redis/monitoring/rd-alerting-alertmanager.png b/docs/images/redis/monitoring/rd-alerting-alertmanager.png new file mode 100644 index 0000000000..ade5c7a552 Binary files /dev/null and b/docs/images/redis/monitoring/rd-alerting-alertmanager.png differ diff --git a/docs/images/redis/monitoring/rd-alerting-prom-alerts-firing.png b/docs/images/redis/monitoring/rd-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..38266d0e4e Binary files /dev/null and b/docs/images/redis/monitoring/rd-alerting-prom-alerts-firing.png differ diff --git a/docs/images/redis/monitoring/rd-alerting-prom-alerts.png b/docs/images/redis/monitoring/rd-alerting-prom-alerts.png new file mode 100644 index 0000000000..da82896062 Binary files /dev/null and b/docs/images/redis/monitoring/rd-alerting-prom-alerts.png differ diff --git a/docs/images/redis/monitoring/rd-alerting-prom-rules.png b/docs/images/redis/monitoring/rd-alerting-prom-rules.png new file mode 100644 index 0000000000..817d033f3b Binary files /dev/null and b/docs/images/redis/monitoring/rd-alerting-prom-rules.png differ diff --git a/docs/images/redis/monitoring/rd-alerting-prom-target.png b/docs/images/redis/monitoring/rd-alerting-prom-target.png new file mode 100644 index 0000000000..4d4f6dd278 Binary files /dev/null and b/docs/images/redis/monitoring/rd-alerting-prom-target.png differ diff --git a/docs/images/singlestore/monitoring/singlestore-alerting-alertmanager-firing.png b/docs/images/singlestore/monitoring/singlestore-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..c46a394fe9 Binary files /dev/null and b/docs/images/singlestore/monitoring/singlestore-alerting-alertmanager-firing.png differ diff --git a/docs/images/singlestore/monitoring/singlestore-alerting-alertmanager.png b/docs/images/singlestore/monitoring/singlestore-alerting-alertmanager.png new file mode 100644 index 0000000000..9d5de85c26 Binary files /dev/null and b/docs/images/singlestore/monitoring/singlestore-alerting-alertmanager.png differ diff --git a/docs/images/singlestore/monitoring/singlestore-alerting-prom-alerts-firing.png b/docs/images/singlestore/monitoring/singlestore-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..9d1deb356a Binary files /dev/null and b/docs/images/singlestore/monitoring/singlestore-alerting-prom-alerts-firing.png differ diff --git a/docs/images/singlestore/monitoring/singlestore-alerting-prom-alerts.png b/docs/images/singlestore/monitoring/singlestore-alerting-prom-alerts.png new file mode 100644 index 0000000000..f3cb89523c Binary files /dev/null and b/docs/images/singlestore/monitoring/singlestore-alerting-prom-alerts.png differ diff --git a/docs/images/singlestore/monitoring/singlestore-alerting-prom-rules.png b/docs/images/singlestore/monitoring/singlestore-alerting-prom-rules.png new file mode 100644 index 0000000000..6c122ad12a Binary files /dev/null and b/docs/images/singlestore/monitoring/singlestore-alerting-prom-rules.png differ diff --git a/docs/images/singlestore/monitoring/singlestore-alerting-prom-target.png b/docs/images/singlestore/monitoring/singlestore-alerting-prom-target.png new file mode 100644 index 0000000000..5682882e79 Binary files /dev/null and b/docs/images/singlestore/monitoring/singlestore-alerting-prom-target.png differ diff --git a/docs/images/solr/monitoring/solr-alerting-alertmanager-firing.png b/docs/images/solr/monitoring/solr-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..ed191b2c66 Binary files /dev/null and b/docs/images/solr/monitoring/solr-alerting-alertmanager-firing.png differ diff --git a/docs/images/solr/monitoring/solr-alerting-alertmanager.png b/docs/images/solr/monitoring/solr-alerting-alertmanager.png new file mode 100644 index 0000000000..b1b36ca35e Binary files /dev/null and b/docs/images/solr/monitoring/solr-alerting-alertmanager.png differ diff --git a/docs/images/solr/monitoring/solr-alerting-prom-alerts-firing.png b/docs/images/solr/monitoring/solr-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..ebe4a71123 Binary files /dev/null and b/docs/images/solr/monitoring/solr-alerting-prom-alerts-firing.png differ diff --git a/docs/images/solr/monitoring/solr-alerting-prom-alerts.png b/docs/images/solr/monitoring/solr-alerting-prom-alerts.png new file mode 100644 index 0000000000..952f5939b7 Binary files /dev/null and b/docs/images/solr/monitoring/solr-alerting-prom-alerts.png differ diff --git a/docs/images/solr/monitoring/solr-alerting-prom-rules.png b/docs/images/solr/monitoring/solr-alerting-prom-rules.png new file mode 100644 index 0000000000..ca164d964e Binary files /dev/null and b/docs/images/solr/monitoring/solr-alerting-prom-rules.png differ diff --git a/docs/images/solr/monitoring/solr-alerting-prom-target.png b/docs/images/solr/monitoring/solr-alerting-prom-target.png new file mode 100644 index 0000000000..a6e450f463 Binary files /dev/null and b/docs/images/solr/monitoring/solr-alerting-prom-target.png differ diff --git a/docs/images/zookeeper/monitoring/zk-alerting-alertmanager-firing.png b/docs/images/zookeeper/monitoring/zk-alerting-alertmanager-firing.png new file mode 100644 index 0000000000..8490ce621d Binary files /dev/null and b/docs/images/zookeeper/monitoring/zk-alerting-alertmanager-firing.png differ diff --git a/docs/images/zookeeper/monitoring/zk-alerting-prom-alerts-firing.png b/docs/images/zookeeper/monitoring/zk-alerting-prom-alerts-firing.png new file mode 100644 index 0000000000..da05fe63d1 Binary files /dev/null and b/docs/images/zookeeper/monitoring/zk-alerting-prom-alerts-firing.png differ diff --git a/docs/images/zookeeper/monitoring/zk-alerting-prom-alerts.png b/docs/images/zookeeper/monitoring/zk-alerting-prom-alerts.png new file mode 100644 index 0000000000..978e2daa05 Binary files /dev/null and b/docs/images/zookeeper/monitoring/zk-alerting-prom-alerts.png differ diff --git a/docs/images/zookeeper/monitoring/zk-alerting-prom-rules.png b/docs/images/zookeeper/monitoring/zk-alerting-prom-rules.png new file mode 100644 index 0000000000..08c3b1dd94 Binary files /dev/null and b/docs/images/zookeeper/monitoring/zk-alerting-prom-rules.png differ diff --git a/docs/images/zookeeper/monitoring/zk-alerting-prom-target.png b/docs/images/zookeeper/monitoring/zk-alerting-prom-target.png new file mode 100644 index 0000000000..22ed8eabac Binary files /dev/null and b/docs/images/zookeeper/monitoring/zk-alerting-prom-target.png differ