From 5e0727c35170682f14db6d67889ecc243dbb239a Mon Sep 17 00:00:00 2001 From: Smana Date: Mon, 21 Sep 2026 23:14:15 +0200 Subject: [PATCH 1/3] chore(observability): vendor kubernetes-event-exporter and drop the Bitnami source The image already ran a community fork (ghcr.io/civitatis/kubernetes-event-exporter); only the chart itself still came from bitnami's OCI HelmRepository. Neither maintained fork chart (itakurah, ownkube) clears this repo's bar - one has no resources field and an unfixed template bug, the other is an unverified rewrite - and upstream ships no chart at all, so the chart's own rendered output is vendored as plain manifests instead. See ADR-0040. Closes the last item of #1089. --- flux/sources/helmrepo-bitnami.yaml | 9 -- .../clusterrole.yaml | 16 +++ .../clusterrolebinding.yaml | 15 +++ .../kubernetes-event-exporter/configmap.yaml | 35 ++++++ .../kubernetes-event-exporter/deployment.yaml | 100 ++++++++++++++++++ .../helmrelease.yaml | 67 ------------ .../kustomization.yaml | 10 +- .../networkpolicy.yaml | 20 ++++ .../base/kubernetes-event-exporter/pdb.yaml | 13 +++ .../kubernetes-event-exporter/service.yaml | 22 ++++ .../serviceaccount.yaml | 8 ++ .../servicemonitor.yaml | 21 ++++ ...dor-kubernetes-event-exporter-manifests.md | 55 ++++++++++ website/content/docs/decisions/_index.md | 1 + .../docs/platform/observability/logs.md | 14 +-- 15 files changed, 323 insertions(+), 83 deletions(-) delete mode 100644 flux/sources/helmrepo-bitnami.yaml create mode 100644 observability/base/kubernetes-event-exporter/clusterrole.yaml create mode 100644 observability/base/kubernetes-event-exporter/clusterrolebinding.yaml create mode 100644 observability/base/kubernetes-event-exporter/configmap.yaml create mode 100644 observability/base/kubernetes-event-exporter/deployment.yaml delete mode 100644 observability/base/kubernetes-event-exporter/helmrelease.yaml create mode 100644 observability/base/kubernetes-event-exporter/networkpolicy.yaml create mode 100644 observability/base/kubernetes-event-exporter/pdb.yaml create mode 100644 observability/base/kubernetes-event-exporter/service.yaml create mode 100644 observability/base/kubernetes-event-exporter/serviceaccount.yaml create mode 100644 observability/base/kubernetes-event-exporter/servicemonitor.yaml create mode 100644 website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md diff --git a/flux/sources/helmrepo-bitnami.yaml b/flux/sources/helmrepo-bitnami.yaml deleted file mode 100644 index 2a3dfb04c..000000000 --- a/flux/sources/helmrepo-bitnami.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: bitnami - namespace: flux-system -spec: - type: "oci" - interval: 5m - url: oci://registry-1.docker.io/bitnamicharts diff --git a/observability/base/kubernetes-event-exporter/clusterrole.yaml b/observability/base/kubernetes-event-exporter/clusterrole.yaml new file mode 100644 index 000000000..ad07060e7 --- /dev/null +++ b/observability/base/kubernetes-event-exporter/clusterrole.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubernetes-event-exporter-observability + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter +rules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - watch + - list diff --git a/observability/base/kubernetes-event-exporter/clusterrolebinding.yaml b/observability/base/kubernetes-event-exporter/clusterrolebinding.yaml new file mode 100644 index 000000000..48fbf9895 --- /dev/null +++ b/observability/base/kubernetes-event-exporter/clusterrolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-event-exporter-observability + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-event-exporter-observability +subjects: + - kind: ServiceAccount + name: kubernetes-event-exporter + namespace: observability diff --git a/observability/base/kubernetes-event-exporter/configmap.yaml b/observability/base/kubernetes-event-exporter/configmap.yaml new file mode 100644 index 000000000..79c9f4145 --- /dev/null +++ b/observability/base/kubernetes-event-exporter/configmap.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter +data: + # No "dump" file receiver: with layout: {} (empty-but-present) the fork's + # file sink takes the convertLayoutTemplate branch over zero keys and emits + # contentless "{}" lines (pkg/sinks/file.go) — every cluster Event became a + # noise line that Vector's unfiltered kubernetes_logs source re-ingested + # into VictoriaLogs. It was never a fallback either: if the informer stops, + # no receiver gets anything. Events now flow solely via the loki receiver. + # + # The loki URL below targets vlsingle (see observability/base/victoria-logs/), + # the deployed variant — it pointed at the vlcluster vlinsert Service (never + # deployed here) from 2025-08-23 to 2026-08-29, so event push silently went + # nowhere the whole time. + config.yaml: | + clusterName: ${cluster_name} + leaderElection: {} + logFormat: json + logLevel: info + metricsNamePrefix: event_exporter_ + receivers: + - loki: + streamLabels: + source: kubernetes-event-exporter + url: http://victoria-logs-victoria-logs-single-server.observability.svc.cluster.local:9428/insert/loki/api/v1/push + name: victorialogs + route: + routes: + - match: + - receiver: victorialogs diff --git a/observability/base/kubernetes-event-exporter/deployment.yaml b/observability/base/kubernetes-event-exporter/deployment.yaml new file mode 100644 index 000000000..09dee7c14 --- /dev/null +++ b/observability/base/kubernetes-event-exporter/deployment.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + app.kubernetes.io/component: kubernetes-event-exporter +spec: + replicas: 1 + revisionHistoryLimit: 10 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + template: + metadata: + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + app.kubernetes.io/component: kubernetes-event-exporter + spec: + serviceAccountName: kubernetes-event-exporter + automountServiceAccountToken: true + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + app.kubernetes.io/component: kubernetes-event-exporter + topologyKey: kubernetes.io/hostname + weight: 1 + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + containers: + - name: event-exporter + # Renovate: kubernetes manager tracks this tag (.github/renovate.json). + image: ghcr.io/civitatis/kubernetes-event-exporter:1.8 + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + args: + - -conf=/data/config.yaml + livenessProbe: + failureThreshold: 5 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 2 + httpGet: + path: /-/healthy + port: http + readinessProbe: + failureThreshold: 1 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 2 + httpGet: + path: /-/ready + port: http + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi + ports: + - name: http + containerPort: 2112 + protocol: TCP + volumeMounts: + - mountPath: /data + name: cfg + volumes: + - name: cfg + configMap: + name: kubernetes-event-exporter diff --git a/observability/base/kubernetes-event-exporter/helmrelease.yaml b/observability/base/kubernetes-event-exporter/helmrelease.yaml deleted file mode 100644 index 42ced494b..000000000 --- a/observability/base/kubernetes-event-exporter/helmrelease.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: kubernetes-event-exporter -spec: - interval: 30m - driftDetection: - mode: enabled - chart: - spec: - chart: kubernetes-event-exporter - version: "3.6.3" - sourceRef: - kind: HelmRepository - name: bitnami - namespace: flux-system - interval: 12h - values: - global: - security: - allowInsecureImages: true - # Using a custom image from https://github.com/civitatis/kubernetes-event-exporter because of this issue: https://github.com/civitatis/kubernetes-event-exporter - image: - registry: ghcr.io - repository: civitatis/kubernetes-event-exporter - tag: "1.8" - config: - logLevel: info - logFormat: json - metricsNamePrefix: "event_exporter_" - clusterName: "${cluster_name}" - receivers: - # No "dump" file receiver: with layout: {} (empty-but-present) the fork's - # file sink takes the convertLayoutTemplate branch over zero keys and emits - # contentless "{}" lines (pkg/sinks/file.go) — every cluster Event became a - # noise line that Vector's unfiltered kubernetes_logs source re-ingested - # into VictoriaLogs. It was never a fallback either: if the informer stops, - # no receiver gets anything. Events now flow solely via the loki receiver. - - name: "victorialogs" - loki: - # vlsingle is the deployed variant (see observability/base/victoria-logs/). - # This URL pointed at the vlcluster vlinsert Service from 2025-08-23 to - # 2026-08-29 — a Service that never existed here — so direct event push - # silently went nowhere the whole time. - url: "http://victoria-logs-victoria-logs-single-server.observability.svc.cluster.local:9428/insert/loki/api/v1/push" - streamLabels: - source: kubernetes-event-exporter - - route: - routes: - - match: - - receiver: "victorialogs" - - metrics: - enabled: true - - # Chart-native discovery object; the VM operator's Prometheus converter - # turns it into a VMServiceScrape. - serviceMonitor: - enabled: true - namespace: "observability" - - # The watch-errors alert is platform-authored, not a chart default, so it - # lives in ./vmrule.yaml like every other alert in this repo — not as a - # values-embedded PrometheusRule. - prometheusRule: - enabled: false diff --git a/observability/base/kubernetes-event-exporter/kustomization.yaml b/observability/base/kubernetes-event-exporter/kustomization.yaml index 3fece8645..dd460f8bc 100644 --- a/observability/base/kubernetes-event-exporter/kustomization.yaml +++ b/observability/base/kubernetes-event-exporter/kustomization.yaml @@ -2,5 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: observability resources: - - helmrelease.yaml + - serviceaccount.yaml + - configmap.yaml + - clusterrole.yaml + - clusterrolebinding.yaml + - deployment.yaml + - service.yaml + - servicemonitor.yaml + - networkpolicy.yaml + - pdb.yaml - vmrule.yaml diff --git a/observability/base/kubernetes-event-exporter/networkpolicy.yaml b/observability/base/kubernetes-event-exporter/networkpolicy.yaml new file mode 100644 index 000000000..48aa4f8c7 --- /dev/null +++ b/observability/base/kubernetes-event-exporter/networkpolicy.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + policyTypes: + - Ingress + - Egress + egress: + - {} + ingress: + - ports: + - port: 2112 diff --git a/observability/base/kubernetes-event-exporter/pdb.yaml b/observability/base/kubernetes-event-exporter/pdb.yaml new file mode 100644 index 000000000..35e976da2 --- /dev/null +++ b/observability/base/kubernetes-event-exporter/pdb.yaml @@ -0,0 +1,13 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter diff --git a/observability/base/kubernetes-event-exporter/service.yaml b/observability/base/kubernetes-event-exporter/service.yaml new file mode 100644 index 000000000..7e998f22a --- /dev/null +++ b/observability/base/kubernetes-event-exporter/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-event-exporter-metrics + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + app.kubernetes.io/component: metrics + annotations: + prometheus.io/port: "2112" + prometheus.io/scrape: "true" +spec: + type: ClusterIP + ports: + - name: http + port: 2112 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + app.kubernetes.io/component: kubernetes-event-exporter diff --git a/observability/base/kubernetes-event-exporter/serviceaccount.yaml b/observability/base/kubernetes-event-exporter/serviceaccount.yaml new file mode 100644 index 000000000..8886e14ed --- /dev/null +++ b/observability/base/kubernetes-event-exporter/serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter +automountServiceAccountToken: false diff --git a/observability/base/kubernetes-event-exporter/servicemonitor.yaml b/observability/base/kubernetes-event-exporter/servicemonitor.yaml new file mode 100644 index 000000000..ff15d565c --- /dev/null +++ b/observability/base/kubernetes-event-exporter/servicemonitor.yaml @@ -0,0 +1,21 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kubernetes-event-exporter + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + app.kubernetes.io/component: kubernetes-event-exporter +spec: + endpoints: + - port: http + path: /metrics + interval: 30s + namespaceSelector: + matchNames: + - observability + selector: + matchLabels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter + app.kubernetes.io/component: metrics diff --git a/website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md b/website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md new file mode 100644 index 000000000..f1062cbe3 --- /dev/null +++ b/website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md @@ -0,0 +1,55 @@ +--- +title: Vendor kubernetes-event-exporter as plain manifests instead of a Helm chart +linkTitle: 0040 · Vendor kubernetes-event-exporter manifests +weight: 400 +description: The last Bitnami dependency (issue #1089) is the kubernetes-event-exporter chart itself — the image already moved to a community fork. No maintained chart exists that both wraps the same binary and meets this repo's resource-limits bar, so the chart's own rendered output is vendored as plain manifests and the bitnami HelmRepository is deleted. Consequence: no chart to upgrade, config edits are hand edits, and only the image tag stays Renovate-tracked. +lastVerified: 2026-09-21 +--- + +**Status**: Accepted +**Date**: 2026-09-21 +**Deciders**: Smana (Platform Owner) + +--- + +## Context + +Issue #1089 tracks migrating every Bitnami-sourced dependency off `oci://registry-1.docker.io/bitnamicharts`, +ahead of Broadcom's `bitnamilegacy` deprecation. `kubernetes-event-exporter` was the last one: the +container image already runs `ghcr.io/civitatis/kubernetes-event-exporter:1.8` (a community fork), +with `global.security.allowInsecureImages: true` set so the Bitnami chart accepts a non-Bitnami +image. Only the chart itself — `kubernetes-event-exporter` v3.6.3, sourced from the `bitnami` +`HelmRepository` — was still Bitnami's. + +No comment on #1089 names a mandated replacement chart. Two independent Helm charts exist on +ArtifactHub; both were rendered with this repo's values (`helm template`) and diffed against the +Bitnami baseline before this decision. + +## Decision + +Vendor the Bitnami chart's own rendered output as plain manifests under +`observability/base/kubernetes-event-exporter/` (`Deployment`, `ConfigMap`, `Service`, +`ServiceMonitor`, `NetworkPolicy`, `PodDisruptionBudget`, RBAC), stripped of Helm-only metadata, and +delete `flux/sources/helmrepo-bitnami.yaml`. `${cluster_name}` stays in the `ConfigMap` for Flux +`postBuild` substitution, unchanged from how the `HelmRelease` fed it. + +## Alternatives rejected + +| Option | Rejected because | +|---|---| +| **`itakurah/kubernetes-event-exporter` chart (0.2.3)** | Wraps the same upstream binary this repo already runs, but ships no `resources` field anywhere in values or templates — violates this repo's requests+limits non-negotiable and would need a `postRenderers` JSON6902 patch to add them. Also carries an open, unfixed template bug (an all-hex 8-char config checksum can parse as a YAML number and reject the apply), unpatched for 6+ months. | +| **`ownkube/kubernetes-events-exporter` chart (0.1.2)** | A full Go rewrite, not the resmoio/mustafaakin codebase this repo runs — `loki` receiver and `route.match` compatibility is unverified, not just untested. Its apparent activity doesn't hold up either: no human-merged commit in 5 months, only 11 open Dependabot PRs. | +| **Keep pulling the Bitnami OCI chart** | Exactly what #1089 asks to stop. Sibling components (Valkey, RabbitMQ images) are already mid-migration off Bitnami, confirming the source itself is on a deprecation path, not just this one chart. | +| **Use the real upstream chart** | Doesn't exist. The upstream repo (`resmoio`, now pushed under `mustafaakin/kubernetes-event-exporter`) ships a raw `deploy/` manifest directory, not a Helm chart. | + +## Consequences + +- **No chart to upgrade.** A future exporter version means re-rendering and re-diffing by hand, not + bumping a chart version. +- **Config changes are hand edits.** `config.yaml` (routes, receivers, log settings) is now a + committed `ConfigMap` literal, edited directly instead of through `values.yaml`. +- **Only the image tag is Renovate-tracked**, via the repo-wide `kubernetes` manager + (`.github/renovate.json`) that already scans every manifest outside `clusters/` and `opentofu/`. +- **No checksum-triggered rollout.** The Helm chart's `checksum/config` pod annotation was dropped + with the rest of the Helm-only metadata; a `config.yaml` edit no longer forces a new + `ReplicaSet` on its own and needs an explicit rollout restart. diff --git a/website/content/docs/decisions/_index.md b/website/content/docs/decisions/_index.md index a01615362..dca81a92d 100644 --- a/website/content/docs/decisions/_index.md +++ b/website/content/docs/decisions/_index.md @@ -82,5 +82,6 @@ single-file fixes never need one. | [0037]({{< relref "/docs/decisions/0037-alertmanager-native-slack-templates.md" >}}) | Slack notifications are rendered by Alertmanager's own templates, not by a Block Kit bridge | Accepted | 2026-09-12 | | [0038]({{< relref "/docs/decisions/0038-agent-instructions-in-agents-md.md" >}}) | Agent instructions and skills are authored once in the open formats, with the Claude-specific paths as symlinks | Accepted | 2026-09-17 | | [0039]({{< relref "/docs/decisions/0039-go-task-as-the-entry-point.md" >}}) | go-task is the entry point to the scripts, locally and in CI, and no script depends on it | Accepted | 2026-09-17 | +| [0040]({{< relref "/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md" >}}) | Vendor kubernetes-event-exporter as plain manifests instead of a Helm chart | Accepted | 2026-09-21 | Starting a new one? Copy the [template]({{< relref "/docs/decisions/template.md" >}}). diff --git a/website/content/docs/platform/observability/logs.md b/website/content/docs/platform/observability/logs.md index 82137f81b..8df9a1dc0 100644 --- a/website/content/docs/platform/observability/logs.md +++ b/website/content/docs/platform/observability/logs.md @@ -98,7 +98,7 @@ queries that silently return nothing, not an error: Watches Kubernetes Events cluster-wide (`clusterName: "${cluster_name}"` tag) and pushes every event to the deployed vlsingle's Loki-compatible endpoint — `victoria-logs-victoria-logs-single-server` on port 9428 -(`observability/base/kubernetes-event-exporter/helmrelease.yaml`). That is +(`observability/base/kubernetes-event-exporter/configmap.yaml`). That is the **only** path events take, and it works as of 2026-08-29. Both halves of that sentence earn their date. From 2025-08-23 to 2026-08-29 @@ -114,16 +114,18 @@ exporter's own operational logs are structured now as well Its metrics are real for the first time: `metrics.enabled: true` — it was `false`, silently discarding the `serviceMonitor.enabled: true` nested under -it — with a chart-native `ServiceMonitor` in `observability` and one repaired -alert, `KubernetesEventExporterWatchErrors` (`severity: warning`, sustained +it — with a `ServiceMonitor` in `observability` and one repaired alert, +`KubernetesEventExporterWatchErrors` (`severity: warning`, sustained `rate > 0` for 15m), authored as a standalone `VMRule` (`observability/base/kubernetes-event-exporter/vmrule.yaml`) like every other alert in this repository. The alert's message deliberately names no namespace: the fork registers `WatchErrors` as a labelless counter, so the previous per-namespace grouping could never have -matched anything. Image stays overridden to -`ghcr.io/civitatis/kubernetes-event-exporter:1.8`, a community fork, not the -Bitnami-published image the chart normally pulls. +matched anything. Image stays `ghcr.io/civitatis/kubernetes-event-exporter:1.8`, +a community fork. The chart that pulled it is gone entirely: Bitnami's is +dropped, and its rendered output is vendored as plain manifests under +`observability/base/kubernetes-event-exporter/` +([ADR-0040]({{< relref "/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md" >}})). ## loggen From 1b52e852fd383c5844005f2ce86a905b4346685d Mon Sep 17 00:00:00 2001 From: Smana Date: Mon, 21 Sep 2026 23:34:08 +0200 Subject: [PATCH 2/3] fix(docs): correct ADR-0040 review findings and fix a broken doc path verify-doc-paths.sh failed on the ADR's backticked flux/sources/helmrepo-bitnami.yaml, deleted in the previous commit. Also: ownkube was never rendered/diffed (it's a rewrite, judged on source and maintenance record instead), Valkey and RabbitMQ are not "mid-migration" off Bitnami (Valkey is done per SPEC-012, RabbitMQ's only user was removed with grafana-oncall), the image tag is Renovate-tracked for the first time rather than merely "only" tracked, and logs.md still described chart values (metrics.enabled/serviceMonitor.enabled) that no longer exist. --- ...dor-kubernetes-event-exporter-manifests.md | 28 +++++++++++-------- .../docs/platform/observability/logs.md | 12 ++++---- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md b/website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md index f1062cbe3..f03d0446b 100644 --- a/website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md +++ b/website/content/docs/decisions/0040-vendor-kubernetes-event-exporter-manifests.md @@ -2,7 +2,7 @@ title: Vendor kubernetes-event-exporter as plain manifests instead of a Helm chart linkTitle: 0040 · Vendor kubernetes-event-exporter manifests weight: 400 -description: The last Bitnami dependency (issue #1089) is the kubernetes-event-exporter chart itself — the image already moved to a community fork. No maintained chart exists that both wraps the same binary and meets this repo's resource-limits bar, so the chart's own rendered output is vendored as plain manifests and the bitnami HelmRepository is deleted. Consequence: no chart to upgrade, config edits are hand edits, and only the image tag stays Renovate-tracked. +description: The last Bitnami dependency (issue #1089) is the kubernetes-event-exporter chart itself — the image already moved to a community fork. No maintained chart exists that both wraps the same binary and meets this repo's resource-limits bar, so the chart's own rendered output is vendored as plain manifests and the bitnami HelmRepository is deleted. Consequence: no chart to upgrade, config edits are hand edits, and the image tag is Renovate-tracked for the first time. lastVerified: 2026-09-21 --- @@ -22,16 +22,18 @@ image. Only the chart itself — `kubernetes-event-exporter` v3.6.3, sourced fro `HelmRepository` — was still Bitnami's. No comment on #1089 names a mandated replacement chart. Two independent Helm charts exist on -ArtifactHub; both were rendered with this repo's values (`helm template`) and diffed against the -Bitnami baseline before this decision. +ArtifactHub. `itakurah`'s was rendered with this repo's values (`helm template`) and diffed against +the Bitnami baseline; `ownkube`'s is a full rewrite of a different codebase, so it was judged on its +source and maintenance record instead — a `helm template` diff can't show whether a reimplemented +`loki` receiver behaves the same as the one already running. ## Decision Vendor the Bitnami chart's own rendered output as plain manifests under `observability/base/kubernetes-event-exporter/` (`Deployment`, `ConfigMap`, `Service`, `ServiceMonitor`, `NetworkPolicy`, `PodDisruptionBudget`, RBAC), stripped of Helm-only metadata, and -delete `flux/sources/helmrepo-bitnami.yaml`. `${cluster_name}` stays in the `ConfigMap` for Flux -`postBuild` substitution, unchanged from how the `HelmRelease` fed it. +delete the `bitnami` `HelmRepository` under `flux/sources/`. `${cluster_name}` stays in the +`ConfigMap` for Flux `postBuild` substitution, unchanged from how the `HelmRelease` fed it. ## Alternatives rejected @@ -39,7 +41,7 @@ delete `flux/sources/helmrepo-bitnami.yaml`. `${cluster_name}` stays in the `Con |---|---| | **`itakurah/kubernetes-event-exporter` chart (0.2.3)** | Wraps the same upstream binary this repo already runs, but ships no `resources` field anywhere in values or templates — violates this repo's requests+limits non-negotiable and would need a `postRenderers` JSON6902 patch to add them. Also carries an open, unfixed template bug (an all-hex 8-char config checksum can parse as a YAML number and reject the apply), unpatched for 6+ months. | | **`ownkube/kubernetes-events-exporter` chart (0.1.2)** | A full Go rewrite, not the resmoio/mustafaakin codebase this repo runs — `loki` receiver and `route.match` compatibility is unverified, not just untested. Its apparent activity doesn't hold up either: no human-merged commit in 5 months, only 11 open Dependabot PRs. | -| **Keep pulling the Bitnami OCI chart** | Exactly what #1089 asks to stop. Sibling components (Valkey, RabbitMQ images) are already mid-migration off Bitnami, confirming the source itself is on a deprecation path, not just this one chart. | +| **Keep pulling the Bitnami OCI chart** | Exactly what #1089 asks to stop. It would also be the last Bitnami dependency left in the repo: Valkey moved off `bitnamilegacy` in SPEC-012 (2026-07-21), and RabbitMQ's only use was `grafana-oncall`, removed outright when ADR-0029 chose RunLore instead. | | **Use the real upstream chart** | Doesn't exist. The upstream repo (`resmoio`, now pushed under `mustafaakin/kubernetes-event-exporter`) ships a raw `deploy/` manifest directory, not a Helm chart. | ## Consequences @@ -47,9 +49,11 @@ delete `flux/sources/helmrepo-bitnami.yaml`. `${cluster_name}` stays in the `Con - **No chart to upgrade.** A future exporter version means re-rendering and re-diffing by hand, not bumping a chart version. - **Config changes are hand edits.** `config.yaml` (routes, receivers, log settings) is now a - committed `ConfigMap` literal, edited directly instead of through `values.yaml`. -- **Only the image tag is Renovate-tracked**, via the repo-wide `kubernetes` manager - (`.github/renovate.json`) that already scans every manifest outside `clusters/` and `opentofu/`. -- **No checksum-triggered rollout.** The Helm chart's `checksum/config` pod annotation was dropped - with the rest of the Helm-only metadata; a `config.yaml` edit no longer forces a new - `ReplicaSet` on its own and needs an explicit rollout restart. + committed file, edited directly instead of through `values.yaml`. Kustomize's + `configMapGenerator` still rolls the pod on a change (see `kustomization.yaml`), so this only + removes the values layer, not the rollout behaviour. +- **The image tag is Renovate-tracked for the first time.** The `HelmRelease`'s split + `image.registry`/`image.repository`/`image.tag` values were read by no Renovate manager; the + plain `image:` reference in the vendored `Deployment` is picked up by the repo-wide `kubernetes` + manager (`.github/renovate.json`), which already scans every manifest outside `clusters/` and + `opentofu/`. diff --git a/website/content/docs/platform/observability/logs.md b/website/content/docs/platform/observability/logs.md index 8df9a1dc0..874077fcb 100644 --- a/website/content/docs/platform/observability/logs.md +++ b/website/content/docs/platform/observability/logs.md @@ -98,7 +98,7 @@ queries that silently return nothing, not an error: Watches Kubernetes Events cluster-wide (`clusterName: "${cluster_name}"` tag) and pushes every event to the deployed vlsingle's Loki-compatible endpoint — `victoria-logs-victoria-logs-single-server` on port 9428 -(`observability/base/kubernetes-event-exporter/configmap.yaml`). That is +(`observability/base/kubernetes-event-exporter/config.yaml`). That is the **only** path events take, and it works as of 2026-08-29. Both halves of that sentence earn their date. From 2025-08-23 to 2026-08-29 @@ -112,11 +112,11 @@ was never a fallback (if the informer stops, no receiver gets anything). The exporter's own operational logs are structured now as well (`logFormat: json`, `logLevel: info`). -Its metrics are real for the first time: `metrics.enabled: true` — it was -`false`, silently discarding the `serviceMonitor.enabled: true` nested under -it — with a `ServiceMonitor` in `observability` and one repaired alert, -`KubernetesEventExporterWatchErrors` (`severity: warning`, sustained -`rate > 0` for 15m), authored as a standalone `VMRule` +Its metrics are real for the first time: the old chart's `metrics.enabled: false` +silently discarded the `serviceMonitor.enabled: true` nested under it, so nothing +ever scraped it. A `ServiceMonitor` now ships directly in `observability`, plus +one repaired alert, `KubernetesEventExporterWatchErrors` (`severity: warning`, +sustained `rate > 0` for 15m), authored as a standalone `VMRule` (`observability/base/kubernetes-event-exporter/vmrule.yaml`) like every other alert in this repository. The alert's message deliberately names no namespace: the fork registers `WatchErrors` as a From 2a33cd72422d67510719fd25421c97c85cf8961f Mon Sep 17 00:00:00 2001 From: Smana Date: Mon, 21 Sep 2026 23:34:18 +0200 Subject: [PATCH 3/3] fix(observability): roll kubernetes-event-exporter's pod on config changes The exporter reads -conf once at startup with no file watcher, and vendoring the config as a plain ConfigMap (previous commit) dropped the HelmRelease's checksum/config annotation that used to force a new ReplicaSet on a config edit. Switch to Kustomize's configMapGenerator: its content-hashed name still rolls the pod, and the built-in name-reference transformer updates the Deployment's volume automatically. This is the repo's first configMapGenerator, noted inline. --- .../kubernetes-event-exporter/config.yaml | 15 ++++++++ .../kubernetes-event-exporter/configmap.yaml | 35 ------------------- .../kustomization.yaml | 24 ++++++++++++- 3 files changed, 38 insertions(+), 36 deletions(-) create mode 100644 observability/base/kubernetes-event-exporter/config.yaml delete mode 100644 observability/base/kubernetes-event-exporter/configmap.yaml diff --git a/observability/base/kubernetes-event-exporter/config.yaml b/observability/base/kubernetes-event-exporter/config.yaml new file mode 100644 index 000000000..9d2e7b6c9 --- /dev/null +++ b/observability/base/kubernetes-event-exporter/config.yaml @@ -0,0 +1,15 @@ +clusterName: ${cluster_name} +leaderElection: {} +logFormat: json +logLevel: info +metricsNamePrefix: event_exporter_ +receivers: +- loki: + streamLabels: + source: kubernetes-event-exporter + url: http://victoria-logs-victoria-logs-single-server.observability.svc.cluster.local:9428/insert/loki/api/v1/push + name: victorialogs +route: + routes: + - match: + - receiver: victorialogs diff --git a/observability/base/kubernetes-event-exporter/configmap.yaml b/observability/base/kubernetes-event-exporter/configmap.yaml deleted file mode 100644 index 79c9f4145..000000000 --- a/observability/base/kubernetes-event-exporter/configmap.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kubernetes-event-exporter - labels: - app.kubernetes.io/instance: kubernetes-event-exporter - app.kubernetes.io/name: kubernetes-event-exporter -data: - # No "dump" file receiver: with layout: {} (empty-but-present) the fork's - # file sink takes the convertLayoutTemplate branch over zero keys and emits - # contentless "{}" lines (pkg/sinks/file.go) — every cluster Event became a - # noise line that Vector's unfiltered kubernetes_logs source re-ingested - # into VictoriaLogs. It was never a fallback either: if the informer stops, - # no receiver gets anything. Events now flow solely via the loki receiver. - # - # The loki URL below targets vlsingle (see observability/base/victoria-logs/), - # the deployed variant — it pointed at the vlcluster vlinsert Service (never - # deployed here) from 2025-08-23 to 2026-08-29, so event push silently went - # nowhere the whole time. - config.yaml: | - clusterName: ${cluster_name} - leaderElection: {} - logFormat: json - logLevel: info - metricsNamePrefix: event_exporter_ - receivers: - - loki: - streamLabels: - source: kubernetes-event-exporter - url: http://victoria-logs-victoria-logs-single-server.observability.svc.cluster.local:9428/insert/loki/api/v1/push - name: victorialogs - route: - routes: - - match: - - receiver: victorialogs diff --git a/observability/base/kubernetes-event-exporter/kustomization.yaml b/observability/base/kubernetes-event-exporter/kustomization.yaml index dd460f8bc..e4f15cc63 100644 --- a/observability/base/kubernetes-event-exporter/kustomization.yaml +++ b/observability/base/kubernetes-event-exporter/kustomization.yaml @@ -3,7 +3,6 @@ kind: Kustomization namespace: observability resources: - serviceaccount.yaml - - configmap.yaml - clusterrole.yaml - clusterrolebinding.yaml - deployment.yaml @@ -12,3 +11,26 @@ resources: - networkpolicy.yaml - pdb.yaml - vmrule.yaml + +# No "dump" file receiver in config.yaml: with layout: {} (empty-but-present) the +# fork's file sink takes the convertLayoutTemplate branch over zero keys and emits +# contentless "{}" lines (pkg/sinks/file.go) — every cluster Event became a noise +# line that Vector's unfiltered kubernetes_logs source re-ingested into VictoriaLogs. +# It was never a fallback either: if the informer stops, no receiver gets anything. +# Events flow solely via the loki receiver, targeting vlsingle (see +# observability/base/victoria-logs/) — it pointed at the vlcluster vlinsert Service +# (never deployed here) from 2025-08-23 to 2026-08-29, so event push silently went +# nowhere the whole time. +# +# The repo's first configMapGenerator: the exporter reads -conf once at startup with +# no file watcher, so a plain ConfigMap wouldn't roll the pod on a config change. The +# generated name's content hash does, the same way a HelmRelease's checksum/config +# annotation did before this was vendored (ADR-0040). +configMapGenerator: + - name: kubernetes-event-exporter + files: + - config.yaml + options: + labels: + app.kubernetes.io/instance: kubernetes-event-exporter + app.kubernetes.io/name: kubernetes-event-exporter