fix: durable ClickHouse retention — 15Gi volume, 3d system-log TTL, 7d otel exporter TTL

charts/krateo-observability/Chart.yaml

@@ -17,7 +17,13 @@ type: application
 # The mount was never applied (upstream renders only cluster.spec; the clickhouse.com CRD has no
 # extraVolumes field; subPath into config.d/ also crashloops on kind) so /events 404'd and the only
 # path that worked was the forbidden raw native-SQL/dynamic-query API. New version forces the re-pull.
-version: 0.1.8
+# 0.1.9: durable retention fix. The 10Gi data volume filled in 8 days (no TTL anywhere: otel_*
+# tables unbounded + ClickHouse system log tables unbounded ~3.6GiB) → inserts failed with
+# code 243 → the OTel collectors OOM-crash-looped → telemetry dead. Now: dataVolumeClaimSpec
+# 15Gi (matches the live relief) + 3-day system-log ttl (query_log/text_log/metric_log/part_log/
+# asynchronous_metric_log/trace_log) via settings.extraConfig. Pairs with otel-collector-daemonset
+# 0.1.5 + otel-collector-deployment 0.3.3 (clickhouse exporter ttl: 168h on the otel_* tables).
+version: 0.1.9
 appVersion: "3.0.0"
 dependencies:
   - name: clickstack

charts/krateo-observability/values.yaml

@@ -67,6 +67,21 @@ clickstack:
             requests:
               cpu: "1"
               memory: 2Gi
+        # RETENTION INCIDENT (2026-07): the upstream default 10Gi data volume filled in
+        # 8 DAYS on a live bench — no TTL anywhere (otel_* tables unbounded + the
+        # system.* log tables unbounded at ~3.6GiB) → every insert failed with code 243
+        # (NOT_ENOUGH_SPACE) → the OTel collectors buffered, OOM-crash-looped and
+        # telemetry went fully dead. 15Gi matches the live relief (PVC expanded
+        # 10Gi->15Gi); the paired fixes are the 3-day system-log ttl under
+        # settings.extraConfig below and the 7-day otel_* TTL in the collector charts'
+        # clickhouse exporter (ttl: 168h). Only the overridden key is set here — Helm
+        # deep-merges cluster.spec over the upstream default (same pattern as
+        # containerTemplate.resources above, which inherits upstream's image), so
+        # accessModes etc. still come from upstream clickstack.
+        dataVolumeClaimSpec:
+          resources:
+            requests:
+              storage: 15Gi
         # ClickHouse server config merged into /etc/clickhouse-server/config.d/ by the
         # clickhouse.com operator. The GET /events?composition_id=<uid> handler MUST be
         # registered here, NOT via clickhouse.extraVolumes (see below): a predefined HTTP
@@ -91,6 +106,29 @@ clickstack:
         # serving its default routes, so the operator probes + native query API must be re-declared.
         settings:
           extraConfig:
+            # SYSTEM-LOG RETENTION — DO NOT REMOVE (see the dataVolumeClaimSpec comment
+            # above: with no <ttl> the system.* log tables grow unbounded and were
+            # ~3.6GiB of the 10Gi disk when the bench filled up and inserts died with
+            # code 243). Each key below merges into the server's default <query_log>/…
+            # section via the same operator-native extraConfig->config.d merge that
+            # http_handlers uses (YAML maps render as nested config elements). `ttl` is
+            # the documented per-system-log-table engine TTL and is applied when
+            # ClickHouse (re)creates the table. NOTE for EXISTING installs: on the next
+            # server start ClickHouse detects the changed table definition, renames the
+            # old table (e.g. query_log_0) and creates a fresh one with the TTL — the
+            # renamed leftovers keep their disk until manually DROPped.
+            query_log:
+              ttl: "event_date + INTERVAL 3 DAY DELETE"
+            text_log:
+              ttl: "event_date + INTERVAL 3 DAY DELETE"
+            metric_log:
+              ttl: "event_date + INTERVAL 3 DAY DELETE"
+            part_log:
+              ttl: "event_date + INTERVAL 3 DAY DELETE"
+            asynchronous_metric_log:
+              ttl: "event_date + INTERVAL 3 DAY DELETE"
+            trace_log:
+              ttl: "event_date + INTERVAL 3 DAY DELETE"
             http_handlers:
               rule:
                 # GET /events?composition_id=<uid>
@@ -139,11 +177,12 @@ clickstack:
                   handler:
                     type: dynamic_query_handler
                     query_param_name: query
-    # STILL INERT (same wrong-path bug): upstream ignores these top-level keys. NOT moved to
-    # cluster.spec on purpose — applying them live = ClickHouse version upgrade (25.7->26.3)
-    # + PVC resize (10Gi->50Gi), a deliberate change that must not ride along with the memory
-    # fix. To realize later: image -> cluster.spec.containerTemplate.image; size ->
-    # cluster.spec.dataVolumeClaimSpec.resources.requests.storage.
+    # STILL INERT (same wrong-path bug): upstream ignores these top-level keys. The image
+    # stays inert on purpose — realizing it live = ClickHouse version upgrade (25.7->26.3),
+    # a deliberate change that must not ride along. To realize later: image ->
+    # cluster.spec.containerTemplate.image. Storage is now REAL at 15Gi via
+    # cluster.spec.dataVolumeClaimSpec above (retention incident); the 50Gi below remains
+    # an inert aspiration — raising the real value later is a plain PVC expansion.
     image: "clickhouse/clickhouse-server:26.3-alpine"
     persistence:
       enabled: true

charts/otel-collector-daemonset/Chart.yaml

@@ -5,7 +5,9 @@ description: >-
   upstream opentelemetry-collector chart to collect pod logs, host metrics,
   kubelet metrics and OTLP traces and export them to ClickHouse.
 type: application
-version: 0.1.4
+# 0.1.5: clickhouse exporter ttl: 168h — 7-day retention stamped into freshly created otel_*
+# tables (retention incident: 10Gi filled in 8 days → insert code 243 → collector OOM loop).
+version: 0.1.5
 appVersion: "0.1.0"
 dependencies:
   - name: opentelemetry-collector

charts/otel-collector-daemonset/values.schema.json

@@ -378,6 +378,10 @@
                     "create_schema": {
                       "type": "boolean"
                     },
+                    "ttl": {
+                      "type": "string",
+                      "description": "Retention TTL stamped into the otel_* tables at creation (e.g. 168h = 7d). Guards the ClickHouse data volume from filling (insert code 243 -> collector OOM loop). Only applies to freshly created tables."
+                    },
                     "timeout": {
                       "type": "string"
                     },

charts/otel-collector-daemonset/values.yaml

@@ -115,6 +115,17 @@ opentelemetry-collector:
         # pipeline below — the gateway has create_schema:true but no traces pipeline, so
         # without this nothing ever creates otel_traces. KOS-1 #112 bug #1.
         create_schema: true
+        # RETENTION — DO NOT REMOVE. Incident 2026-07: with no TTL the otel_* tables grew
+        # unbounded and filled the 10Gi ClickHouse volume in 8 days → every insert failed
+        # with code 243 (NOT_ENOUGH_SPACE) → this collector buffered and OOM-crash-looped →
+        # telemetry dead. ttl: 168h stamps a native 7-day TTL into the CREATE TABLE the
+        # exporter issues, so FRESH installs are bounded from day one. NOTE: create_schema
+        # is CREATE ... IF NOT EXISTS — this ttl only applies at table CREATION; tables that
+        # already exist (e.g. the live bench) were fixed with a live
+        # `ALTER TABLE ... MODIFY TTL` (7 days) and are untouched by this setting.
+        # Disk-size sibling fixes: krateo-observability 0.1.9 (15Gi volume + 3-day
+        # system-log ttl via cluster.spec.settings.extraConfig).
+        ttl: 168h
         timeout: 10s
         retry_on_failure:
           enabled: true

charts/otel-collector-deployment/Chart.yaml

@@ -5,7 +5,9 @@ description: >-
   upstream opentelemetry-collector chart with a custom otelcol-krateo image that
   enriches K8s events with krateo.io/composition-id and exports to ClickHouse.
 type: application
-version: 0.3.2
+# 0.3.3: clickhouse exporter ttl: 168h — 7-day retention stamped into freshly created otel_*
+# tables (retention incident: 10Gi filled in 8 days → insert code 243 → collector OOM loop).
+version: 0.3.3
 appVersion: "1.0.2"
 dependencies:
   - name: opentelemetry-collector

charts/otel-collector-deployment/values.schema.json

@@ -364,6 +364,10 @@
                     "create_schema": {
                       "type": "boolean"
                     },
+                    "ttl": {
+                      "type": "string",
+                      "description": "Retention TTL stamped into the otel_* tables at creation (e.g. 168h = 7d). Guards the ClickHouse data volume from filling (insert code 243 -> collector OOM loop). Only applies to freshly created tables."
+                    },
                     "timeout": {
                       "type": "string"
                     },

charts/otel-collector-deployment/values.yaml

@@ -211,6 +211,17 @@ opentelemetry-collector:
         traces_table_name: otel_traces
         metrics_table_name: otel_metrics
         create_schema: true
+        # RETENTION — DO NOT REMOVE. Incident 2026-07: with no TTL the otel_* tables grew
+        # unbounded and filled the 10Gi ClickHouse volume in 8 days → every insert failed
+        # with code 243 (NOT_ENOUGH_SPACE) → the collectors buffered and OOM-crash-looped →
+        # telemetry dead. ttl: 168h stamps a native 7-day TTL into the CREATE TABLE the
+        # exporter issues, so FRESH installs are bounded from day one. NOTE: create_schema
+        # is CREATE ... IF NOT EXISTS — this ttl only applies at table CREATION; tables that
+        # already exist (e.g. the live bench) were fixed with a live
+        # `ALTER TABLE ... MODIFY TTL` (7 days) and are untouched by this setting.
+        # Disk-size sibling fixes: krateo-observability 0.1.9 (15Gi volume + 3-day
+        # system-log ttl via cluster.spec.settings.extraConfig).
+        ttl: 168h
         timeout: 10s
         retry_on_failure:
           enabled: true