From b560332aa172df91875f2c0ee2d26ffc22ae1d23 Mon Sep 17 00:00:00 2001 From: Diego Braga Date: Wed, 22 Jul 2026 22:51:08 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20durable=20ClickHouse=20retention=20?= =?UTF-8?q?=E2=80=94=2015Gi=20volume,=203d=20system-log=20ttl,=207d=20otel?= =?UTF-8?q?=5F*=20exporter=20ttl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incident: on a live bench the ClickHouse 10Gi data volume filled in 8 days — no TTL anywhere (otel_* tables unbounded; system.* log tables unbounded at ~3.6GiB) — so every insert failed with code 243 (NOT_ENOUGH_SPACE), the OTel collectors buffered + OOM-crash-looped, and telemetry went fully dead. Live relief was applied by hand (PVC 10->15Gi, system tables truncated, partitions dropped, ALTER TABLE ... MODIFY TTL 7d on otel_*). This makes fresh installs never hit it: - krateo-observability 0.1.9: * cluster.spec.dataVolumeClaimSpec.resources.requests.storage: 15Gi (minimal deep-merge override of the upstream clickstack 10Gi default; accessModes still inherited from upstream) * settings.extraConfig: 3-day engine ttl on query_log, text_log, metric_log, part_log, asynchronous_metric_log, trace_log — same operator-native extraConfig->config.d merge the live http_handlers already proves out - otel-collector-daemonset 0.1.5 + otel-collector-deployment 0.3.3: * clickhouse exporter ttl: 168h — 7-day TTL stamped into the otel_* tables at creation (create_schema is CREATE IF NOT EXISTS, so pre-existing benches keep their live-ALTERed TTL; this covers fresh installs) * values.schema.json: ttl enumerated alongside the other exporter keys Validated: helm lint + helm template clean on all three charts; rendered ClickHouseCluster CR carries 15Gi + all six system-log ttl entries merged with the upstream settings; both rendered collector configs carry ttl: 168h. Co-Authored-By: Claude Opus 4.8 --- charts/krateo-observability/Chart.yaml | 8 ++- charts/krateo-observability/values.yaml | 49 +++++++++++++++++-- charts/otel-collector-daemonset/Chart.yaml | 4 +- .../values.schema.json | 4 ++ charts/otel-collector-daemonset/values.yaml | 11 +++++ charts/otel-collector-deployment/Chart.yaml | 4 +- .../values.schema.json | 4 ++ charts/otel-collector-deployment/values.yaml | 11 +++++ 8 files changed, 87 insertions(+), 8 deletions(-) diff --git a/charts/krateo-observability/Chart.yaml b/charts/krateo-observability/Chart.yaml index 3ef5e0c..cb03f30 100644 --- a/charts/krateo-observability/Chart.yaml +++ b/charts/krateo-observability/Chart.yaml @@ -17,7 +17,13 @@ type: application # The mount was never applied (upstream renders only cluster.spec; the clickhouse.com CRD has no # extraVolumes field; subPath into config.d/ also crashloops on kind) so /events 404'd and the only # path that worked was the forbidden raw native-SQL/dynamic-query API. New version forces the re-pull. -version: 0.1.8 +# 0.1.9: durable retention fix. The 10Gi data volume filled in 8 days (no TTL anywhere: otel_* +# tables unbounded + ClickHouse system log tables unbounded ~3.6GiB) → inserts failed with +# code 243 → the OTel collectors OOM-crash-looped → telemetry dead. Now: dataVolumeClaimSpec +# 15Gi (matches the live relief) + 3-day system-log ttl (query_log/text_log/metric_log/part_log/ +# asynchronous_metric_log/trace_log) via settings.extraConfig. Pairs with otel-collector-daemonset +# 0.1.5 + otel-collector-deployment 0.3.3 (clickhouse exporter ttl: 168h on the otel_* tables). +version: 0.1.9 appVersion: "3.0.0" dependencies: - name: clickstack diff --git a/charts/krateo-observability/values.yaml b/charts/krateo-observability/values.yaml index 49f2a7d..72e087f 100644 --- a/charts/krateo-observability/values.yaml +++ b/charts/krateo-observability/values.yaml @@ -67,6 +67,21 @@ clickstack: requests: cpu: "1" memory: 2Gi + # RETENTION INCIDENT (2026-07): the upstream default 10Gi data volume filled in + # 8 DAYS on a live bench — no TTL anywhere (otel_* tables unbounded + the + # system.* log tables unbounded at ~3.6GiB) → every insert failed with code 243 + # (NOT_ENOUGH_SPACE) → the OTel collectors buffered, OOM-crash-looped and + # telemetry went fully dead. 15Gi matches the live relief (PVC expanded + # 10Gi->15Gi); the paired fixes are the 3-day system-log ttl under + # settings.extraConfig below and the 7-day otel_* TTL in the collector charts' + # clickhouse exporter (ttl: 168h). Only the overridden key is set here — Helm + # deep-merges cluster.spec over the upstream default (same pattern as + # containerTemplate.resources above, which inherits upstream's image), so + # accessModes etc. still come from upstream clickstack. + dataVolumeClaimSpec: + resources: + requests: + storage: 15Gi # ClickHouse server config merged into /etc/clickhouse-server/config.d/ by the # clickhouse.com operator. The GET /events?composition_id= handler MUST be # registered here, NOT via clickhouse.extraVolumes (see below): a predefined HTTP @@ -91,6 +106,29 @@ clickstack: # serving its default routes, so the operator probes + native query API must be re-declared. settings: extraConfig: + # SYSTEM-LOG RETENTION — DO NOT REMOVE (see the dataVolumeClaimSpec comment + # above: with no the system.* log tables grow unbounded and were + # ~3.6GiB of the 10Gi disk when the bench filled up and inserts died with + # code 243). Each key below merges into the server's default /… + # section via the same operator-native extraConfig->config.d merge that + # http_handlers uses (YAML maps render as nested config elements). `ttl` is + # the documented per-system-log-table engine TTL and is applied when + # ClickHouse (re)creates the table. NOTE for EXISTING installs: on the next + # server start ClickHouse detects the changed table definition, renames the + # old table (e.g. query_log_0) and creates a fresh one with the TTL — the + # renamed leftovers keep their disk until manually DROPped. + query_log: + ttl: "event_date + INTERVAL 3 DAY DELETE" + text_log: + ttl: "event_date + INTERVAL 3 DAY DELETE" + metric_log: + ttl: "event_date + INTERVAL 3 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 3 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 3 DAY DELETE" + trace_log: + ttl: "event_date + INTERVAL 3 DAY DELETE" http_handlers: rule: # GET /events?composition_id= @@ -139,11 +177,12 @@ clickstack: handler: type: dynamic_query_handler query_param_name: query - # STILL INERT (same wrong-path bug): upstream ignores these top-level keys. NOT moved to - # cluster.spec on purpose — applying them live = ClickHouse version upgrade (25.7->26.3) - # + PVC resize (10Gi->50Gi), a deliberate change that must not ride along with the memory - # fix. To realize later: image -> cluster.spec.containerTemplate.image; size -> - # cluster.spec.dataVolumeClaimSpec.resources.requests.storage. + # STILL INERT (same wrong-path bug): upstream ignores these top-level keys. The image + # stays inert on purpose — realizing it live = ClickHouse version upgrade (25.7->26.3), + # a deliberate change that must not ride along. To realize later: image -> + # cluster.spec.containerTemplate.image. Storage is now REAL at 15Gi via + # cluster.spec.dataVolumeClaimSpec above (retention incident); the 50Gi below remains + # an inert aspiration — raising the real value later is a plain PVC expansion. image: "clickhouse/clickhouse-server:26.3-alpine" persistence: enabled: true diff --git a/charts/otel-collector-daemonset/Chart.yaml b/charts/otel-collector-daemonset/Chart.yaml index 4363a69..066e122 100644 --- a/charts/otel-collector-daemonset/Chart.yaml +++ b/charts/otel-collector-daemonset/Chart.yaml @@ -5,7 +5,9 @@ description: >- upstream opentelemetry-collector chart to collect pod logs, host metrics, kubelet metrics and OTLP traces and export them to ClickHouse. type: application -version: 0.1.4 +# 0.1.5: clickhouse exporter ttl: 168h — 7-day retention stamped into freshly created otel_* +# tables (retention incident: 10Gi filled in 8 days → insert code 243 → collector OOM loop). +version: 0.1.5 appVersion: "0.1.0" dependencies: - name: opentelemetry-collector diff --git a/charts/otel-collector-daemonset/values.schema.json b/charts/otel-collector-daemonset/values.schema.json index 3a010bd..99bf053 100644 --- a/charts/otel-collector-daemonset/values.schema.json +++ b/charts/otel-collector-daemonset/values.schema.json @@ -378,6 +378,10 @@ "create_schema": { "type": "boolean" }, + "ttl": { + "type": "string", + "description": "Retention TTL stamped into the otel_* tables at creation (e.g. 168h = 7d). Guards the ClickHouse data volume from filling (insert code 243 -> collector OOM loop). Only applies to freshly created tables." + }, "timeout": { "type": "string" }, diff --git a/charts/otel-collector-daemonset/values.yaml b/charts/otel-collector-daemonset/values.yaml index afc53ea..e2e23c9 100644 --- a/charts/otel-collector-daemonset/values.yaml +++ b/charts/otel-collector-daemonset/values.yaml @@ -115,6 +115,17 @@ opentelemetry-collector: # pipeline below — the gateway has create_schema:true but no traces pipeline, so # without this nothing ever creates otel_traces. KOS-1 #112 bug #1. create_schema: true + # RETENTION — DO NOT REMOVE. Incident 2026-07: with no TTL the otel_* tables grew + # unbounded and filled the 10Gi ClickHouse volume in 8 days → every insert failed + # with code 243 (NOT_ENOUGH_SPACE) → this collector buffered and OOM-crash-looped → + # telemetry dead. ttl: 168h stamps a native 7-day TTL into the CREATE TABLE the + # exporter issues, so FRESH installs are bounded from day one. NOTE: create_schema + # is CREATE ... IF NOT EXISTS — this ttl only applies at table CREATION; tables that + # already exist (e.g. the live bench) were fixed with a live + # `ALTER TABLE ... MODIFY TTL` (7 days) and are untouched by this setting. + # Disk-size sibling fixes: krateo-observability 0.1.9 (15Gi volume + 3-day + # system-log ttl via cluster.spec.settings.extraConfig). + ttl: 168h timeout: 10s retry_on_failure: enabled: true diff --git a/charts/otel-collector-deployment/Chart.yaml b/charts/otel-collector-deployment/Chart.yaml index fe893c0..4c06386 100644 --- a/charts/otel-collector-deployment/Chart.yaml +++ b/charts/otel-collector-deployment/Chart.yaml @@ -5,7 +5,9 @@ description: >- upstream opentelemetry-collector chart with a custom otelcol-krateo image that enriches K8s events with krateo.io/composition-id and exports to ClickHouse. type: application -version: 0.3.2 +# 0.3.3: clickhouse exporter ttl: 168h — 7-day retention stamped into freshly created otel_* +# tables (retention incident: 10Gi filled in 8 days → insert code 243 → collector OOM loop). +version: 0.3.3 appVersion: "1.0.2" dependencies: - name: opentelemetry-collector diff --git a/charts/otel-collector-deployment/values.schema.json b/charts/otel-collector-deployment/values.schema.json index 0ec9c98..83ac2cb 100644 --- a/charts/otel-collector-deployment/values.schema.json +++ b/charts/otel-collector-deployment/values.schema.json @@ -364,6 +364,10 @@ "create_schema": { "type": "boolean" }, + "ttl": { + "type": "string", + "description": "Retention TTL stamped into the otel_* tables at creation (e.g. 168h = 7d). Guards the ClickHouse data volume from filling (insert code 243 -> collector OOM loop). Only applies to freshly created tables." + }, "timeout": { "type": "string" }, diff --git a/charts/otel-collector-deployment/values.yaml b/charts/otel-collector-deployment/values.yaml index aaf6099..22aaef9 100644 --- a/charts/otel-collector-deployment/values.yaml +++ b/charts/otel-collector-deployment/values.yaml @@ -211,6 +211,17 @@ opentelemetry-collector: traces_table_name: otel_traces metrics_table_name: otel_metrics create_schema: true + # RETENTION — DO NOT REMOVE. Incident 2026-07: with no TTL the otel_* tables grew + # unbounded and filled the 10Gi ClickHouse volume in 8 days → every insert failed + # with code 243 (NOT_ENOUGH_SPACE) → the collectors buffered and OOM-crash-looped → + # telemetry dead. ttl: 168h stamps a native 7-day TTL into the CREATE TABLE the + # exporter issues, so FRESH installs are bounded from day one. NOTE: create_schema + # is CREATE ... IF NOT EXISTS — this ttl only applies at table CREATION; tables that + # already exist (e.g. the live bench) were fixed with a live + # `ALTER TABLE ... MODIFY TTL` (7 days) and are untouched by this setting. + # Disk-size sibling fixes: krateo-observability 0.1.9 (15Gi volume + 3-day + # system-log ttl via cluster.spec.settings.extraConfig). + ttl: 168h timeout: 10s retry_on_failure: enabled: true