From 0411c88929ea039ee518f0ee1649758f7f98a1d3 Mon Sep 17 00:00:00 2001 From: Diego Braga Date: Thu, 6 Aug 2026 18:58:57 +0200 Subject: [PATCH] fix(charts): explicit lenient probe tolerances (sse-proxy, otel wrappers) + Quantity schema typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two org-standard sweeps in one pass over this repo's charts: 1) Probe tolerances. The Kubernetes probe defaults (timeoutSeconds: 1, failureThreshold: 3) kill HEALTHY pods under CPU contention: a slow-but-alive health reply counts as a failure and the resulting restart storm worsens the contention. Per the org standard (precedent: the chart-inspector block in the core-provider chart, authn PR #13 and the sibling sweep PRs): - livenessProbe: timeoutSeconds: 5, failureThreshold: 6 — lenient, so only a truly dead process is restarted (sse-proxy: 10s x 6 = 60s window; otel wrappers: implicit 10s default x 6 = 60s window). - readinessProbe: timeoutSeconds: 5, failureThreshold: 3 — endpoint removal stays prompt, but load alone must not flap it. krateo-sse-proxy gets this directly in templates/deployment.yaml (probes are hardcoded there, existing periodSeconds kept). The two otel-collector wrappers get it via values passthrough to the upstream opentelemetry-collector chart 0.158.1, whose templates/_pod.tpl renders exactly these keys; the wrappers' values.schema.json is extended to type the new livenessProbe/readinessProbe keys. Resources untouched; no probes added to containers that have none (krateo-observability ships no workload templates of its own). 2) Quantity schema typing scan. values.schema.json fields for Kubernetes Quantities (cpu/memory/storage) typed as bare integer/number propagate into crdgen-generated composition CRDs and break the krateo-composition-version MutatingAdmissionPolicy typed conversion on Quantity strings (krateo-core-provider#66; snowplow had it, fixed in snowplow PR #140). Scanned all four charts' schemas: CLEAN — every cpu/memory/persistence-size field is already typed "string" (the only integer size-like fields, max_log_size/send_batch_size, are OTel collector config counts, not k8s Quantities). No changes needed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LJsLqtryCgWwEt8FnPE1se --- .../templates/deployment.yaml | 13 ++++++ .../values.schema.json | 42 +++++++++++++++++++ charts/otel-collector-daemonset/values.yaml | 15 +++++++ .../values.schema.json | 42 +++++++++++++++++++ charts/otel-collector-deployment/values.yaml | 15 +++++++ 5 files changed, 127 insertions(+) diff --git a/charts/krateo-sse-proxy/templates/deployment.yaml b/charts/krateo-sse-proxy/templates/deployment.yaml index c61ba48..b79d308 100644 --- a/charts/krateo-sse-proxy/templates/deployment.yaml +++ b/charts/krateo-sse-proxy/templates/deployment.yaml @@ -47,18 +47,31 @@ spec: optional: {{ .Values.clickhouse.passwordSecret.optional }} - name: LISTEN_ADDR value: {{ .Values.listenAddr | quote }} + # Lenient liveness tolerance (org standard): the Kubernetes defaults + # (timeoutSeconds: 1, failureThreshold: 3) kill HEALTHY pods under CPU + # contention — a slow-but-alive /health reply counts as a failure and the + # resulting restart storm worsens the contention. Only a truly dead process + # should die: 10s period x 6 failures = 60s of continuous failure before + # a restart. livenessProbe: httpGet: path: /health port: 8080 initialDelaySeconds: 5 periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + # Readiness stays prompt (5s period, 3 failures) so a genuinely unhealthy + # endpoint is removed from the Service quickly, but timeoutSeconds: 5 keeps + # load alone from flapping the endpoint in and out. readinessProbe: httpGet: path: /health port: 8080 initialDelaySeconds: 3 periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 resources: {{- toYaml .Values.resources | nindent 12 }} securityContext: diff --git a/charts/otel-collector-daemonset/values.schema.json b/charts/otel-collector-daemonset/values.schema.json index 99bf053..8897d0d 100644 --- a/charts/otel-collector-daemonset/values.schema.json +++ b/charts/otel-collector-daemonset/values.schema.json @@ -528,6 +528,48 @@ } } } + }, + "livenessProbe": { + "type": "object", + "additionalProperties": true, + "properties": { + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + }, + "failureThreshold": { + "type": "integer" + }, + "terminationGracePeriodSeconds": { + "type": "integer" + } + } + }, + "readinessProbe": { + "type": "object", + "additionalProperties": true, + "properties": { + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + }, + "successThreshold": { + "type": "integer" + }, + "failureThreshold": { + "type": "integer" + } + } } }, "description": "Values forwarded to the upstream opentelemetry-collector subchart." diff --git a/charts/otel-collector-daemonset/values.yaml b/charts/otel-collector-daemonset/values.yaml index e2e23c9..3ba4ad7 100644 --- a/charts/otel-collector-daemonset/values.yaml +++ b/charts/otel-collector-daemonset/values.yaml @@ -176,3 +176,18 @@ opentelemetry-collector: requests: cpu: 100m memory: 256Mi + + # Probe tolerances, passed through to the upstream opentelemetry-collector chart's + # templates/_pod.tpl (org standard). The Kubernetes defaults (timeoutSeconds: 1, + # failureThreshold: 3) kill HEALTHY pods under CPU contention — a slow-but-alive + # health reply counts as a failure and the resulting restart storm worsens the + # contention. Liveness is lenient so only a truly dead process dies: implicit + # periodSeconds default 10 x 6 failures = 60s of continuous failure before restart. + livenessProbe: + timeoutSeconds: 5 + failureThreshold: 6 + # Readiness keeps prompt endpoint removal (failureThreshold: 3, implicit 10s period) + # but timeoutSeconds: 5 keeps load alone from flapping the endpoint in and out. + readinessProbe: + timeoutSeconds: 5 + failureThreshold: 3 diff --git a/charts/otel-collector-deployment/values.schema.json b/charts/otel-collector-deployment/values.schema.json index 83ac2cb..c7aa423 100644 --- a/charts/otel-collector-deployment/values.schema.json +++ b/charts/otel-collector-deployment/values.schema.json @@ -515,6 +515,48 @@ } } } + }, + "livenessProbe": { + "type": "object", + "additionalProperties": true, + "properties": { + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + }, + "failureThreshold": { + "type": "integer" + }, + "terminationGracePeriodSeconds": { + "type": "integer" + } + } + }, + "readinessProbe": { + "type": "object", + "additionalProperties": true, + "properties": { + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + }, + "successThreshold": { + "type": "integer" + }, + "failureThreshold": { + "type": "integer" + } + } } }, "description": "Values forwarded to the upstream opentelemetry-collector subchart." diff --git a/charts/otel-collector-deployment/values.yaml b/charts/otel-collector-deployment/values.yaml index 8f8fd38..bea82f0 100644 --- a/charts/otel-collector-deployment/values.yaml +++ b/charts/otel-collector-deployment/values.yaml @@ -263,3 +263,18 @@ opentelemetry-collector: requests: cpu: 100m memory: 256Mi + + # Probe tolerances, passed through to the upstream opentelemetry-collector chart's + # templates/_pod.tpl (org standard). The Kubernetes defaults (timeoutSeconds: 1, + # failureThreshold: 3) kill HEALTHY pods under CPU contention — a slow-but-alive + # health reply counts as a failure and the resulting restart storm worsens the + # contention. Liveness is lenient so only a truly dead process dies: implicit + # periodSeconds default 10 x 6 failures = 60s of continuous failure before restart. + livenessProbe: + timeoutSeconds: 5 + failureThreshold: 6 + # Readiness keeps prompt endpoint removal (failureThreshold: 3, implicit 10s period) + # but timeoutSeconds: 5 keeps load alone from flapping the endpoint in and out. + readinessProbe: + timeoutSeconds: 5 + failureThreshold: 3