From 26807752dd1f0ec940616814f480c9138d6f845a Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 17:13:07 +0100 Subject: [PATCH 1/2] chore: Follow-ups to the containerdebug log collection --- CHANGELOG.md | 3 ++- .../src/controller/build/properties/env_vars.rs | 8 +++----- .../src/controller/build/volumes.rs | 8 +++----- rust/operator-binary/src/crd/mod.rs | 14 +++++++++++++- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6cd879..33b7d1fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#868]). - Webservers now have a default affinity to the OPA Pods when OPA authorization is configured ([#872]). - The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to - the Vector aggregator ([#888]). + the Vector aggregator ([#888], [#892]). ### Changed @@ -112,6 +112,7 @@ [#886]: https://github.com/stackabletech/airflow-operator/pull/886 [#887]: https://github.com/stackabletech/airflow-operator/pull/887 [#888]: https://github.com/stackabletech/airflow-operator/pull/888 +[#892]: https://github.com/stackabletech/airflow-operator/pull/892 ## [26.7.0] - 2026-07-21 diff --git a/rust/operator-binary/src/controller/build/properties/env_vars.rs b/rust/operator-binary/src/controller/build/properties/env_vars.rs index a74d6dd3..91674d8c 100644 --- a/rust/operator-binary/src/controller/build/properties/env_vars.rs +++ b/rust/operator-binary/src/controller/build/properties/env_vars.rs @@ -15,7 +15,8 @@ use stackable_operator::{ use crate::{ controller::{ValidatedCluster, ValidatedLogging}, crd::{ - AirflowExecutor, AirflowRole, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, + AirflowExecutor, AirflowRole, CONTAINERDEBUG_LOG_DIR, HTTP_PORT, LOG_CONFIG_DIR, + TEMPLATE_LOCATION, TEMPLATE_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -188,10 +189,7 @@ pub fn build_airflow_statefulset_envs( // other telemetry-based process in it would log into the same directory and the Vector // agent would label its events with `container: containerdebug`. env_vars = env_vars - .with_value( - &FILE_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), - ) + .with_value(&FILE_LOG_DIRECTORY, CONTAINERDEBUG_LOG_DIR) .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") .with_value(&FILE_LOG_MAX_FILES, "2") // apply overrides last of all; `EnvVarSet` is keyed by name, so iteration is already diff --git a/rust/operator-binary/src/controller/build/volumes.rs b/rust/operator-binary/src/controller/build/volumes.rs index 30f9821b..06e6d787 100644 --- a/rust/operator-binary/src/controller/build/volumes.rs +++ b/rust/operator-binary/src/controller/build/volumes.rs @@ -32,11 +32,9 @@ pub fn create_volumes( name: LOG_VOLUME_NAME.to_string(), empty_dir: Some(EmptyDirVolumeSource { medium: None, - // Only the Airflow log files are budgeted here. The `containerdebug` files in this - // volume are rotated by time, which bounds their number but not their size, so they - // are an unbudgeted writer. They stay negligible at the default INFO level and a 30 - // minute loop, but raising `FILE_LOG_LEVEL` via `envOverrides` can exceed the limit, - // which makes the kubelet evict the Pod. + // Only the Airflow log files are budgeted. The `containerdebug` files in this + // volume are rotated by time, not by size, so they have no budget; raising + // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. size_limit: Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_LOG_FILES_SIZE], )), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 89b8bce3..50c3742b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeSet, ops::Deref, str::FromStr}; +use const_format::concatcp; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -84,6 +85,11 @@ pub const CONFIG_PATH: &str = "/stackable/app/config"; pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config"; pub const AIRFLOW_HOME: &str = "/stackable/airflow"; +/// Directory which the `containerdebug` process writes its log files to. It must be exactly one +/// level below the log directory, because the Vector agent derives the container name from that +/// path segment. +pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); + constant!(pub TEMPLATE_VOLUME_NAME: VolumeName = "airflow-executor-pod-template"); pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; @@ -776,7 +782,13 @@ impl Deref for AirflowRole { } fn container_debug_command() -> String { - format!("containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &") + // The log directory is created here, because `FILE_LOG_MAX_FILES` makes the telemetry library + // prune old log files before it creates the first one, which complains about the directory not + // existing yet. + format!( + "mkdir --parents {CONTAINERDEBUG_LOG_DIR}\n\ + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &" + ) } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] From 905db68f6965989b42c49516011fe6d5da85a7b5 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 17:50:46 +0100 Subject: [PATCH 2/2] chore: Create the containerdebug log directory via the environment variable --- .../src/controller/build/properties/env_vars.rs | 11 ++++++++--- rust/operator-binary/src/crd/mod.rs | 8 +------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/env_vars.rs b/rust/operator-binary/src/controller/build/properties/env_vars.rs index 91674d8c..6bb930af 100644 --- a/rust/operator-binary/src/controller/build/properties/env_vars.rs +++ b/rust/operator-binary/src/controller/build/properties/env_vars.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeSet, str::FromStr}; +use const_format::concatcp; use stackable_operator::{ constant, crd::{authentication::oidc, git_sync}, @@ -15,8 +16,7 @@ use stackable_operator::{ use crate::{ controller::{ValidatedCluster, ValidatedLogging}, crd::{ - AirflowExecutor, AirflowRole, CONTAINERDEBUG_LOG_DIR, HTTP_PORT, LOG_CONFIG_DIR, - TEMPLATE_LOCATION, TEMPLATE_NAME, + AirflowExecutor, AirflowRole, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -189,7 +189,12 @@ pub fn build_airflow_statefulset_envs( // other telemetry-based process in it would log into the same directory and the Vector // agent would label its events with `container: containerdebug`. env_vars = env_vars - .with_value(&FILE_LOG_DIRECTORY, CONTAINERDEBUG_LOG_DIR) + // The directory must be exactly one level below the log directory, because the Vector + // agent derives the container name from that path segment. + .with_value( + &FILE_LOG_DIRECTORY, + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), + ) .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") .with_value(&FILE_LOG_MAX_FILES, "2") // apply overrides last of all; `EnvVarSet` is keyed by name, so iteration is already diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 50c3742b..b7cf46a2 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,5 @@ use std::{collections::BTreeSet, ops::Deref, str::FromStr}; -use const_format::concatcp; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -85,11 +84,6 @@ pub const CONFIG_PATH: &str = "/stackable/app/config"; pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config"; pub const AIRFLOW_HOME: &str = "/stackable/airflow"; -/// Directory which the `containerdebug` process writes its log files to. It must be exactly one -/// level below the log directory, because the Vector agent derives the container name from that -/// path segment. -pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); - constant!(pub TEMPLATE_VOLUME_NAME: VolumeName = "airflow-executor-pod-template"); pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; @@ -786,7 +780,7 @@ fn container_debug_command() -> String { // prune old log files before it creates the first one, which complains about the directory not // existing yet. format!( - "mkdir --parents {CONTAINERDEBUG_LOG_DIR}\n\ + "mkdir --parents \"$FILE_LOG_DIRECTORY\"\n\ containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &" ) }