From 91a7f505b6104b005bbb1c88b3d2ec89a8affdd4 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 16:11:18 +0100 Subject: [PATCH 1/7] feat: Collect the containerdebug logs --- CHANGELOG.md | 7 + .../druid/pages/usage-guide/logging.adoc | 8 + .../src/controller/build/container.rs | 32 +- .../build/properties/product_logging/mod.rs | 2 + .../product_logging/vector-test.yaml | 279 ++++++++++++++++++ .../properties/product_logging/vector.yaml | 87 ++++++ .../controller/build/resource/statefulset.rs | 8 +- rust/operator-binary/src/crd/mod.rs | 7 + .../01_druid-vector-aggregator-values.yaml.j2 | 188 +++++++++--- 9 files changed, 571 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea9ac8730..9ab65ca3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file. - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field ([#876]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#880]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#893]). ### Changed @@ -48,6 +50,10 @@ All notable changes to this project will be documented in this file. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#860]). - The operator now watches all resources that it creates and early-exits the reconcile action when the cluster is marked for deletion ([#867]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it + moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is + rotated hourly and at most two files are kept, because rotation is opt-in ([#893]). ### Remove @@ -71,6 +77,7 @@ All notable changes to this project will be documented in this file. [#889]: https://github.com/stackabletech/druid-operator/pull/889 [#891]: https://github.com/stackabletech/druid-operator/pull/891 [#892]: https://github.com/stackabletech/druid-operator/pull/892 +[#893]: https://github.com/stackabletech/druid-operator/pull/893 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/druid/pages/usage-guide/logging.adoc b/docs/modules/druid/pages/usage-guide/logging.adoc index 83388b818..c07c2fc7c 100644 --- a/docs/modules/druid/pages/usage-guide/logging.adoc +++ b/docs/modules/druid/pages/usage-guide/logging.adoc @@ -30,4 +30,12 @@ spec: enableVectorAgent: true ---- +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `druid` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/druid` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index ef3358e16..5d1aa59ed 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -39,8 +39,9 @@ use crate::{ }, }, crd::{ - Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, HDFS_CONFIG_DIRECTORY, - LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, + CONTAINERDEBUG_LOG_DIR, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, + HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, + RW_CONFIG_DIRECTORY, }, }; @@ -51,8 +52,13 @@ stackable_operator::constant!(pub(crate) LOG_CONFIG_VOLUME_NAME: VolumeName = "l stackable_operator::constant!(pub(crate) LOG_VOLUME_NAME: VolumeName = "log"); stackable_operator::constant!(pub(crate) RW_CONFIG_VOLUME_NAME: VolumeName = "rwconfig"); -// Needed for the `containerdebug` process to log its tracing information to. -stackable_operator::constant!(pub(crate) CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); +// Let the `containerdebug` process write its tracing information to a file. These are the generic +// `stackable-telemetry` variables and apply to the whole Druid container, so any other +// telemetry-based process in it would log into the same directory and the Vector agent would label +// its events with `container: containerdebug`. +stackable_operator::constant!(pub(crate) FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); +stackable_operator::constant!(pub(crate) FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +stackable_operator::constant!(pub(crate) FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); // volume mount directory (not a volume name) const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; @@ -269,10 +275,11 @@ pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result = env .iter() - .filter(|env_var| env_var.name == CONTAINERDEBUG_LOG_DIRECTORY.as_ref()) + .filter(|env_var| env_var.name == FILE_LOG_DIRECTORY.as_ref()) .collect(); assert_eq!( containerdebug.len(), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 0a6697f9d..55ef2f2de 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,5 +1,6 @@ use std::{collections::HashSet, ops::Deref, str::FromStr}; +use const_format::concatcp; use indoc::formatdoc; use security::add_cert_to_jvm_trust_store_cmd; use serde::{Deserialize, Serialize}; @@ -70,6 +71,11 @@ pub const HDFS_CONFIG_DIRECTORY: &str = "/stackable/hdfs"; pub const LOG_CONFIG_DIRECTORY: &str = "/stackable/log_config"; pub const RW_CONFIG_DIRECTORY: &str = "/stackable/rwconfig"; +/// Directory which the `containerdebug` process writes its log files to. It must be exactly one +/// level below the log directory, because the Vector agent derives the container name from that +/// path segment. +pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); + pub const TRUST_STORE_FILE: &str = "truststore.p12"; pub const KEY_STORE_FILE: &str = "keystore.p12"; @@ -485,6 +491,7 @@ impl DruidRole { {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + mkdir --parents {CONTAINERDEBUG_LOG_DIR} containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & /stackable/druid/bin/run-druid {process_name} {RW_CONFIG_DIRECTORY} & echo \"$!\" >> /tmp/DRUID_PID diff --git a/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 index b8cc3c1af..c613ff199 100644 --- a/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 @@ -22,191 +22,293 @@ customConfig: transforms: validEvents: type: filter - inputs: [vector] + inputs: + - vector condition: is_null(.errors) filteredAutomaticLogConfigBrokerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigBrokerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigBrokerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-broker-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigBrokerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigBrokerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "druid" filteredCustomLogConfigBrokerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigBrokerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-broker-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigBrokerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigCoordinatorDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigCoordinatorPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigCoordinatorContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-coordinator-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigCoordinatorVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigCoordinatorDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "druid" filteredCustomLogConfigCoordinatorPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigCoordinatorContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-coordinator-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigCoordinatorVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigHistoricalDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigHistoricalPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigHistoricalContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-historical-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigHistoricalVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigHistoricalDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "druid" filteredCustomLogConfigHistoricalPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigHistoricalContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-historical-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigHistoricalVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigMiddlemanagerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigMiddlemanagerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigMiddlemanagerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-middlemanager-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigMiddlemanagerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigMiddlemanagerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "druid" filteredCustomLogConfigMiddlemanagerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigMiddlemanagerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-middlemanager-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigMiddlemanagerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigRouterDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigRouterPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigRouterContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-router-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigRouterVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigRouterDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "druid" filteredCustomLogConfigRouterPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigRouterContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-router-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigRouterVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "vector" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -214,11 +316,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: From 1a0e998915f928a810549d965098c2783955577d Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 16:14:44 +0100 Subject: [PATCH 2/7] chore: Update changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ab65ca3f..2e2bab9aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file. ([#876]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#880]). - The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to - the Vector aggregator ([#893]). + the Vector aggregator ([#894]). ### Changed @@ -53,7 +53,7 @@ All notable changes to this project will be documented in this file. - The `containerdebug` process writes its log file again. The operator set `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is - rotated hourly and at most two files are kept, because rotation is opt-in ([#893]). + rotated hourly and at most two files are kept, because rotation is opt-in ([#894]). ### Remove @@ -77,7 +77,7 @@ All notable changes to this project will be documented in this file. [#889]: https://github.com/stackabletech/druid-operator/pull/889 [#891]: https://github.com/stackabletech/druid-operator/pull/891 [#892]: https://github.com/stackabletech/druid-operator/pull/892 -[#893]: https://github.com/stackabletech/druid-operator/pull/893 +[#894]: https://github.com/stackabletech/druid-operator/pull/894 ## [26.7.0] - 2026-07-21 From 73c0fba9bf18a40aaa745df9362921c9b5562eb9 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 16:58:00 +0100 Subject: [PATCH 3/7] chore: Update comment --- rust/operator-binary/src/controller/build/container.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index 5d1aa59ed..78b7a8498 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -460,11 +460,9 @@ fn add_log_volume_and_volume_mounts( VolumeBuilder::new(&*LOG_VOLUME_NAME) .with_empty_dir( Some(""), - // Only the Druid log files are budgeted here. The `containerdebug` files in - // this volume are rotated by time, which bounds their number but not their - // size, so they are an unbudgeted writer. They stay negligible at the default - // INFO level and a 30 minute loop, but raising `FILE_LOG_LEVEL` via - // `envOverrides` can exceed the limit, which makes the kubelet evict the Pod. + // Only the Druid log files are budgeted. The `containerdebug` files in this + // volume are rotated by time, not by size, so they have no budget; raising + // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_DRUID_LOG_FILES_SIZE], )), From 5be56424ef5e01cce5696f74e473a8912d67684c Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 17:51:09 +0100 Subject: [PATCH 4/7] chore: Create the containerdebug log directory via the environment variable --- .../src/controller/build/container.rs | 13 +++++++++---- rust/operator-binary/src/crd/mod.rs | 8 +------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index 78b7a8498..7af983814 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -7,6 +7,7 @@ use std::str::FromStr; +use const_format::concatcp; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::pod::{ @@ -39,9 +40,8 @@ use crate::{ }, }, crd::{ - CONTAINERDEBUG_LOG_DIR, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, - HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, - RW_CONFIG_DIRECTORY, + Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, HDFS_CONFIG_DIRECTORY, + LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, }, }; @@ -276,7 +276,12 @@ pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result> /tmp/DRUID_PID From 59891cd5debff16d261f40bfcdf34fcf681100da Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 9 Nov 2026 15:14:56 +0100 Subject: [PATCH 5/7] fix: Do not hard-code the environment variable name "FILE_LOG_DIRECTORY" --- .../src/controller/build/container.rs | 18 ++++-------------- .../controller/build/resource/statefulset.rs | 7 ++----- rust/operator-binary/src/crd/mod.rs | 13 ++++++++++++- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index 7af983814..35719b81f 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -18,7 +18,7 @@ use stackable_operator::{ }, product_logging, v2::{ - builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, + builder::pod::container::{EnvVarSet, new_container_builder}, product_logging::framework::{ STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, }, @@ -40,8 +40,9 @@ use crate::{ }, }, crd::{ - Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, HDFS_CONFIG_DIRECTORY, - LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, + Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, FILE_LOG_DIRECTORY, FILE_LOG_MAX_FILES, + FILE_LOG_ROTATION_PERIOD, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, + METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, }, }; @@ -52,14 +53,6 @@ stackable_operator::constant!(pub(crate) LOG_CONFIG_VOLUME_NAME: VolumeName = "l stackable_operator::constant!(pub(crate) LOG_VOLUME_NAME: VolumeName = "log"); stackable_operator::constant!(pub(crate) RW_CONFIG_VOLUME_NAME: VolumeName = "rwconfig"); -// Let the `containerdebug` process write its tracing information to a file. These are the generic -// `stackable-telemetry` variables and apply to the whole Druid container, so any other -// telemetry-based process in it would log into the same directory and the Vector agent would label -// its events with `container: containerdebug`. -stackable_operator::constant!(pub(crate) FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); -stackable_operator::constant!(pub(crate) FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); -stackable_operator::constant!(pub(crate) FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); - // volume mount directory (not a volume name) const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; @@ -488,8 +481,5 @@ mod tests { let _ = *LOG_CONFIG_VOLUME_NAME; let _ = *LOG_VOLUME_NAME; let _ = *RW_CONFIG_VOLUME_NAME; - let _ = *FILE_LOG_DIRECTORY; - let _ = *FILE_LOG_MAX_FILES; - let _ = *FILE_LOG_ROTATION_PERIOD; } } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index b083885fb..ad356be6f 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -126,13 +126,10 @@ mod tests { use super::*; use crate::{ controller::{ - build::{ - container::{FILE_LOG_DIRECTORY, LOG_VOLUME_NAME}, - test_support::role_group_builder, - }, + build::{container::LOG_VOLUME_NAME, test_support::role_group_builder}, validate::test_support::{MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster}, }, - crd::{Container, DruidRole}, + crd::{Container, DruidRole, FILE_LOG_DIRECTORY}, }; /// The user-supplied `envOverrides` must be merged in after all operator-set environment diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 16bad7433..42ad5b033 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -91,6 +91,13 @@ pub const METRICS_PORT: Port = Port(9090); constant!(pub COOKIE_PASSPHRASE_ENV: EnvVarName = "OIDC_COOKIE_PASSPHRASE"); constant!(pub COOKIE_PASSPHRASE_SECRET_KEY: SecretKey = "OIDC_COOKIE_PASSPHRASE"); +// The `stackable-telemetry` variables which configure the file logs of the `containerdebug` +// process. They are set on the Druid container, and the start command below references the +// directory as a shell variable, so that an `envOverrides` entry takes effect in both places. +constant!(pub FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); +constant!(pub FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +constant!(pub FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); + /// Formats a Druid [dynamic config](https://druid.apache.org/docs/latest/operations/dynamic-config-provider) /// reference to an environment variable, i.e. `${env:NAME}`. pub(crate) fn env_var_reference(name: impl std::fmt::Display) -> String { @@ -485,7 +492,7 @@ impl DruidRole { {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers - mkdir --parents \"$FILE_LOG_DIRECTORY\" + mkdir --parents \"${file_log_directory}\" || true containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & /stackable/druid/bin/run-druid {process_name} {RW_CONFIG_DIRECTORY} & echo \"$!\" >> /tmp/DRUID_PID @@ -493,6 +500,7 @@ impl DruidRole { {create_vector_shutdown_file_command} ", process_name = self.get_process_name(), + file_log_directory = FILE_LOG_DIRECTORY.as_ref(), remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), create_vector_shutdown_file_command = @@ -685,6 +693,9 @@ mod tests { let _ = *HISTORICAL_ROLE_NAME; let _ = *MIDDLE_MANAGER_ROLE_NAME; let _ = *ROUTER_ROLE_NAME; + let _ = *FILE_LOG_DIRECTORY; + let _ = *FILE_LOG_MAX_FILES; + let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *COOKIE_PASSPHRASE_ENV; let _ = *COOKIE_PASSPHRASE_SECRET_KEY; let _ = *DRUID_DEFAULT_LISTENER_CLASS; From ba12aad3c388a86a40d4c7323310f13fa15e5625 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 9 Nov 2026 15:54:17 +0100 Subject: [PATCH 6/7] test(smoke): Fix test assertion --- .../properties/product_logging/vector.yaml | 2 +- tests/templates/kuttl/smoke/53-assert.yaml.j2 | 87 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index b9e446d39..f94d39156 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -179,7 +179,6 @@ transforms: } } - # Extends the processed files with the fields "container" and "file" # Transforms the tracing-rs JSON logs written by the containerdebug process processed_files_tracing_rs: inputs: @@ -260,6 +259,7 @@ transforms: } } + # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - processed_files_stdout diff --git a/tests/templates/kuttl/smoke/53-assert.yaml.j2 b/tests/templates/kuttl/smoke/53-assert.yaml.j2 index 5ec8ee2d0..d9979111e 100644 --- a/tests/templates/kuttl/smoke/53-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/53-assert.yaml.j2 @@ -27,6 +27,12 @@ sources: - ${LOG_DIR}/*/*.log4j2.xml line_delimiter: "\r\n" + # Reads the logs of the containerdebug process + files_tracing_rs: + type: file + include: + - ${LOG_DIR}/*/*.tracing-rs.json + transforms: processed_files_stdout: inputs: @@ -175,12 +181,93 @@ transforms: } } + # Transforms the tracing-rs JSON logs written by the containerdebug process + processed_files_tracing_rs: + inputs: + - files_tracing_rs + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%+") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.target) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger/target not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], upcase(level)) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = upcase(level) + } + + fields, err = object(event.fields) + if err == null { + message, err = string(fields.message) + if err == null { + # The message is prepended to the other fields, so remove it from + # them. + del(fields.message) + } else { + # Keep an unusable message among the other fields instead of + # discarding its content. + message = "" + } + + other_fields = encode_key_value(fields, field_delimiter: "\n") + .message = join!(compact([message, other_fields]), "\n\n") + } else { + .errors = push(.errors, "Fields are not an object.") + # Fall back to the raw line, so that its content is not lost. + .message = raw_message + } + + if is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + } + # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - processed_files_stdout - processed_files_stderr - processed_files_log4j2 + - processed_files_tracing_rs type: remap source: | del(.source_type) From 83a7a2a3636933933e0178f7ede238aa826c721d Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 10 Nov 2026 09:36:29 +0100 Subject: [PATCH 7/7] chore: Explain the FILE_LOG_* variables where their values are set --- rust/operator-binary/src/controller/build/container.rs | 4 ++++ rust/operator-binary/src/crd/mod.rs | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index 35719b81f..d07ee0e5d 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -268,6 +268,10 @@ pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result