containers: switch environmentd and clusterd to distroless base image

misc/images/distroless-prod-base/Dockerfile

@@ -10,7 +10,26 @@
 # This is a separate mzimage so that we don't have to re-install the apt things
 # every time we get a CI builder with a cold cache.
 
+# Extract libeatmydata from a Debian image for CI use. eatmydata disables
+# fsync for faster test execution. Activated via LD_PRELOAD=libeatmydata.so
+# set as a container environment variable (must be set before process start).
+FROM debian:13-slim AS eatmydata
+RUN apt-get update && apt-get install -y --no-install-recommends eatmydata \
+    && rm -rf /var/lib/apt/lists/*
+
+# Statically-linked `tini`, the PID 1 init for all distroless prod images.
+# Distroless ships no init, so without this each leaf binary would run as PID 1
+# and have to forward signals and reap zombies (e.g. ssh tunnel subprocesses)
+# itself. Leaf images opt in via ENTRYPOINT ["/usr/bin/tini", "--", <binary>].
+MZFROM tini-static AS tini
+
 FROM gcr.io/distroless/cc-debian13:nonroot-28078d2e5e77671d2046dcc9e2c75334e31efa4d
+
+# Copy libeatmydata for CI performance optimization (no-op unless
+# LD_PRELOAD=libeatmydata.so is set as a container env var).
+COPY --from=eatmydata /usr/lib/*/libeatmydata.so /usr/lib/
+COPY --from=tini /output/tini /usr/bin/tini
+
 USER nonroot
 ENV HOME=/home/nonroot
 ENTRYPOINT []

misc/images/tini-static/Dockerfile

@@ -0,0 +1,49 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+# Build a statically-linked `tini` init binary. tini is the PID 1 we run in
+# every production image: it forwards signals to the child and reaps orphaned
+# zombies (e.g. ssh tunnel subprocesses). The static binary (~600KB, glibc)
+# lets distroless images, which ship no init and no apt, keep using tini
+# without a package manager. This replaces the apt-installed tini from the
+# Debian-based prod-base.
+#
+# tini's CMake build emits both a dynamic `tini` and a static `tini-static`
+# target. We keep only the static one.
+#
+# Usage:
+#   docker build -t tini-static .
+#   docker create --name extract tini-static
+#   docker cp extract:/output/tini ./tini
+#   docker rm extract
+
+FROM ubuntu:noble-20260210.1 AS builder
+
+ARG TINI_VERSION=v0.19.0
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    cmake \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build/tini
+RUN git clone --depth 1 --branch ${TINI_VERSION} https://github.com/krallin/tini.git . \
+    && cmake . \
+    && make tini-static \
+    && strip tini-static
+
+# Verify the binary is not dynamically linked and is functional.
+RUN ! ldd tini-static 2>/dev/null \
+    && ./tini-static --version
+
+# Output stage: just the binary.
+FROM scratch
+COPY --from=builder /build/tini/tini-static /output/tini

src/environmentd/ci/entrypoint.sh → misc/images/tini-static/mzbuild.yml

@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
 # Copyright Materialize, Inc. and contributors. All rights reserved.
 #
 # Use of this software is governed by the Business Source License
@@ -9,17 +7,5 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-set -euo pipefail
-
-if [ -z "${MZ_EAT_MY_DATA:-}" ]; then
-    unset LD_PRELOAD
-else
-    export LD_PRELOAD=libeatmydata.so
-fi
-
-if environmentd "$@"; then
-    echo "environmentd exited gracefully; sleeping forever" >&2
-    sleep infinity
-else
-    exit $?
-fi
+name: tini-static
+description: Statically-linked tini init, the PID 1 used in distroless prod images.

misc/python/materialize/mzcompose/services/clusterd.py

@@ -47,13 +47,13 @@ def __init__(
             "CLUSTERD_USE_CTP=true",
             "MZ_SOFT_ASSERTIONS=1",
             "MZ_EAT_MY_DATA=1",
+            "LD_PRELOAD=libeatmydata.so",
             # Defaults that were previously set by the clusterd entrypoint.sh.
             "CLUSTERD_STORAGE_CONTROLLER_LISTEN_ADDR=0.0.0.0:2100",
             "CLUSTERD_COMPUTE_CONTROLLER_LISTEN_ADDR=0.0.0.0:2101",
             "CLUSTERD_INTERNAL_HTTP_LISTEN_ADDR=0.0.0.0:6878",
             "CLUSTERD_SECRETS_READER=local-file",
             "CLUSTERD_SECRETS_READER_LOCAL_FILE_DIR=/mzdata/secrets",
-            "LD_PRELOAD=libeatmydata.so",
             f"CLUSTERD_PERSIST_PUBSUB_URL=http://{mz_service}:6879",
             *environment_extra,
         ]
@@ -90,6 +90,9 @@ def __init__(
         # Override the materialized entrypoint so that `clusterd` is invoked
         # via the command rather than via the entrypoint. This keeps
         # `c.exec()` working (it prepends the entrypoint to exec commands).
+        # Note: mzcompose uses the Ubuntu-based `materialized` image (with
+        # tini/bash), while production uses the distroless `clusterd` image.
+        # Keep this in mind when debugging CI-vs-prod discrepancies.
         config["entrypoint"] = ["tini", "--"]
 
         # Depending on the Docker Compose version, this may either work or be

src/clusterd/ci/Dockerfile

@@ -7,10 +7,12 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-MZFROM prod-base
+MZFROM openssh-static AS openssh
 
-COPY clusterd entrypoint.sh /usr/local/bin/
+MZFROM distroless-prod-base
 
-USER materialize
+COPY clusterd /usr/local/bin/
+COPY --from=openssh /output/ssh /usr/bin/ssh
 
-ENTRYPOINT ["tini", "--", "entrypoint.sh"]
+# tini (from distroless-prod-base) is PID 1 so ssh tunnel subprocesses get reaped.
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/clusterd"]

src/environmentd/ci/Dockerfile

@@ -7,10 +7,12 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-MZFROM prod-base
+MZFROM openssh-static AS openssh
 
-COPY environmentd entrypoint.sh /usr/local/bin/
+MZFROM distroless-prod-base
 
-USER materialize
+COPY environmentd /usr/local/bin/
+COPY --from=openssh /output/ssh /usr/bin/ssh
 
-ENTRYPOINT ["tini", "--", "entrypoint.sh"]
+# tini (from distroless-prod-base) is PID 1 so ssh tunnel subprocesses get reaped.
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/environmentd"]

src/environmentd/src/deployment/preflight.rs

@@ -41,6 +41,9 @@ pub struct PreflightInput {
     pub ddl_check_interval: Duration,
     pub panic_after_timeout: bool,
     pub bootstrap_args: BootstrapArgs,
+    /// Whether to idle in place, rather than exit, when fenced out by a newer
+    /// generation. See [`crate::Config::idle_when_fenced_out`].
+    pub idle_when_fenced_out: bool,
 }
 
 /// Output of preflight checks.
@@ -67,6 +70,7 @@ pub async fn preflight_0dt(
         ddl_check_interval,
         panic_after_timeout,
         bootstrap_args,
+        idle_when_fenced_out,
     }: PreflightInput,
 ) -> Result<PreflightOutput, CatalogError> {
     info!(%deploy_generation, ?caught_up_max_wait, "performing 0dt preflight checks");
@@ -221,6 +225,14 @@ pub async fn preflight_0dt(
             read_only: false,
             caught_up_trigger: None,
         })
+    } else if idle_when_fenced_out {
+        // Exiting under a Kubernetes StatefulSet (`restartPolicy: Always`) would
+        // restart the container straight back into this fenced-out state, a
+        // crash loop until the orchestrator deletes the StatefulSet. Idle until
+        // pod deletion's SIGTERM terminates us instead.
+        info!("this deployment has been fenced out; idling until terminated");
+        std::future::pending::<()>().await;
+        unreachable!("pending() never resolves");
     } else {
         exit!(0, "this deployment has been fenced out");
     }

src/environmentd/src/environmentd/main.rs

@@ -1103,6 +1103,7 @@ fn run(mut args: Args) -> Result<(), anyhow::Error> {
                 secrets_controller,
                 cloud_resource_controller,
                 system_dyncfgs,
+                idle_when_fenced_out: matches!(args.orchestrator, OrchestratorKind::Kubernetes),
                 // Storage options.
                 storage_usage_collection_interval: args.storage_usage_collection_interval_sec,
                 storage_usage_retention_period: args.storage_usage_retention_period,

src/environmentd/src/lib.rs

@@ -140,6 +140,11 @@ pub struct Config {
     pub cloud_resource_controller: Option<Arc<dyn CloudResourceController>>,
     /// The process-wide live system dyncfg set.
     pub system_dyncfgs: Arc<ConfigSet>,
+    /// Whether to idle in place, rather than exit, when this generation is
+    /// fenced out by a newer one. Set when a restart-on-exit process manager (a
+    /// Kubernetes StatefulSet) supervises the process directly, unset when a
+    /// supervising entrypoint sleeps on graceful exit instead.
+    pub idle_when_fenced_out: bool,
 
     // === Storage options. ===
     /// The interval at which to collect storage usage information.
@@ -654,6 +659,7 @@ impl Listeners {
             panic_after_timeout: enable_0dt_deployment_panic_after_timeout,
             bootstrap_args,
             ddl_check_interval: with_0dt_deployment_ddl_check_interval,
+            idle_when_fenced_out: config.idle_when_fenced_out,
         };
         let PreflightOutput {
             openable_adapter_storage,

src/environmentd/src/test_util.rs

@@ -886,6 +886,7 @@ impl Listeners {
                 secrets_controller,
                 cloud_resource_controller: None,
                 system_dyncfgs,
+                idle_when_fenced_out: false,
                 tls: config.tls,
                 frontegg: config.frontegg,
                 frontegg_oauth_issuer_url: None,

src/sqllogictest/src/runner.rs

@@ -1229,6 +1229,7 @@ impl<'a> RunnerInner<'a> {
             secrets_controller,
             cloud_resource_controller: None,
             system_dyncfgs,
+            idle_when_fenced_out: false,
             tls: None,
             frontegg: None,
             frontegg_oauth_issuer_url: None,