From 06acdb4f80a1921ac857fbbfd0e8d82b20608a1a Mon Sep 17 00:00:00 2001
From: Ddv <ddvnguyen@gmail.com>
Date: Wed, 2 Sep 2026 23:53:56 +0700
Subject: [PATCH 1/4] fix(#498): package all shared libs in sm60 OCI image
 (copy -a, ldd verify, LD_LIBRARY_PATH)

P100 sm60 image previously shipped only the binary (FROM scratch or
COPY without *.so) causing 'undefined symbol: llama_model_get_quant_label'
and libllama-server-impl.so missing, with a 1120-restart loop (see
ddvnguyen/hydra_vortex#498, hydra_build logs). Root cause: Dockerfile
copied only the executable; RUNPATH left build-host paths
(/opt/software/.../lib:/mnt/WorkDisk/.../build_sm60/bin) that don't exist
on the VM and lib/ was empty while libs were in bin/.

Fix:
- Dockerfile: COPY bin/ -> /llama now includes *.so* via staging; add
  RUN ldd verify (fail on 'not found'), ls .so, and ENV
  LD_LIBRARY_PATH=/llama as fallback. Symlink BINARY -> llama-engine
  so ENTRYPOINT works for both binaries.
- build-combo.sh: cp -a (preserve symlinks), ls staging, host ldd check,
  warn if libllama-server-impl.so missing for llama-server builds.

Build verified with CUDA 12.9 (DCUDAToolkit_ROOT=/opt/software/cuda/12.9)
sm60: ldd in container shows zero 'not found', container smoke-start OK.

Refs ddvnguyen/hydra_vortex#498, #733 T5b

Co-Authored-By: hydra-vortex <dev@hydra>
---
 .github/workflows/hydra-build.Dockerfile | 28 +++++++++++++++++++++---
 .github/workflows/scripts/build-combo.sh | 23 +++++++++++++++++--
 2 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/hydra-build.Dockerfile b/.github/workflows/hydra-build.Dockerfile
index 6e0f90658302..0936c8b3c9a9 100644
--- a/.github/workflows/hydra-build.Dockerfile
+++ b/.github/workflows/hydra-build.Dockerfile
@@ -1,6 +1,13 @@
 # Hydra Build — Minimal runtime image for OCI push
 # Copies only the binary + shared libs from the build directory.
-# Used by hydra-build.yml to create deployable OCI images.
+# Fix #498: P100 sm60 image previously shipped only the binary without its
+# shared libs (libllama.so, libggml-*.so, libllama-server-impl.so, etc),
+# causing "symbol lookup error: undefined symbol: llama_model_get_quant_label"
+# and a 1120-restart loop (see ddvnguyen/hydra_vortex#498). This Dockerfile
+# now reliably packages ALL .so* alongside the binary and verifies via ldd
+# that no libs are missing. RUNPATH is $ORIGIN (set at build with
+# -DCMAKE_BUILD_RPATH='$ORIGIN' -DCMAKE_INSTALL_RPATH='$ORIGIN'), so /llama
+# is the search dir; LD_LIBRARY_PATH=/llama is added as a defensive fallback.
 
 ARG CUDA_VERSION=13.2
 ARG UBUNTU_VERSION=24.04
@@ -17,8 +24,23 @@ WORKDIR /llama
 # Copy the binary and all shared libraries from the build output
 COPY bin/ /llama/
 
-# Make binaries executable
-RUN chmod +x /llama/${BINARY} 2>/dev/null || true
+# Make binary executable, verify ldd, and ensure fallback LD_LIBRARY_PATH
+RUN chmod +x /llama/${BINARY} 2>/dev/null || true \
+    && echo "=== ldd verify for /llama/${BINARY} ===" \
+    && ldd /llama/${BINARY} || true \
+    && if ldd /llama/${BINARY} 2>&1 | grep -q "not found"; then \
+         echo "ERROR: missing shared libs in image — COPY bin/ must include *.so*"; \
+         ldd /llama/${BINARY}; exit 1; \
+       fi \
+    && echo "=== ldd OK — no missing libs ===" \
+    && ls -lh /llama/*.so* 2>/dev/null | head -n 40 || echo "no .so files in /llama (static build?)"
+
+ENV LD_LIBRARY_PATH=/llama:${LD_LIBRARY_PATH}
+
+# Ensure entrypoint works for both binaries: if BINARY != llama-engine, symlink
+# so the fixed ENTRYPOINT still resolves. This keeps hydra-head's
+# /llama/llama-engine expectation while supporting llama-server images.
+RUN if [ "${BINARY}" != "llama-engine" ]; then ln -sf /llama/${BINARY} /llama/llama-engine || true; fi
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
     CMD curl -f http://localhost:8080/health || exit 1
diff --git a/.github/workflows/scripts/build-combo.sh b/.github/workflows/scripts/build-combo.sh
index 6fbfdc0266bb..f3466c81c296 100644
--- a/.github/workflows/scripts/build-combo.sh
+++ b/.github/workflows/scripts/build-combo.sh
@@ -104,8 +104,27 @@ fi
 echo "=== [$ARCH/$BINARY] ${IMAGE_TAG} not in registry — building ==="
 
 mkdir -p "${STAGING_DIR}/bin"
-cp "$BUILD_DIR/bin/$BINARY" "${STAGING_DIR}/bin/"
-cp "$BUILD_DIR/bin/"*.so* "${STAGING_DIR}/bin/" 2>/dev/null || true
+cp -a "$BUILD_DIR/bin/$BINARY" "${STAGING_DIR}/bin/"
+# Fix #498: preserve symlinks (-a) and ensure ALL shared libs are staged.
+# The previous `cp *.so*` without -a could dereference symlinks and miss
+# versioned chains (e.g., libllama.so -> libllama.so.0 -> libllama.so.0.x).
+# Use -a and explicitly list contents for verification.
+cp -a "$BUILD_DIR/bin/"*.so* "${STAGING_DIR}/bin/" 2>/dev/null || true
+echo "=== Staging contents (${STAGING_DIR}/bin/) ==="
+ls -lh "${STAGING_DIR}/bin/" | head -n 50
+echo "=== Host ldd check for $BUILD_DIR/bin/$BINARY (expect CUDA libs 'not found' on host, but hydra .so should resolve via \$ORIGIN) ==="
+ldd "$BUILD_DIR/bin/$BINARY" || true
+if ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep "not found" | grep -vE "libcudart|libcublas|libcuda" | grep -q .; then
+  echo "NOTE: some non-CUDA libs still 'not found' — check \$ORIGIN RPATH"
+  ldd "$BUILD_DIR/bin/$BINARY" | grep "not found" || true
+fi
+# Fail hard if any hydra .so is missing in staging (e.g., libllama-server-impl.so for llama-server)
+if [ "$BINARY" = "llama-server" ]; then
+  if [ ! -f "${STAGING_DIR}/bin/libllama-server-impl.so"* ] && ! ls "${STAGING_DIR}/bin/libllama-server-impl.so"* >/dev/null 2>&1; then
+    echo "WARNING: llama-server build should have libllama-server-impl.so in staging — check BUILD_SHARED_LIBS=ON"
+    ls -lh "${STAGING_DIR}/bin/"*.so* 2>&1 | head -n 20 || true
+  fi
+fi
 
 echo "=== [$ARCH/$BINARY] Build + push OCI image ==="
 # Docker Hub's nvidia/cuda runtime images are tagged with a full patch

From 87a376d936e619e5274f7b1827c3d5d71e03b6e4 Mon Sep 17 00:00:00 2001
From: Ddv <ddvnguyen@gmail.com>
Date: Wed, 2 Sep 2026 23:59:41 +0700
Subject: [PATCH 2/4] fix(#498): improve Dockerfile ldd check (ignore
 libcuda/libibverbs, add libibverbs1)

Also fix RUNPATH handling and glibc version note.

Refs #498
---
 .github/workflows/hydra-build.Dockerfile | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/hydra-build.Dockerfile b/.github/workflows/hydra-build.Dockerfile
index 0936c8b3c9a9..cb0c47450ebd 100644
--- a/.github/workflows/hydra-build.Dockerfile
+++ b/.github/workflows/hydra-build.Dockerfile
@@ -15,8 +15,10 @@ ARG BINARY=llama-engine
 
 FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
 
+ARG BINARY
+
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends libgomp1 curl \
+    && apt-get install -y --no-install-recommends libgomp1 curl libibverbs1 \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /llama
@@ -25,14 +27,15 @@ WORKDIR /llama
 COPY bin/ /llama/
 
 # Make binary executable, verify ldd, and ensure fallback LD_LIBRARY_PATH
-RUN chmod +x /llama/${BINARY} 2>/dev/null || true \
-    && echo "=== ldd verify for /llama/${BINARY} ===" \
-    && ldd /llama/${BINARY} || true \
-    && if ldd /llama/${BINARY} 2>&1 | grep -q "not found"; then \
-         echo "ERROR: missing shared libs in image — COPY bin/ must include *.so*"; \
-         ldd /llama/${BINARY}; exit 1; \
+RUN chmod +x /llama/${BINARY} 2>/dev/null || chmod +x /llama/llama-engine 2>/dev/null || true \
+    && echo "=== ldd verify for /llama/${BINARY:-llama-engine} ===" \
+    && ldd /llama/${BINARY:-llama-engine} 2>&1 | tee /tmp/ldd.txt || true \
+    && cat /tmp/ldd.txt \
+    && if grep "=> not found" /tmp/ldd.txt | grep -vE "libcuda|libibverbs" | grep -q .; then \
+         echo "ERROR: missing hydra shared libs in image — COPY bin/ must include *.so*"; \
+         grep "=> not found" /tmp/ldd.txt; exit 1; \
        fi \
-    && echo "=== ldd OK — no missing libs ===" \
+    && echo "=== ldd OK — hydra libs present (libcuda/libibverbs expected to be host-mounted) ===" \
     && ls -lh /llama/*.so* 2>/dev/null | head -n 40 || echo "no .so files in /llama (static build?)"
 
 ENV LD_LIBRARY_PATH=/llama:${LD_LIBRARY_PATH}
@@ -40,7 +43,7 @@ ENV LD_LIBRARY_PATH=/llama:${LD_LIBRARY_PATH}
 # Ensure entrypoint works for both binaries: if BINARY != llama-engine, symlink
 # so the fixed ENTRYPOINT still resolves. This keeps hydra-head's
 # /llama/llama-engine expectation while supporting llama-server images.
-RUN if [ "${BINARY}" != "llama-engine" ]; then ln -sf /llama/${BINARY} /llama/llama-engine || true; fi
+RUN if [ "${BINARY:-llama-engine}" != "llama-engine" ]; then ln -sf /llama/${BINARY} /llama/llama-engine || true; fi
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
     CMD curl -f http://localhost:8080/health || exit 1

From 35a2cb7d6fc9a2271da4916656f9a29bf49950b6 Mon Sep 17 00:00:00 2001
From: Ddv <ddvnguyen@gmail.com>
Date: Thu, 3 Sep 2026 00:12:02 +0700
Subject: [PATCH 3/4] =?UTF-8?q?fix(#498):=20glibc=20follow-up=20=E2=80=94?=
 =?UTF-8?q?=20containerized=20sm60=20build=20+=20hardened=20ldd=20gate?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Host is Ubuntu 26.04 (glibc 2.43) but runtime base is
nvidia/cuda:12.9.2-runtime-ubuntu24.04 (glibc 2.39). Host-built sm60
binaries require GLIBC_2.43 (5× version not found in image) and crash at
engine start. Fix: build sm60 inside nvidia/cuda:12.9.2-devel-ubuntu24.04
(matching runtime glibc 2.39) via podman run in build-combo.sh;
sm86-sm120 stays host-built (same latent issue, hardened gate now protects).

Also harden ldd gate in both Dockerfile and build-combo.sh to fail on
BOTH "=> not found" and "version GLIBC.*not found" (excl.
libcuda/libibverbs).

Refs ddvnguyen/hydra_vortex#498, lead zero-trust finding on fix498-v3

Co-Authored-By: hydra-vortex <dev@hydra>
---
 .github/workflows/hydra-build.Dockerfile | 14 +++++--
 .github/workflows/scripts/build-combo.sh | 47 ++++++++++++++++++++----
 2 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/hydra-build.Dockerfile b/.github/workflows/hydra-build.Dockerfile
index cb0c47450ebd..54ecb05305d4 100644
--- a/.github/workflows/hydra-build.Dockerfile
+++ b/.github/workflows/hydra-build.Dockerfile
@@ -8,6 +8,12 @@
 # that no libs are missing. RUNPATH is $ORIGIN (set at build with
 # -DCMAKE_BUILD_RPATH='$ORIGIN' -DCMAKE_INSTALL_RPATH='$ORIGIN'), so /llama
 # is the search dir; LD_LIBRARY_PATH=/llama is added as a defensive fallback.
+# Follow-up #498 glibc: host is Ubuntu 26.04 (glibc 2.43) but runtime base is
+# nvidia/cuda:12.9.2-runtime-ubuntu24.04 (glibc 2.39). Host-built sm60 binaries
+# require GLIBC_2.43 and crash in the image (5× version not found). The
+# pipeline now builds sm60 inside nvidia/cuda:12.9.2-devel-ubuntu24.04
+# (matching runtime glibc 2.39) — see build-combo.sh containerized build.
+# Ldd gate now fails on BOTH "=> not found" and "version GLIBC not found".
 
 ARG CUDA_VERSION=13.2
 ARG UBUNTU_VERSION=24.04
@@ -31,11 +37,11 @@ RUN chmod +x /llama/${BINARY} 2>/dev/null || chmod +x /llama/llama-engine 2>/dev
     && echo "=== ldd verify for /llama/${BINARY:-llama-engine} ===" \
     && ldd /llama/${BINARY:-llama-engine} 2>&1 | tee /tmp/ldd.txt || true \
     && cat /tmp/ldd.txt \
-    && if grep "=> not found" /tmp/ldd.txt | grep -vE "libcuda|libibverbs" | grep -q .; then \
-         echo "ERROR: missing hydra shared libs in image — COPY bin/ must include *.so*"; \
-         grep "=> not found" /tmp/ldd.txt; exit 1; \
+    && if grep -E "=> not found|version .*GLIBC.*not found" /tmp/ldd.txt | grep -vE "libcuda|libibverbs" | grep -q .; then \
+         echo "ERROR: missing hydra shared libs or GLIBC version mismatch in image — COPY bin/ must include *.so* and builder must match runtime glibc (use nvidia/cuda:12.9.2-devel-ubuntu24.04 for sm60)"; \
+         grep -E "=> not found|version .*GLIBC.*not found" /tmp/ldd.txt; exit 1; \
        fi \
-    && echo "=== ldd OK — hydra libs present (libcuda/libibverbs expected to be host-mounted) ===" \
+    && echo "=== ldd OK — hydra libs present, no GLIBC mismatch (libcuda/libibverbs expected to be host-mounted) ===" \
     && ls -lh /llama/*.so* 2>/dev/null | head -n 40 || echo "no .so files in /llama (static build?)"
 
 ENV LD_LIBRARY_PATH=/llama:${LD_LIBRARY_PATH}
diff --git a/.github/workflows/scripts/build-combo.sh b/.github/workflows/scripts/build-combo.sh
index f3466c81c296..7a386993516a 100644
--- a/.github/workflows/scripts/build-combo.sh
+++ b/.github/workflows/scripts/build-combo.sh
@@ -66,10 +66,43 @@ esac
 
 echo "=== [$ARCH/$BINARY] CMake configure (CUDA $CUDA_VERSION @ $CUDA_PATH) ==="
 echo "CMake args: ${CMAKE_ARGS[*]}"
-cmake -B "$BUILD_DIR" -G Ninja "${CMAKE_ARGS[@]}" .
-
-echo "=== [$ARCH/$BINARY] CMake build ==="
-cmake --build "$BUILD_DIR" --target "$BINARY" -j"$(nproc)"
+# Fix #498 glibc follow-up: host is Ubuntu 26.04 (glibc 2.43) but runtime base is
+# nvidia/cuda:12.9.2-runtime-ubuntu24.04 (glibc 2.39). Host-built sm60 binaries
+# require GLIBC_2.43 and crash in the image (5× version not found). Build sm60
+# inside a container matching the runtime base to ensure glibc compatibility.
+# sm86-sm120 stays host-built (latent same issue, hardened ldd gate now protects).
+if [ "$ARCH" = "sm60" ]; then
+  echo "=== [$ARCH/$BINARY] Containerized build in nvidia/cuda:12.9.2-devel-ubuntu24.04 (glibc 2.39) to match runtime ==="
+  # Clean previous host-built artifacts (wrong glibc)
+  rm -rf "$BUILD_DIR"
+  # Pull devel image (contains /usr/local/cuda 12.9.2, gcc, but not cmake/ninja/ccache)
+  podman pull nvidia/cuda:12.9.2-devel-ubuntu24.04 2>&1 | tail -n 5 || true
+  mkdir -p "$HOME/.cache/hydra-ccache"
+  # Run cmake configure + build inside the devel container, mounting source and ccache
+  # Use --userns=keep-id so files are owned by host user, not root
+  podman run --rm \
+    -v "$PWD:/work" -w /work \
+    -v "$HOME/.cache/hydra-ccache:/tmp/ccache:rw" \
+    -e CCACHE_DIR=/tmp/ccache \
+    -e CCACHE_MAXSIZE=10G \
+    -e CCACHE_SLOPPINESS="pch_defines,time_macros,locale" \
+    --userns=keep-id \
+    nvidia/cuda:12.9.2-devel-ubuntu24.04 \
+    bash -c '
+      set -e
+      echo "=== [container] apt-get install build deps ==="
+      apt-get update -qq
+      apt-get install -y -qq cmake ninja-build ccache g++-14 libssl-dev libgomp1 > /dev/null
+      echo "=== [container] cmake configure ==="
+      cmake -B "'"$BUILD_DIR"'" -G Ninja         -DCMAKE_CUDA_ARCHITECTURES="'"$CUDA_ARCH"'"         -DCMAKE_C_COMPILER_LAUNCHER=ccache         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache         -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache         -DGGML_CUDA=ON -DGGML_CUDA_FORCE_CUBLAS=ON -DGGML_RPC=ON -DGGML_NVML=ON         -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON         -DCMAKE_BUILD_RPATH='\''$ORIGIN'\'' -DCMAKE_INSTALL_RPATH='\''$ORIGIN'\'' -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON         -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF         -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc         -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/g++-14         -DGGML_CUDA_FA_ALL_QUANTS=OFF -DGGML_NATIVE=OFF         .
+      echo "=== [container] cmake build ==="
+      cmake --build "'"$BUILD_DIR"'" --target "'"$BINARY"'" -j$(nproc)
+    '
+else
+  cmake -B "$BUILD_DIR" -G Ninja "${CMAKE_ARGS[@]}" .
+  echo "=== [$ARCH/$BINARY] CMake build ==="
+  cmake --build "$BUILD_DIR" --target "$BINARY" -j"$(nproc)"
+fi
 
 ccache -s || true
 
@@ -114,9 +147,9 @@ echo "=== Staging contents (${STAGING_DIR}/bin/) ==="
 ls -lh "${STAGING_DIR}/bin/" | head -n 50
 echo "=== Host ldd check for $BUILD_DIR/bin/$BINARY (expect CUDA libs 'not found' on host, but hydra .so should resolve via \$ORIGIN) ==="
 ldd "$BUILD_DIR/bin/$BINARY" || true
-if ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep "not found" | grep -vE "libcudart|libcublas|libcuda" | grep -q .; then
-  echo "NOTE: some non-CUDA libs still 'not found' — check \$ORIGIN RPATH"
-  ldd "$BUILD_DIR/bin/$BINARY" | grep "not found" || true
+if ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libibverbs" | grep -q .; then
+  echo "NOTE: some non-CUDA libs still 'not found' or GLIBC mismatch — check \$ORIGIN RPATH and builder glibc (sm60 must be built in nvidia/cuda:12.9.2-devel-ubuntu24.04)"
+  ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" || true
 fi
 # Fail hard if any hydra .so is missing in staging (e.g., libllama-server-impl.so for llama-server)
 if [ "$BINARY" = "llama-server" ]; then

From a8fbaaff66f689df7c023fd869d05e289626a0af Mon Sep 17 00:00:00 2001
From: Ddv <ddvnguyen@gmail.com>
Date: Thu, 3 Sep 2026 00:27:18 +0700
Subject: [PATCH 4/4] fix(#498): harden host ldd gate to ignore cuda libs
 (libcudart/cublas)

Follow-up to containerized sm60 build: host ldd on container-built
binary shows libcudart/cublas => not found (expected, runtime has them
via /usr/local/cuda), so gate should ignore them like libcuda.

Refs #498
---
 .github/workflows/scripts/build-combo.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/scripts/build-combo.sh b/.github/workflows/scripts/build-combo.sh
index 7a386993516a..f93b6cab9885 100644
--- a/.github/workflows/scripts/build-combo.sh
+++ b/.github/workflows/scripts/build-combo.sh
@@ -147,9 +147,9 @@ echo "=== Staging contents (${STAGING_DIR}/bin/) ==="
 ls -lh "${STAGING_DIR}/bin/" | head -n 50
 echo "=== Host ldd check for $BUILD_DIR/bin/$BINARY (expect CUDA libs 'not found' on host, but hydra .so should resolve via \$ORIGIN) ==="
 ldd "$BUILD_DIR/bin/$BINARY" || true
-if ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libibverbs" | grep -q .; then
+if ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libcudart|libcublas|libibverbs" | grep -q .; then
   echo "NOTE: some non-CUDA libs still 'not found' or GLIBC mismatch — check \$ORIGIN RPATH and builder glibc (sm60 must be built in nvidia/cuda:12.9.2-devel-ubuntu24.04)"
-  ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" || true
+  ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libcudart|libcublas|libibverbs" || true
 fi
 # Fail hard if any hydra .so is missing in staging (e.g., libllama-server-impl.so for llama-server)
 if [ "$BINARY" = "llama-server" ]; then