fix(#498): package all shared libs in sm60 OCI image (ldd verify, LD_LIBRARY_PATH)

.github/workflows/hydra-build.Dockerfile

@@ -1,24 +1,55 @@
 # Hydra Build — Minimal runtime image for OCI push
 # Copies only the binary + shared libs from the build directory.
-# Used by hydra-build.yml to create deployable OCI images.
+# Fix #498: P100 sm60 image previously shipped only the binary without its
+# shared libs (libllama.so, libggml-*.so, libllama-server-impl.so, etc),
+# causing "symbol lookup error: undefined symbol: llama_model_get_quant_label"
+# and a 1120-restart loop (see ddvnguyen/hydra_vortex#498). This Dockerfile
+# now reliably packages ALL .so* alongside the binary and verifies via ldd
+# that no libs are missing. RUNPATH is $ORIGIN (set at build with
+# -DCMAKE_BUILD_RPATH='$ORIGIN' -DCMAKE_INSTALL_RPATH='$ORIGIN'), so /llama
+# is the search dir; LD_LIBRARY_PATH=/llama is added as a defensive fallback.
+# Follow-up #498 glibc: host is Ubuntu 26.04 (glibc 2.43) but runtime base is
+# nvidia/cuda:12.9.2-runtime-ubuntu24.04 (glibc 2.39). Host-built sm60 binaries
+# require GLIBC_2.43 and crash in the image (5× version not found). The
+# pipeline now builds sm60 inside nvidia/cuda:12.9.2-devel-ubuntu24.04
+# (matching runtime glibc 2.39) — see build-combo.sh containerized build.
+# Ldd gate now fails on BOTH "=> not found" and "version GLIBC not found".
 
 ARG CUDA_VERSION=13.2
 ARG UBUNTU_VERSION=24.04
 ARG BINARY=llama-engine
 
 FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
 
+ARG BINARY
+
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends libgomp1 curl \
+    && apt-get install -y --no-install-recommends libgomp1 curl libibverbs1 \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /llama
 
 # Copy the binary and all shared libraries from the build output
 COPY bin/ /llama/
 
-# Make binaries executable
-RUN chmod +x /llama/${BINARY} 2>/dev/null || true
+# Make binary executable, verify ldd, and ensure fallback LD_LIBRARY_PATH
+RUN chmod +x /llama/${BINARY} 2>/dev/null || chmod +x /llama/llama-engine 2>/dev/null || true \
+    && echo "=== ldd verify for /llama/${BINARY:-llama-engine} ===" \
+    && ldd /llama/${BINARY:-llama-engine} 2>&1 | tee /tmp/ldd.txt || true \
+    && cat /tmp/ldd.txt \
+    && if grep -E "=> not found|version .*GLIBC.*not found" /tmp/ldd.txt | grep -vE "libcuda|libibverbs" | grep -q .; then \
+         echo "ERROR: missing hydra shared libs or GLIBC version mismatch in image — COPY bin/ must include *.so* and builder must match runtime glibc (use nvidia/cuda:12.9.2-devel-ubuntu24.04 for sm60)"; \
+         grep -E "=> not found|version .*GLIBC.*not found" /tmp/ldd.txt; exit 1; \
+       fi \
+    && echo "=== ldd OK — hydra libs present, no GLIBC mismatch (libcuda/libibverbs expected to be host-mounted) ===" \
+    && ls -lh /llama/*.so* 2>/dev/null | head -n 40 || echo "no .so files in /llama (static build?)"
+
+ENV LD_LIBRARY_PATH=/llama:${LD_LIBRARY_PATH}
+
+# Ensure entrypoint works for both binaries: if BINARY != llama-engine, symlink
+# so the fixed ENTRYPOINT still resolves. This keeps hydra-head's
+# /llama/llama-engine expectation while supporting llama-server images.
+RUN if [ "${BINARY:-llama-engine}" != "llama-engine" ]; then ln -sf /llama/${BINARY} /llama/llama-engine || true; fi
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
     CMD curl -f http://localhost:8080/health || exit 1

.github/workflows/scripts/build-combo.sh

@@ -66,10 +66,43 @@ esac
 
 echo "=== [$ARCH/$BINARY] CMake configure (CUDA $CUDA_VERSION @ $CUDA_PATH) ==="
 echo "CMake args: ${CMAKE_ARGS[*]}"
-cmake -B "$BUILD_DIR" -G Ninja "${CMAKE_ARGS[@]}" .
-
-echo "=== [$ARCH/$BINARY] CMake build ==="
-cmake --build "$BUILD_DIR" --target "$BINARY" -j"$(nproc)"
+# Fix #498 glibc follow-up: host is Ubuntu 26.04 (glibc 2.43) but runtime base is
+# nvidia/cuda:12.9.2-runtime-ubuntu24.04 (glibc 2.39). Host-built sm60 binaries
+# require GLIBC_2.43 and crash in the image (5× version not found). Build sm60
+# inside a container matching the runtime base to ensure glibc compatibility.
+# sm86-sm120 stays host-built (latent same issue, hardened ldd gate now protects).
+if [ "$ARCH" = "sm60" ]; then
+  echo "=== [$ARCH/$BINARY] Containerized build in nvidia/cuda:12.9.2-devel-ubuntu24.04 (glibc 2.39) to match runtime ==="
+  # Clean previous host-built artifacts (wrong glibc)
+  rm -rf "$BUILD_DIR"
+  # Pull devel image (contains /usr/local/cuda 12.9.2, gcc, but not cmake/ninja/ccache)
+  podman pull nvidia/cuda:12.9.2-devel-ubuntu24.04 2>&1 | tail -n 5 || true
+  mkdir -p "$HOME/.cache/hydra-ccache"
+  # Run cmake configure + build inside the devel container, mounting source and ccache
+  # Use --userns=keep-id so files are owned by host user, not root
+  podman run --rm \
+    -v "$PWD:/work" -w /work \
+    -v "$HOME/.cache/hydra-ccache:/tmp/ccache:rw" \
+    -e CCACHE_DIR=/tmp/ccache \
+    -e CCACHE_MAXSIZE=10G \
+    -e CCACHE_SLOPPINESS="pch_defines,time_macros,locale" \
+    --userns=keep-id \
+    nvidia/cuda:12.9.2-devel-ubuntu24.04 \
+    bash -c '
+      set -e
+      echo "=== [container] apt-get install build deps ==="
+      apt-get update -qq
+      apt-get install -y -qq cmake ninja-build ccache g++-14 libssl-dev libgomp1 > /dev/null
+      echo "=== [container] cmake configure ==="
+      cmake -B "'"$BUILD_DIR"'" -G Ninja         -DCMAKE_CUDA_ARCHITECTURES="'"$CUDA_ARCH"'"         -DCMAKE_C_COMPILER_LAUNCHER=ccache         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache         -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache         -DGGML_CUDA=ON -DGGML_CUDA_FORCE_CUBLAS=ON -DGGML_RPC=ON -DGGML_NVML=ON         -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON         -DCMAKE_BUILD_RPATH='\''$ORIGIN'\'' -DCMAKE_INSTALL_RPATH='\''$ORIGIN'\'' -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON         -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF         -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc         -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/g++-14         -DGGML_CUDA_FA_ALL_QUANTS=OFF -DGGML_NATIVE=OFF         .
+      echo "=== [container] cmake build ==="
+      cmake --build "'"$BUILD_DIR"'" --target "'"$BINARY"'" -j$(nproc)
+    '
+else
+  cmake -B "$BUILD_DIR" -G Ninja "${CMAKE_ARGS[@]}" .
+  echo "=== [$ARCH/$BINARY] CMake build ==="
+  cmake --build "$BUILD_DIR" --target "$BINARY" -j"$(nproc)"
+fi
 
 ccache -s || true
 
@@ -104,8 +137,27 @@ fi
 echo "=== [$ARCH/$BINARY] ${IMAGE_TAG} not in registry — building ==="
 
 mkdir -p "${STAGING_DIR}/bin"
-cp "$BUILD_DIR/bin/$BINARY" "${STAGING_DIR}/bin/"
-cp "$BUILD_DIR/bin/"*.so* "${STAGING_DIR}/bin/" 2>/dev/null || true
+cp -a "$BUILD_DIR/bin/$BINARY" "${STAGING_DIR}/bin/"
+# Fix #498: preserve symlinks (-a) and ensure ALL shared libs are staged.
+# The previous `cp *.so*` without -a could dereference symlinks and miss
+# versioned chains (e.g., libllama.so -> libllama.so.0 -> libllama.so.0.x).
+# Use -a and explicitly list contents for verification.
+cp -a "$BUILD_DIR/bin/"*.so* "${STAGING_DIR}/bin/" 2>/dev/null || true
+echo "=== Staging contents (${STAGING_DIR}/bin/) ==="
+ls -lh "${STAGING_DIR}/bin/" | head -n 50
+echo "=== Host ldd check for $BUILD_DIR/bin/$BINARY (expect CUDA libs 'not found' on host, but hydra .so should resolve via \$ORIGIN) ==="
+ldd "$BUILD_DIR/bin/$BINARY" || true
+if ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libcudart|libcublas|libibverbs" | grep -q .; then
+  echo "NOTE: some non-CUDA libs still 'not found' or GLIBC mismatch — check \$ORIGIN RPATH and builder glibc (sm60 must be built in nvidia/cuda:12.9.2-devel-ubuntu24.04)"
+  ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libcudart|libcublas|libibverbs" || true
+fi
+# Fail hard if any hydra .so is missing in staging (e.g., libllama-server-impl.so for llama-server)
+if [ "$BINARY" = "llama-server" ]; then
+  if [ ! -f "${STAGING_DIR}/bin/libllama-server-impl.so"* ] && ! ls "${STAGING_DIR}/bin/libllama-server-impl.so"* >/dev/null 2>&1; then
+    echo "WARNING: llama-server build should have libllama-server-impl.so in staging — check BUILD_SHARED_LIBS=ON"
+    ls -lh "${STAGING_DIR}/bin/"*.so* 2>&1 | head -n 20 || true
+  fi
+fi
 
 echo "=== [$ARCH/$BINARY] Build + push OCI image ==="
 # Docker Hub's nvidia/cuda runtime images are tagged with a full patch