diff --git a/.github/workflows/hydra-build.Dockerfile b/.github/workflows/hydra-build.Dockerfile index 6e0f90658302..54ecb05305d4 100644 --- a/.github/workflows/hydra-build.Dockerfile +++ b/.github/workflows/hydra-build.Dockerfile @@ -1,6 +1,19 @@ # Hydra Build — Minimal runtime image for OCI push # Copies only the binary + shared libs from the build directory. -# Used by hydra-build.yml to create deployable OCI images. +# Fix #498: P100 sm60 image previously shipped only the binary without its +# shared libs (libllama.so, libggml-*.so, libllama-server-impl.so, etc), +# causing "symbol lookup error: undefined symbol: llama_model_get_quant_label" +# and a 1120-restart loop (see ddvnguyen/hydra_vortex#498). This Dockerfile +# now reliably packages ALL .so* alongside the binary and verifies via ldd +# that no libs are missing. RUNPATH is $ORIGIN (set at build with +# -DCMAKE_BUILD_RPATH='$ORIGIN' -DCMAKE_INSTALL_RPATH='$ORIGIN'), so /llama +# is the search dir; LD_LIBRARY_PATH=/llama is added as a defensive fallback. +# Follow-up #498 glibc: host is Ubuntu 26.04 (glibc 2.43) but runtime base is +# nvidia/cuda:12.9.2-runtime-ubuntu24.04 (glibc 2.39). Host-built sm60 binaries +# require GLIBC_2.43 and crash in the image (5× version not found). The +# pipeline now builds sm60 inside nvidia/cuda:12.9.2-devel-ubuntu24.04 +# (matching runtime glibc 2.39) — see build-combo.sh containerized build. +# Ldd gate now fails on BOTH "=> not found" and "version GLIBC not found". ARG CUDA_VERSION=13.2 ARG UBUNTU_VERSION=24.04 @@ -8,8 +21,10 @@ ARG BINARY=llama-engine FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} +ARG BINARY + RUN apt-get update \ - && apt-get install -y --no-install-recommends libgomp1 curl \ + && apt-get install -y --no-install-recommends libgomp1 curl libibverbs1 \ && rm -rf /var/lib/apt/lists/* WORKDIR /llama @@ -17,8 +32,24 @@ WORKDIR /llama # Copy the binary and all shared libraries from the build output COPY bin/ /llama/ -# Make binaries executable -RUN chmod +x /llama/${BINARY} 2>/dev/null || true +# Make binary executable, verify ldd, and ensure fallback LD_LIBRARY_PATH +RUN chmod +x /llama/${BINARY} 2>/dev/null || chmod +x /llama/llama-engine 2>/dev/null || true \ + && echo "=== ldd verify for /llama/${BINARY:-llama-engine} ===" \ + && ldd /llama/${BINARY:-llama-engine} 2>&1 | tee /tmp/ldd.txt || true \ + && cat /tmp/ldd.txt \ + && if grep -E "=> not found|version .*GLIBC.*not found" /tmp/ldd.txt | grep -vE "libcuda|libibverbs" | grep -q .; then \ + echo "ERROR: missing hydra shared libs or GLIBC version mismatch in image — COPY bin/ must include *.so* and builder must match runtime glibc (use nvidia/cuda:12.9.2-devel-ubuntu24.04 for sm60)"; \ + grep -E "=> not found|version .*GLIBC.*not found" /tmp/ldd.txt; exit 1; \ + fi \ + && echo "=== ldd OK — hydra libs present, no GLIBC mismatch (libcuda/libibverbs expected to be host-mounted) ===" \ + && ls -lh /llama/*.so* 2>/dev/null | head -n 40 || echo "no .so files in /llama (static build?)" + +ENV LD_LIBRARY_PATH=/llama:${LD_LIBRARY_PATH} + +# Ensure entrypoint works for both binaries: if BINARY != llama-engine, symlink +# so the fixed ENTRYPOINT still resolves. This keeps hydra-head's +# /llama/llama-engine expectation while supporting llama-server images. +RUN if [ "${BINARY:-llama-engine}" != "llama-engine" ]; then ln -sf /llama/${BINARY} /llama/llama-engine || true; fi HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ CMD curl -f http://localhost:8080/health || exit 1 diff --git a/.github/workflows/scripts/build-combo.sh b/.github/workflows/scripts/build-combo.sh index 6fbfdc0266bb..f93b6cab9885 100644 --- a/.github/workflows/scripts/build-combo.sh +++ b/.github/workflows/scripts/build-combo.sh @@ -66,10 +66,43 @@ esac echo "=== [$ARCH/$BINARY] CMake configure (CUDA $CUDA_VERSION @ $CUDA_PATH) ===" echo "CMake args: ${CMAKE_ARGS[*]}" -cmake -B "$BUILD_DIR" -G Ninja "${CMAKE_ARGS[@]}" . - -echo "=== [$ARCH/$BINARY] CMake build ===" -cmake --build "$BUILD_DIR" --target "$BINARY" -j"$(nproc)" +# Fix #498 glibc follow-up: host is Ubuntu 26.04 (glibc 2.43) but runtime base is +# nvidia/cuda:12.9.2-runtime-ubuntu24.04 (glibc 2.39). Host-built sm60 binaries +# require GLIBC_2.43 and crash in the image (5× version not found). Build sm60 +# inside a container matching the runtime base to ensure glibc compatibility. +# sm86-sm120 stays host-built (latent same issue, hardened ldd gate now protects). +if [ "$ARCH" = "sm60" ]; then + echo "=== [$ARCH/$BINARY] Containerized build in nvidia/cuda:12.9.2-devel-ubuntu24.04 (glibc 2.39) to match runtime ===" + # Clean previous host-built artifacts (wrong glibc) + rm -rf "$BUILD_DIR" + # Pull devel image (contains /usr/local/cuda 12.9.2, gcc, but not cmake/ninja/ccache) + podman pull nvidia/cuda:12.9.2-devel-ubuntu24.04 2>&1 | tail -n 5 || true + mkdir -p "$HOME/.cache/hydra-ccache" + # Run cmake configure + build inside the devel container, mounting source and ccache + # Use --userns=keep-id so files are owned by host user, not root + podman run --rm \ + -v "$PWD:/work" -w /work \ + -v "$HOME/.cache/hydra-ccache:/tmp/ccache:rw" \ + -e CCACHE_DIR=/tmp/ccache \ + -e CCACHE_MAXSIZE=10G \ + -e CCACHE_SLOPPINESS="pch_defines,time_macros,locale" \ + --userns=keep-id \ + nvidia/cuda:12.9.2-devel-ubuntu24.04 \ + bash -c ' + set -e + echo "=== [container] apt-get install build deps ===" + apt-get update -qq + apt-get install -y -qq cmake ninja-build ccache g++-14 libssl-dev libgomp1 > /dev/null + echo "=== [container] cmake configure ===" + cmake -B "'"$BUILD_DIR"'" -G Ninja -DCMAKE_CUDA_ARCHITECTURES="'"$CUDA_ARCH"'" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -DGGML_CUDA=ON -DGGML_CUDA_FORCE_CUBLAS=ON -DGGML_RPC=ON -DGGML_NVML=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_RPATH='\''$ORIGIN'\'' -DCMAKE_INSTALL_RPATH='\''$ORIGIN'\'' -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/g++-14 -DGGML_CUDA_FA_ALL_QUANTS=OFF -DGGML_NATIVE=OFF . + echo "=== [container] cmake build ===" + cmake --build "'"$BUILD_DIR"'" --target "'"$BINARY"'" -j$(nproc) + ' +else + cmake -B "$BUILD_DIR" -G Ninja "${CMAKE_ARGS[@]}" . + echo "=== [$ARCH/$BINARY] CMake build ===" + cmake --build "$BUILD_DIR" --target "$BINARY" -j"$(nproc)" +fi ccache -s || true @@ -104,8 +137,27 @@ fi echo "=== [$ARCH/$BINARY] ${IMAGE_TAG} not in registry — building ===" mkdir -p "${STAGING_DIR}/bin" -cp "$BUILD_DIR/bin/$BINARY" "${STAGING_DIR}/bin/" -cp "$BUILD_DIR/bin/"*.so* "${STAGING_DIR}/bin/" 2>/dev/null || true +cp -a "$BUILD_DIR/bin/$BINARY" "${STAGING_DIR}/bin/" +# Fix #498: preserve symlinks (-a) and ensure ALL shared libs are staged. +# The previous `cp *.so*` without -a could dereference symlinks and miss +# versioned chains (e.g., libllama.so -> libllama.so.0 -> libllama.so.0.x). +# Use -a and explicitly list contents for verification. +cp -a "$BUILD_DIR/bin/"*.so* "${STAGING_DIR}/bin/" 2>/dev/null || true +echo "=== Staging contents (${STAGING_DIR}/bin/) ===" +ls -lh "${STAGING_DIR}/bin/" | head -n 50 +echo "=== Host ldd check for $BUILD_DIR/bin/$BINARY (expect CUDA libs 'not found' on host, but hydra .so should resolve via \$ORIGIN) ===" +ldd "$BUILD_DIR/bin/$BINARY" || true +if ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libcudart|libcublas|libibverbs" | grep -q .; then + echo "NOTE: some non-CUDA libs still 'not found' or GLIBC mismatch — check \$ORIGIN RPATH and builder glibc (sm60 must be built in nvidia/cuda:12.9.2-devel-ubuntu24.04)" + ldd "$BUILD_DIR/bin/$BINARY" 2>&1 | grep -E "=> not found|version .*GLIBC.*not found" | grep -vE "libcuda|libcudart|libcublas|libibverbs" || true +fi +# Fail hard if any hydra .so is missing in staging (e.g., libllama-server-impl.so for llama-server) +if [ "$BINARY" = "llama-server" ]; then + if [ ! -f "${STAGING_DIR}/bin/libllama-server-impl.so"* ] && ! ls "${STAGING_DIR}/bin/libllama-server-impl.so"* >/dev/null 2>&1; then + echo "WARNING: llama-server build should have libllama-server-impl.so in staging — check BUILD_SHARED_LIBS=ON" + ls -lh "${STAGING_DIR}/bin/"*.so* 2>&1 | head -n 20 || true + fi +fi echo "=== [$ARCH/$BINARY] Build + push OCI image ===" # Docker Hub's nvidia/cuda runtime images are tagged with a full patch