diff --git a/os/mini/release-check.sh b/os/mini/release-check.sh index 8434e7234..fb04a5121 100755 --- a/os/mini/release-check.sh +++ b/os/mini/release-check.sh @@ -788,8 +788,13 @@ payload_modes_noexec() { # -> the binaries present but not execu # `[ -x ]` is the right predicate and NOT a stricter "is it 0755" (engine#2424's phrasing, worth keeping): the appliance runs as # root, and access(2) documents that a real UID of zero passes X_OK "if execute permission is enabled for any of the file owner, # group, or other". So the property is *can it execute*, which is exactly what the row that would start the engine tests. + # The HRX backend is the fourth binary and the one that actually failed first (2026-11-15): a payload whose + # `llama-server` was 0644 answered the first chat request with "Failed to execute: ... Permission denied" and + # HTTP 500 in one second, while this row - which had just been fixed to require the other three - was silent + # about it. The three are the ones the appliance starts by name; this one is the one `serve` spawns for a model. _pe_dir=${1:-} _pe_out= - for _pe_b in "$_pe_dir/1bit/1bit" "$_pe_dir/bin/lemond" "$_pe_dir/lemond-onebit"; do + for _pe_b in "$_pe_dir/1bit/1bit" "$_pe_dir/bin/lemond" "$_pe_dir/lemond-onebit" \ + "$_pe_dir/1bit/hrx/llama/bin/llama-server"; do [ -f "$_pe_b" ] || continue { [ -x "$_pe_b" ] || _pe_out="$_pe_out ${_pe_b#"$_pe_dir"/}"; } 2>/dev/null || true done @@ -2248,6 +2253,46 @@ else fi fi +payload_backend_missing() { # -> the HRX backend pieces absent from it, payload-relative and space-separated + # Measured 2026-11-15 in an `npu-hold`+`gpu-hold` demo rehearsal: a partial payload dir launched `lemond-onebit`, + # listed all three demo models, and then answered `/api/v1/chat/completions` with HTTP 500 in 1 s: + # "Failed to execute: /1bit/hrx/llama/bin/llama-server" + # `payload-modes` catches that particular payload on its entry floor (94 entries), but the floor is a count: a + # payload with hundreds of entries can still be missing the one binary every HRX model spawns, and the first + # request is where that surfaces - for a user, not for a checker. + # Two pieces, because `llama-server` present and its HRX compute library absent is the same failure one step later + # (the exec succeeds and the `dlopen` of `libggml-hrx.so` does not). + # + # Absence of the CATALOG is not this function's question and neither is a payload that ships no HRX model: the + # caller skips both. This returns only "which pieces are missing from a payload that should have them". + _pb_dir=${1:-} _pb_out= + [ -f "$_pb_dir/1bit/hrx/llama/bin/llama-server" ] || _pb_out="$_pb_out 1bit/hrx/llama/bin/llama-server" + if ! ls "$_pb_dir"/1bit/hrx/llama/bin/libggml-hrx.so* > /dev/null 2>&1; then + _pb_out="$_pb_out 1bit/hrx/llama/bin/libggml-hrx.so*" + fi + printf '%s' "$_pb_out" +} + +# ── payload-backend: the executable `serve` spawns for an HRX model ───────────────────────────────────────────── +# The presence question comes from the payload's OWN catalog, so a payload that ships no HRX model is not asked for an +# HRX backend, and a payload with no catalog at all is a different row's FAIL (payload-modes' floor) rather than a +# claim invented here. +_cat=$PDIR/bin/resources/server_models.json +if [ ! -d "$PDIR" ]; then + skip payload-backend "no $PDIR on this system: the payload is not unpacked where an appliance keeps it" +elif [ ! -f "$_cat" ]; then + skip payload-backend "the payload has no catalog at $_cat, so what it will spawn cannot be read" +elif ! grep -qE '"recipe"[[:space:]]*:[[:space:]]*"(onebit|llamacpp-hrx)"' "$_cat" 2> /dev/null; then + skip payload-backend "the payload's catalog ships no HRX model, so it needs no HRX backend" +else + _pb_miss=$(payload_backend_missing "$PDIR") + if [ -n "$_pb_miss" ]; then + fail payload-backend "the payload's catalog ships HRX models but the backend is incomplete:${_pb_miss} - the first chat request fails with \"Failed to execute\" (HTTP 500), which is what a partial extraction looks like" + else + pass payload-backend "llama-server and libggml-hrx are both in the payload beside the catalog that needs them" + fi +fi + # ── the BASE layer's modes (engine#1474) ──────────────────────────────────────────────────────────────────────── # The scan is the root filesystem only, and the keep-list is the ONE source of truth for what may stay writable: the prune # list is BUILT from it rather than copied beside it (the first version hardcoded the same eleven entries by hand, so the diff --git a/tests/release_check_payload_modes_test.sh b/tests/release_check_payload_modes_test.sh index b019e3e23..00cca3042 100755 --- a/tests/release_check_payload_modes_test.sh +++ b/tests/release_check_payload_modes_test.sh @@ -47,6 +47,12 @@ case "$RC" in *"this line is never matched"*) : ;; esac grep -q 'payload_modes_noexec "\$PDIR"' "$RC" || { echo "FAIL payload_modes_noexec() is defined but never called on the payload - a guard nothing calls"; exit 1; } +# The second half of the same defect: the `payload-backend` row owns ABSENCE (the mode function owns the mode), so it needs +# the same two things - the decision is a function, and the row actually calls it. A row that exists to catch a missing +# backend and has no arm of its own is the defect this PR is about, one level up. +FN_BK=$(awk '/^ *payload_backend_missing\(\) \{/{f=1} f{print} f&&/^ *\}$/{exit}' "$RC") +[ -n "$FN_BK" ] || { echo "FAIL no payload_backend_missing() in $RC - the row cannot require a backend"; exit 1; } +grep -q 'payload_backend_missing "\$PDIR"' "$RC" || { echo "FAIL payload_backend_missing() is defined but never called on the payload"; exit 1; } ROOT=$(mktemp -d "${TMPDIR:-$HOME/.cache}/release-check-pm.XXXXXX") || exit 1 trap 'rm -rf "$ROOT"' EXIT @@ -71,6 +77,44 @@ mktree p755 0755 got=$(bash -c "$FN"$'\npayload_modes_noexec "$1"' _ "$ROOT/p755") [ -z "$got" ] && echo "ok a 0755 payload reports nothing" || { echo "FAIL a healthy payload was reported: '${got}'"; fail=1; } +# The fourth binary: `serve` spawns the HRX backend for every onebit/llamacpp-hrx model, and a payload whose +# `llama-server` was 0644 answered the first chat request with "Failed to execute: ... Permission denied" (measured +# 2026-11-15) while this row - which required the other three - said nothing about it. +mkdir -p "$ROOT/pbackend/1bit/hrx/llama/bin" "$ROOT/pbackend/bin" +printf 'x' > "$ROOT/pbackend/1bit/1bit"; printf 'x' > "$ROOT/pbackend/bin/lemond" +chmod 0755 "$ROOT/pbackend/1bit/1bit" "$ROOT/pbackend/bin/lemond" +printf 'x' > "$ROOT/pbackend/1bit/hrx/llama/bin/llama-server"; chmod 0644 "$ROOT/pbackend/1bit/hrx/llama/bin/llama-server" +got=$(bash -c "$FN"$'\npayload_modes_noexec "$1"' _ "$ROOT/pbackend") +case "$got" in + *hrx/llama/bin/llama-server*) echo "ok a 0644 HRX backend is named even when the three named binaries are 0755" ;; + *) echo "FAIL a 0644 HRX backend went unreported: '${got}'"; fail=1 ;; +esac + +# payload-backend's own decision, three ways: a complete backend reports nothing; an absent one names both pieces +# (the executable and the compute library, which fail at exec and at dlopen respectively); and a backend that is +# present but not executable is NOT this function's claim - the mode function above owns that. +mkdir -p "$ROOT/pbk/1bit/hrx/llama/bin" +: > "$ROOT/pbk/1bit/hrx/llama/bin/llama-server"; : > "$ROOT/pbk/1bit/hrx/llama/bin/libggml-hrx.so.0" +got=$(bash -c "$FN_BK"$'\npayload_backend_missing "$1"' _ "$ROOT/pbk") +[ -z "$got" ] && echo "ok a complete HRX backend reports nothing missing" || { echo "FAIL a complete backend was reported: '${got}'"; fail=1; } + +mkdir -p "$ROOT/pbn/1bit" +got=$(bash -c "$FN_BK"$'\npayload_backend_missing "$1"' _ "$ROOT/pbn") +case "$got" in + *hrx/llama/bin/llama-server*libggml-hrx*) echo "ok a payload with no backend names the executable and the library" ;; + *) echo "FAIL an absent backend was under-reported: '${got}'"; fail=1 ;; +esac + +mkdir -p "$ROOT/pbl/1bit/hrx/llama/bin"; : > "$ROOT/pbl/1bit/hrx/llama/bin/libggml-hrx.so.0" +got=$(bash -c "$FN_BK"$'\npayload_backend_missing "$1"' _ "$ROOT/pbl") +case "$got" in + *llama-server*) case "$got" in + *libggml-hrx*) echo "FAIL a present library was reported missing: '${got}'"; fail=1 ;; + *) echo "ok an executable-only gap names the executable and not the library it does have" ;; + esac ;; + *) echo "FAIL the missing executable was not named: '${got}'"; fail=1 ;; +esac + mktree pnone none got=$(bash -c "$FN"$'\npayload_modes_noexec "$1"' _ "$ROOT/pnone") [ -z "$got" ] && echo "ok a payload with no binaries reports nothing (absence is another row's claim)" || { echo "FAIL a binary-less payload was reported: '${got}'"; fail=1; }