From 94d444ed38af4358de42f5fcf9aaa2ccf990d487 Mon Sep 17 00:00:00 2001 From: pi-release Date: Sat, 7 Nov 2026 22:46:20 -0400 Subject: [PATCH] release-check: the payload's RUNPATH must be $ORIGIN only (the 2026-11-08 incident) An installed engine at ~/.local/share/1bit-hrx-a5271d9 carried RUNPATH into ~/wt/release-rc/work/build, so a rebuild there swapped the libraries under two RUNNING services: coder-llm on :8090 crash-looped 154 times from 19:51 and 1bit-canonical on :8095 755 times from 18:26. Measured, out of that install: libllama.so.0 RUNPATH /home/bcloud/wt/release-rc/work/build/hrx/llama/bin:/home/bcloud/wt/release-rc/work/build/hrx/llama/ ggml/src/ggml-hrx/.../libhrx:/home/bcloud/wt/release-rc/work/hrx-toolchain/.../clang/24/lib/x86_64-... (the same shape in libggml.so.0, libggml-cpu.so.0, libmtmd.so and the rest) Rule from the orchestrator, and now a row: an installed package must carry `$ORIGIN` only - it finds its libraries beside itself, and a path into a build tree is a live wire to whatever the next build leaves behind. The appliance ships no readelf/objdump (checked in c32's sfs: no readelf, no objdump, no eu-readelf; python3.14 IS there), so the row reads the dynamic section itself with python3, with readelf as the host fallback, and SKIPs rather than guessing when neither exists. DT_RPATH is a FAIL on its own even when its value is `$ORIGIN`: it is deprecated and cannot be overridden by LD_LIBRARY_PATH, which is precisely the property that makes a stale path stick. Answering the question this was raised with: the c32 payload is CLEAN - all sixteen of its RUNPATH-carrying ELFs are exactly `$ORIGIN` (libhrx, libllama*, libggml*, libloomc, libmtmd, libomp, llama-server/-bench/-perplexity). Measured both with readelf and with the row's own python scanner. Proved on real ELF dynamic sections, not mocks (the fixtures are built by gcc in the harness): $ORIGIN only PASSes; an absolute build-tree path FAILs naming the file and the path; legacy DT_RPATH FAILs as RPATH; no payload directory SKIPs. 10/10, plus --self-test cases for the pure verdict. --- os/mini/release-check.sh | 111 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/os/mini/release-check.sh b/os/mini/release-check.sh index 1a5153c95..14a27ae24 100755 --- a/os/mini/release-check.sh +++ b/os/mini/release-check.sh @@ -581,6 +581,23 @@ payload_modes_verdict() { fi } +# The payload's RUNPATH (the orchestrator's rule of 2026-11-08, after an installed engine's RUNPATH into a build tree made a +# rebuild swap the libraries under two running services: coder-llm on :8090 crash-looped 154 times and 1bit-canonical on :8095 +# 755 times). An installed package must carry `$ORIGIN` ONLY: it finds its own libraries beside itself, and a path into a build +# tree is a live wire to whatever the next build there leaves behind. This is the shipped half of that rule - the appliance has no +# readelf/objdump, so the row reads DT_RUNPATH/DT_RPATH out of the dynamic section itself (python3, which the image does ship) and +# falls back to readelf on a host. Pure, so --self-test drives it: [] +payload_runpath_verdict() { + _n=${1:-} _sample=${2:-} + case "$_n" in ''|*[!0-9]*) printf 'FAIL the payload'"'"'s RUNPATHs could not be read (%s)' "${_n:-empty}"; return 0 ;; esac + if [ "$_n" = 0 ]; then + printf 'PASS every RUNPATH in the payload is $ORIGIN: the package finds its libraries beside itself and points at no build tree' + else + printf 'FAIL %s payload ELF(s) carry a RUNPATH or RPATH that is not $ORIGIN%s - an installed package must not point into a build tree (a rebuild there swaps the libraries under a running service)' \ + "$_n" "${_sample:+: $_sample}" + fi +} + # --self-test: assert the lemonade verdict on the cases that matter (tests/release_check_row.sh runs # this in CI). The unprepared-stick case is the one that used to false-PASS: 200 with the key, an empty # body and no status without it. @@ -620,6 +637,11 @@ if [ "${1:-}" = --self-test ]; then check FAIL "$(followup_verdict '{"error":{"message":"no answer"}}' '{"choices":[{"message":{"content":"1BIT OK"},"finish_reason":"length"}]}' 0 0)" check FAIL "$(followup_verdict '{"choices":[{"finish_reason":"length"}]}' '{"choices":[{"message":{"content":"1BIT"}}]}' 0 0)" check PASS "$(followup_verdict '{"choices":[{"finish_reason":"length"}]}' '{"choices":[{"message":{"content":"1BIT OK"},"finish_reason":"length"}]}' '' '')" + # the RUNPATH rule (2026-11-08 incident): $ORIGIN only, and RPATH is always a fail. + check PASS "$(payload_runpath_verdict 0)" + check FAIL "$(payload_runpath_verdict 3 '/opt/1bit-package/1bit/1bit runs /home/x/wt/release-rc/work/build')" + check FAIL "$(payload_runpath_verdict 1)" + check FAIL "$(payload_runpath_verdict '')" # engine #1352: the payload's modes. c31/c32 shipped 134 writable files; a payload packed after #1361 has none. check PASS "$(payload_modes_verdict 0)" check FAIL "$(payload_modes_verdict 134)" @@ -1694,6 +1716,95 @@ else esac fi +# ── the payload's RUNPATH (orchestrator's rule, 2026-11-08) ───────────────────────────────────────────────────── +# An installed package must carry `$ORIGIN` ONLY. The incident: an installed engine whose RUNPATH pointed into +# ~/wt/release-rc/work/build had its libraries swapped by a rebuild there, and two running services crash-looped +# (coder-llm on :8090 154x, 1bit-canonical on :8095 755x). The appliance ships no readelf/objdump, so the dynamic section is read +# directly: python3 is in the image, with readelf as the host fallback. DT_RPATH is a fail on its own - it is deprecated and cannot +# be overridden by LD_LIBRARY_PATH, which is exactly the property that makes a stale path stick. +payload_runpath_bad() { # -> "relpath|TAG|value" per offending entry + if command -v python3 >/dev/null 2>&1; then + python3 - "$1" <<'PYSCAN' 2>/dev/null +import os, struct, sys +root = sys.argv[1] +def dyn(path): + try: + with open(path, 'rb') as f: + d = f.read() + except OSError: + return None + if len(d) < 64 or d[:4] != b'\x7fELF': + return None + is64 = d[4] == 2 + try: + if is64: + shoff, = struct.unpack_from('= len(secs): + continue + stroff = secs[link][1] + step = 16 if is64 else 8 + fmt = '/dev/null 2>&1; then + find "$1" -type f -perm /111 2>/dev/null | while IFS= read -r _f; do + readelf -d "$_f" 2>/dev/null | sed -n "s|.*(\(RUNPATH\|RPATH\)).*\[\(.*\)\]|${_f#$1/}|\1|\2|p" + done | awk -F'|' '($2 == "RPATH") || ($3 != "$ORIGIN" && $3 != "$ORIGIN/")' + else + return 2 + fi +} +RUNDIR=${PAYLOAD_DIR:-/opt/1bit-package} +if [ ! -d "$RUNDIR" ]; then + skip payload-runpath "no $RUNDIR on this system: the payload is not unpacked where an appliance keeps it, so its RUNPATHs cannot be read" +else + _rp_bad=$(payload_runpath_bad "$RUNDIR"); _rp_rc=$? + if [ "$_rp_rc" = 2 ]; then + skip payload-runpath "neither python3 nor readelf is here, so the payload's RUNPATHs cannot be read - that is not evidence" + else + _rp_n=$(printf '%s\n' "${_rp_bad:-}" | sed '/^$/d' | wc -l) + _rp_s=$(printf '%s\n' "${_rp_bad:-}" | sed '/^$/d' | head -3 | tr '\n' ' ') + RP=$(payload_runpath_verdict "${_rp_n:-0}" "${_rp_s:-}") + case "$RP" in + PASS*) pass payload-runpath "${RP#PASS }" ;; + *) fail payload-runpath "${RP#FAIL }" ;; + esac + fi +fi + say "" # The back-off leaves a trace: a run that only passed because the partition freed in time must say so, or the next # person reads a clean transcript and believes nothing was ever refused.