Skip to content
19 changes: 15 additions & 4 deletions os/mini/init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,21 @@ fi

# The sealed NPU kernels live on the stick (they used to be inside the RAM image, which made it too big
# to load): link them to the path the engine looks for, beside itself.
if [ -d /data/npu/kernels ] && [ -d /home ]; then
mkdir -p /home/bcloud/wt/npupkg-build/npu 2>/dev/null
ln -sfn /data/npu/kernels /home/bcloud/wt/npupkg-build/npu/kernels 2>/dev/null
say "[kernels] $(ls /data/npu/kernels | wc -l) sealed NPU sets linked beside the engine"
# engine #1543: the link used to name the BUILDER's home directory - a username and a build path compiled
# into this script, and re-created as a directory on every boot. The engine is installed at /opt/1bit with its
# backends beside it (mkimage.sh), so the sets are linked beside the engine there and no /home path exists.
if [ -d /data/npu/kernels ] && [ -d /opt/1bit ]; then
# the parent has to exist before the link, or ln fails and the message below lies about it (engine #1543
# review item 1): the engine looks for <its own directory>/npu/kernels, and its directory is the real one
# /proc/self/exe resolves to - /opt/1bit - not /bin, which is only the symlink on PATH
mkdir -p /opt/1bit/npu
if ln -sfn /data/npu/kernels /opt/1bit/npu/kernels 2>/dev/null && [ -d /opt/1bit/npu/kernels ]; then
say "[kernels] $(ls /data/npu/kernels | wc -l) sealed NPU sets linked beside the engine"
else
say "[kernels] FAILED to link /data/npu/kernels beside the engine: the NPU route will register no device"
fi
elif [ -d /data/npu/kernels ]; then
say "[kernels] /data/npu/kernels is there but the engine is not at /opt/1bit: nothing to link beside"
fi

n=0
Expand Down
85 changes: 79 additions & 6 deletions os/mini/mkimage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -316,12 +316,66 @@ fw "amdgpu/gc_11_5_1_*" "amdgpu/sdma_6_1_1*" "amdgpu/psp_14_0_1*" "amdgpu/smu_14
echo "== the engine"
bin=$build/onebit; [ -x "$bin" ] || bin=$build/1bit
[ -x "$bin" ] || { echo "no engine binary in $build (build with -DONEBIT_NPU=ON -DONEBIT_HRX=ON)"; exit 1; }
copy_elf "$bin"
ln -s "$bin" "$root/bin/1bit"
# The sealed NPU kernel sets travel BESIDE the engine binary (docs/releases.md, "The NPU engine"):
# in a package that is <pkg>/npu/kernels next to <pkg>/1bit, and here the binary keeps its build
# path, so the sets go to the same place relative to it. Without them the image ships the NPU
# route and nothing it can load - the route then refuses rather than falling back.

# engine #1543: the engine used to be installed AT ITS BUILD PATH, with /bin/1bit a symlink into the
# builder's home and the whole /home/<user>/... tree carried in the image (a 4.7 MB binary included), so
# every mini image disclosed the builder's username, work directory and build-directory name. That is a
# fourth leak class - a path delivered as the filesystem LAYOUT, not as a string inside a file - and the
# content rows (scripts/pack-check.sh: build_path_strings) cannot see it.
#
# The engine finds its backends BESIDE ITSELF when the build tree is gone (app/built_path.h: the compiled
# path made relative to ONEBIT_BUILD_DIR, resolved against /proc/self/exe's directory), which is the layout
# scripts/package-linux.sh ships. So it is installed that way here: /opt/1bit (the path 1bit-release-check
# already probes) carrying the engine, its libraries and every backend beside it, $ORIGIN only, and
# /bin/1bit a symlink to that image path - never to a build path.
command -v patchelf >/dev/null || { echo "patchelf is needed to install the engine at \$ORIGIN instead of at its build path: apt install patchelf"; exit 1; }
# shellcheck source=scripts/pack-elf.sh
. "$here/../../scripts/pack-elf.sh" # pack_is_elf, used below and by the Lemonade section
eng=$root/opt/1bit
install_origin() { # install_origin <source elf> <destination dir>: the file, $ORIGIN, and its libs beside
local src=$1 dir=$2 f missing
[ -e "$src" ] || { echo "missing: $src"; exit 1; }
mkdir -p "$dir"; cp -L "$src" "$dir/"
patchelf --set-rpath '$ORIGIN' "$dir/$(basename "$src")"
# ONLY what the image does not already carry. The image has libc, libstdc++, libgcc_s and the loader;
# copying those beside the engine and then pointing RUNPATH at $ORIGIN makes the copy SHADOW the image's
# own libc for every ELF in that directory - a set that works on the builder and fails at boot. A library
# on the standard loader path is the loader's business, on the image as on any machine.
for f in $(ldd "$src" 2>/dev/null | awk '/=> \// {print $3}'); do
case "$f" in /lib/*|/lib64/*|/usr/lib/*|/usr/lib64/*) continue ;; esac
[ -e "$dir/$(basename "$f")" ] || cp -L "$f" "$dir/"
done
# EVERY copy, not just the program: a library from the build tree carries the build tree in its own
# RUNPATH, so patching only the program leaves libggml/libhrx/libloomc looking for each other in the
# builder's tree at boot. They are all in $dir now, so $ORIGIN is the whole answer for each of them.
for f in "$dir"/*; do
pack_is_elf "$f" && patchelf --set-rpath '$ORIGIN' "$f"
done
missing=$(LD_LIBRARY_PATH=$dir ldd "$dir/$(basename "$src")" 2>/dev/null | awk '/not found/ {print $1}' | tr '\n' ' ')
[ -z "${missing// /}" ] || { echo "$(basename "$src") still needs $missing beside it after relocation (engine #1543)"; exit 1; }
}
install_origin "$bin" "$eng"
# install_origin refuses to put a SYSTEM library beside the engine ($ORIGIN would make it shadow the image's
# own), but the image still has to carry them: copy_elf used to stage the engine's whole NEEDED set at its
# system paths through copy_libs, and replacing it dropped that. libgomp and libpcre2-8 in particular have no
# other ELF in the image pulling them in (engine #1543 review, fix 3's hole).
copy_libs "$bin"
ln -sfn ../opt/1bit/1bit "$root/bin/1bit"
# NO backend goes into the initramfs. init.sh points ONEBIT_{,HRX_}LLAMA_SERVER at /opt/1bit-llama, which it
# links from the stick's data partition at boot, and default_llama_server() (app/serve.cpp:292) returns those
# first - so an HRX llama-server and its ~40 MB runtime set inside the image were never executed, while the
# initramfs is already 97.9 MB compressed against the 103 MB that grub failed to load on real hardware. One
# copy, and it is the one on the stick (mkimage.sh puts it there below, engine #1543's review item 2).
for back in vulkan hrx; do
if [ -e "$build/$back/llama/bin/llama-server" ]; then
echo " $back backend: left to the data partition ($(du -sm "$build/$back/llama/bin" 2>/dev/null | cut -f1) MB not in the initramfs)"
fi
done
# The sealed NPU kernel sets travel BESIDE the engine binary (docs/releases.md, "The NPU engine"): in a
# package that is <pkg>/npu/kernels next to <pkg>/1bit. Here they live on the stick's data partition (they
# are ~124 MB and grub would not load an initrd that size) and init.sh links them to /opt/1bit/npu/kernels
# at boot. Without them the image ships the NPU route and nothing it can load - the route then refuses
# rather than falling back.


echo "== Lemonade (lemond, the lemonade CLI, the web app)"
Expand Down Expand Up @@ -696,6 +750,18 @@ root_elfs_load() { # <root>: name every ELF in the image root that cannot reso
echo " ELF does not load: ${f#"$root"}"
printf '%s\n' "$res" | grep -E "undefined symbol|not found" | sed 's/^/ /'
fi
# ... and a library ldd resolved from the BUILD HOST is not one the image has: ldd falls back to this
# machine's loader cache, so a NEEDED entry the image is missing still resolves here and "loads".
# Every path it reports has to be inside the image; the vdso and the interpreter are the exceptions,
# and neither appears as "=> /path" (engine #1543 review).
outside=$(printf '%s\n' "$res" | awk '/=> \// {print $3}' | while IFS= read -r p; do
case "$p" in "$root"/*) ;; *) printf '%s\n' "$p" ;; esac
done | head -3)
if [ -n "$outside" ]; then
bad=$((bad + 1))
echo " ELF would resolve outside the image: ${f#"$root"}"
printf '%s\n' "$outside" | sed 's/^/ /'
fi
done < <(find "$root" -type f -print0 2>/dev/null)
echo " load check: $n ELF files in the image root, $bad that cannot resolve their symbols"
if [ "$bad" != 0 ]; then
Expand All @@ -707,6 +773,13 @@ root_elfs_load() { # <root>: name every ELF in the image root that cannot reso
echo "== load check: every ELF in the image root, against the image's own libraries"
root_elfs_load "$root"

# engine #1543: the layout row. A build-machine path can arrive as a directory name or as a symlink target,
# where the content rows cannot see it (they read packed ELFs), so the tree itself is checked before it is
# packed. It runs here rather than in the release gate because this is where the layout is still ours to fix.
# shellcheck source=scripts/pack-check.sh
. "$src/scripts/pack-check.sh"
layout_bad=$(layout_build_paths "$root") || { printf '%s\n' "$layout_bad"; echo "the image layout names a build machine: refusing to pack it (engine #1543)"; exit 1; }

echo "== initramfs"
# the builder's umask must not reach the image: nothing group- or world-writable, /root private
# (Dropbear, like sshd, refuses keys when the home directory is group-writable)
Expand Down
31 changes: 31 additions & 0 deletions scripts/pack-check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
# build_path_strings <unpacked tree> no packed ELF names a build-machine path (engine #1461)
# runtime_selfcontained <unpacked tree> nothing resolves outside the payload, and the HSA runtime set
# that libhsa opens at run time ships beside it (engine #1411)
# layout_build_paths <unpacked tree> no path in the LAYOUT (a directory name, or a symlink's target)
# names a build machine - the fourth class, which the two rows
# above cannot see because they read file contents (engine #1543)

# What the engine tarball must and must not hold (docs/releases.md, "Private components"):
# It is packed by scripts/package-linux.sh and folded into the Lemonade + onebit package as 1bit/;
Expand Down Expand Up @@ -72,6 +75,7 @@ gen_text() {
}
build_path_strings() { # build_path_strings <unpacked tree> -> one FAIL line per offending file, rc 1
local t=$1 f n pats rc=0 elfs=0 text=0
[ -d "$t" ] || { printf 'FAIL no such tree: %s\n' "${t:-<empty>}"; return 1; }
pats=$(printf '%s\n' /home/ "${SRC:-/nonexistent-build-src}" "${W:-/nonexistent-build-w}" \
"${BUILD:-/nonexistent-build-build}")
while IFS= read -r -d '' f; do
Expand Down Expand Up @@ -109,6 +113,7 @@ build_path_strings() { # build_path_strings <unpacked tree> -> one FAIL line p
# paths (readelf) and what the loader actually does (ldd), plus the runtime set the dlopen needs present.
runtime_selfcontained() { # runtime_selfcontained <unpacked tree> -> FAIL lines, rc 1
local t=$1 tcanon f n abs rc=0
[ -d "$t" ] || { printf 'FAIL no such tree: %s\n' "${t:-<empty>}"; return 1; }
tcanon=$(readlink -f "$t" 2>/dev/null || printf '%s' "$t")
while IFS= read -r -d '' f; do
pack_is_elf "$f" || continue
Expand Down Expand Up @@ -145,3 +150,29 @@ runtime_selfcontained() { # runtime_selfcontained <unpacked tree> -> FAIL line
[ "$rc" = 0 ] && echo "ok every packed ELF resolves inside the payload or the system loader, declares no absolute RUNPATH, and the HSA runtime set it dlopens ships beside it"
return $rc
}

# engine #1543: the FOURTH leak class - a build-machine path delivered as the filesystem LAYOUT rather than as
# a string inside a file. The rows above read packed ELFs, so neither a directory named after the builder nor
# a symlink pointing into their home is visible to them; the mini image shipped
# /bin/1bit -> /home/<user>/.../build/1bit and a whole /home/<user> tree carrying a 4.7 MB binary.
layout_build_paths() { # layout_build_paths <tree> [pattern ...] (default pattern: /home/)
local t=$1 rc=0 f p l pat
[ -d "$t" ] || { printf 'FAIL no such tree: %s\n' "${t:-<empty>}"; return 1; }
shift
[ $# -gt 0 ] || set -- /home/
while IFS= read -r -d '' f; do
p=${f#"$t"}
for pat in "$@"; do
case $p in *"$pat"*) printf 'FAIL %s is a path naming a build machine\n' "$p"; rc=1 ;; esac
done
# a symlink's TARGET is not file content: it has to be read, not searched
if [ -L "$f" ]; then
l=$(readlink "$f")
for pat in "$@"; do
case $l in *"$pat"*) printf 'FAIL %s -> %s: the target names a build machine\n' "$p" "$l"; rc=1 ;; esac
done
fi
done < <(find "$t" -print0)
[ "$rc" = 0 ] && echo "ok no path in the layout (name or symlink target) names a build machine ($*)"
return $rc
}
27 changes: 27 additions & 0 deletions tests/pack_check_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -108,5 +108,32 @@ printf 'const char *p = "/home/builder/wt/roll/src/app/serve.cpp";\nint g(void)
"$cc" -shared -fPIC -o "$w/buildpath/hrx/llama/bin/libbp.so.0" "$w/bp.c" || exit 2
arm "#1461: a build-machine path in a packed ELF fails" fail "names a build-machine path" "$w/buildpath"

# 6. engine #1543: the layout row. A build-machine path as a DIRECTORY NAME and as a SYMLINK TARGET - the
# two shapes the content rows cannot see, because they read file bytes and these are not bytes a grep
# reaches in the same way (a symlink target is not the link's contents).
mkdir -p "$w/layout/home/builder/wt/roll"
cp "$w/clean/hrx/llama/bin/libhrx.so.0" "$w/layout/home/builder/wt/roll/1bit"
ln -s /home/builder/wt/roll/1bit "$w/layout/bin-1bit-link"
arm_layout() { # arm_layout <want> <phrase> <tree>
local want=$1 phrase=$2 t=$3 out rc=0
out=$(layout_build_paths "$t" 2>&1) || rc=1
if [ "$want" = pass ]; then
[ "$rc" = 0 ] && echo "PASS layout: a tree with no build-machine path passes"
[ "$rc" = 0 ] || { echo "FAIL layout: expected pass, got rc=$rc"; printf '%s\n' "$out" | head -3 | sed 's/^/ /'; fail=$((fail+1)); }
else
if [ "$rc" != 0 ] && printf '%s' "$out" | grep -q -- "$phrase"; then echo "PASS $want"
else echo "FAIL $want: expected a FAIL naming '$phrase', got rc=$rc"; printf '%s\n' "$out" | head -3 | sed 's/^/ /'; fail=$((fail+1)); fi
fi
}
arm_layout "#1543: a directory named after the builder fails" "is a path naming a build machine" "$w/layout"
arm_layout "#1543: a symlink into the builder's home fails" "the target names a build machine" "$w/layout"
arm_layout pass "" "$w/clean"
# a row given a tree that does not exist must refuse: an empty path once made runtime_selfcontained print
# "ok" over nothing, which is the one failure mode a green row cannot be allowed to have
out=$( { build_path_strings ""; runtime_selfcontained ""; layout_build_paths "$w/nonexistent"; } 2>&1 ) || true
n=$(printf '%s\n' "$out" | grep -c 'FAIL no such tree')
if [ "$n" = 3 ]; then echo "PASS a missing tree is refused by all three rows"
else echo "FAIL a missing tree: expected 3 refusals, got $n"; printf '%s\n' "$out" | head -4 | sed 's/^/ /'; fail=$((fail+1)); fi

if [ "$fail" = 0 ]; then echo "pack_check: all rows PASS"; else echo "pack_check: $fail row(s) FAILED"; fi
[ "$fail" = 0 ]
Loading