diff --git a/cmake/hrx.cmake b/cmake/hrx.cmake index 2b64394cc..6d3bd2690 100644 --- a/cmake/hrx.cmake +++ b/cmake/hrx.cmake @@ -68,13 +68,18 @@ if(NOT ONEBIT_HRX_TOOLCHAIN) endif() message(STATUS "ONEBIT_HRX: toolchain ${ONEBIT_HRX_TOOLCHAIN}") -# The HRX binaries carry strings from this tree and, with -DONEBIT_GPU_PRIVATE, from the add-on -# checkout; both are rewritten so the shipped libraries name no build-machine path. +# The HRX binaries carry strings from this tree, from the add-on checkout (-DONEBIT_GPU_PRIVATE), and from +# the BUILD tree: ggml-hrx builds its deps as a nested ExternalProject that fetches Loom into +# /hrx/llama/ggml/src/ggml-hrx/hrx/src/ggml-hrx-deps-build/loom, which is where libloomc.so.0's +# __FILE__ strings come from (engine #1461: 7 of the payload's 146 build-machine paths, and mapping only +# the source tree left every one of them absolute). All three are rewritten so the shipped libraries name +# no build-machine path. -ffile-prefix-map covers debug info as well, which the Release -g needs. set(ONEBIT_HRX_PATH_MAP_FLAGS "-ffile-prefix-map=${CMAKE_SOURCE_DIR}=." CACHE STRING "Path mapping for the HRX llama.cpp build") if(ONEBIT_GPU_PRIVATE) string(APPEND ONEBIT_HRX_PATH_MAP_FLAGS " -ffile-prefix-map=${ONEBIT_GPU_PRIVATE}=addon") endif() +string(APPEND ONEBIT_HRX_PATH_MAP_FLAGS " -ffile-prefix-map=${CMAKE_BINARY_DIR}=build") # Flags the HRX llama.cpp build needs from this toolchain, kept apart from the path maps above. set(ONEBIT_HRX_EXTRA_FLAGS "") diff --git a/docs/releases.md b/docs/releases.md index 863bf5e53..ec61c5f7a 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -140,10 +140,17 @@ shipped c32 payload: `bin/lemond` carries 126, `bin/lemonade` 13, `1bit/hrx/llam is 0 since engine#1460), every one of them naming the rolling machine's home directory, e.g. `…/wt/release-rc/work/src/third_party/lemonade/src/cpp/server/server.cpp`. They are `__FILE__` values: the engine's `-ffile-prefix-map` (`cmake/hrx.cmake`) rewrites its own objects but does not reach the nested -`ggml-hrx-deps-build` that compiles Loom, and `scripts/build-lemonade.sh` sets no prefix map at all. So **two rolls of -the same commit produce different payload hashes**, and a sha you are given cannot be reproduced from the source -alone — rebuild, compare behaviour, and compare the engine's own files. engine#1461 carries the mechanism and the fix; -when it lands this paragraph becomes the claim it should have been: the payload is reproducible from the tree. +`ggml-hrx-deps-build` that compiles Loom, and `scripts/build-lemonade.sh` set no prefix map at all — so **two rolls of +the same commit produced different payload hashes**, and a sha you are given could not be reproduced from the source +alone. + +The three mechanisms are fixed (engine#1461), and the payload is now **checked** rather than described: +`scripts/build-lemonade.sh` maps its source tree and its build directory (engine#1464); `cmake/hrx.cmake` also maps +the build tree, which is where Loom's sources are fetched; llama.cpp passes the same flags into its nested +`GGML_HRX_CMAKE_ARGS` (llama.cpp#243), which is where the 7 Loom strings came from; and `scripts/weekly.sh` +(`build_path_strings`) walks **every** packed ELF of both packages — not just the engine binary — counting occurrences +rather than lines, so a roll that leaks one fails instead of shipping. What stays irreproducible from the tree is the +Lemonade dev version's `~N` above, which is a count of commits in the checkout that built it. Artifacts of the gated c19 candidate — sizes and sha256 exactly as its own `SHA256SUMS` records them: diff --git a/registry/architectures.json b/registry/architectures.json index 0d1fd43a4..70c3616c1 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -1,7 +1,7 @@ { "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { - "llama.cpp (hrx)": "fb10c19caf0142d4029421d487de596900e6f73a" + "llama.cpp (hrx)": "6ea38d0457d32c30583ed11d634e9afd146f9103" }, "counts": { "hrx": 298, diff --git a/scripts/weekly.sh b/scripts/weekly.sh index 20f008417..e814bd49c 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -323,6 +323,31 @@ stage_test() { run_tests; } # plaintext ELF; every ELF in the tarball is for the host (x86-64); the packed 1bit has no # symbol table and its strings name no private source file and no build-machine home # Checks the build's 1bit and every file in the tarball. +# engine #1461: no packed ELF may name a build-machine path. One row, used for BOTH packages - the engine +# tarball and the Lemonade package that carries it - over every ELF rather than the packed `1bit` alone: that +# narrow scope is why 146 build paths shipped in c32's payload (lemond 126, lemonade 13, libloomc.so.0 7) +# while the row printed "no build-machine paths", exactly as #1460's `/home/`-only predicate had been too +# narrow a pattern. OCCURRENCES are counted, not lines: a strings run glues paths to their neighbours, so one +# of lemonade's 13 does not begin a line and `grep -c` reports 12 for a file carrying 13 (measured +# 2026-11-08 on c32's own bytes). /home/ covers any builder's home; the source tree, the workdir and the +# build tree cover a build that ran elsewhere on this machine. +build_path_strings() { # build_path_strings -> one FAIL line per offending file, rc 1 + local t=$1 f n pats rc=0 + pats=$(printf '%s\n' /home/ "${SRC:-/nonexistent-build-src}" "${W:-/nonexistent-build-w}" \ + "${BUILD:-/nonexistent-build-build}") + while IFS= read -r -d '' f; do + pack_is_elf "$f" || continue + n=$(strings -a "$f" | grep -o -F -f <(printf '%s\n' "$pats") | wc -l || true) + [ "$n" = 0 ] || { + printf 'FAIL %s names a build-machine path (%s occurrences): %s\n' "$(basename "$f")" "$n" \ + "$(strings -a "$f" | grep -o -F -f <(printf '%s\n' "$pats") | head -2 | tr '\n' ' ')" + rc=1 + } + done < <(find "$t" -type f -print0) + [ "$rc" = 0 ] && echo "ok no packed ELF names a build-machine path (/home/, the source tree, the workdir, the build tree)" + return $rc +} + package_checks() { # package_checks # (no `cmd | grep -q` here: under pipefail grep's early exit fails the pipeline on a match) # (lists go to stderr through `tee >(cat >&2)`, not `tee /dev/stderr`: reopening the log by @@ -439,6 +464,7 @@ package_checks() { # package_checks [ "$m" = 62 ] || { echo "FAIL $f is an ELF for machine $m, not the host (x86-64)"; nk=$((nk + 1)); } done < <(find "$t" -type f -print0) [ "$nk" = 0 ] && echo "ok every ELF in $(basename "$1") is a host (x86-64) program or library" || bad=1 + build_path_strings "$t" || bad=1 f=$(find "$t" -maxdepth 2 -type f -name 1bit) f=${f%%$'\n'*} if [ -n "$f" ]; then @@ -451,17 +477,7 @@ package_checks() { # package_checks done | grep -E '\.(c|cc|cpp|cxx|h|hpp|py)$' | xargs -r -n1 basename | LC_ALL=C sort -u) n=0; [ -z "$privsrc" ] || n=$(strings -a "$f" | grep -c -F -f <(printf '%s\n' "$privsrc") || true) [ "$n" = 0 ] || { echo "FAIL the packed 1bit names a private source file ($n strings)"; b1=1; } - # Any path that names THIS machine's tree, not just /home/: a build under /build or /opt/ci leaks the builder - # the same way, and a /home/-only row let c32 ship a builder path under /opt. Measured 2026-11-08: the c32 and - # c33 tarballs baked /opt/rocm-therock/... and /home//wt/.../libhsa-runtime64.so.1 respectively, and - # only the second tripped this row - the row was right about the leak and too narrow about where it lives. - _badpats=$(printf '%s\n' /home/ "${SRC:-/nonexistent-build-src}" "${W:-/nonexistent-build-w}") - n=$(strings -a "$f" | grep -c -F -f <(printf '%s\n' "$_badpats") || true) - [ "$n" = 0 ] || { - echo "FAIL the packed 1bit names a build-machine path ($n strings): $(strings -a "$f" | grep -F -f <(printf '%s\n' "$_badpats") | head -2 | tr '\n' ' ')" - b1=1 - } - if [ $b1 = 0 ]; then echo "ok the packed 1bit: no symbol table, no private source names, no build-machine paths"; else bad=1; fi + if [ $b1 = 0 ]; then echo "ok the packed 1bit: no symbol table, no private source names"; else bad=1; fi fi for f in LICENSE NOTICE LICENSE-BINARY NOTICE-BINARY; do n=$(find "$t" -maxdepth 2 -type f -name "$f" | wc -l) @@ -497,6 +513,8 @@ lemonade_package_checks() { # lemonade_package_checks n=$(find "$t" -path '*/bin/resources/web-app/index.html' -type f | wc -l) if [ "$n" != 0 ]; then echo "ok Lemonade's web app: bin/resources/web-app/index.html and $(find "$t" -path '*/bin/resources/web-app/*' -type f | wc -l) files in $(basename "$1")" else echo "FAIL no Lemonade web app (bin/resources/web-app/index.html) in $(basename "$1")"; rc=1; fi + build_path_strings "$t" || rc=1 # engine#1461: lemond 126 and lemonade 13 shipped from here + # the engine's own binaries are checked in package_checks; this is the other half of the same payload n=$(find "$t" -maxdepth 2 -name lemond-onebit -type f -exec grep -c '^export ONEBIT_KCRYPT_REQUIRE_SEALED=1$' {} + 2>/dev/null) n=${n%%$'\n'*} if [ "${n:-0}" != 0 ]; then echo "ok lemond-onebit loads sealed NPU kernels only (ONEBIT_KCRYPT_REQUIRE_SEALED=1)" diff --git a/third_party/llama.cpp b/third_party/llama.cpp index fb10c19ca..6ea38d045 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit fb10c19caf0142d4029421d487de596900e6f73a +Subproject commit 6ea38d0457d32c30583ed11d634e9afd146f9103