diff --git a/CMakeLists.txt b/CMakeLists.txt index b6b578068..a2dec637f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -379,6 +379,10 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) +# engine #1504: the hardware gate's build does not start below its memory floor, a memory-guard stop is not +# resumed like a thermal stop, and the stopped compile (or GPU step) is really gone. A fake root with a stub +# cmake, a stub thermal-run and a fake meminfo: no GPU, no compiler, runs in CI. +add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_test.sh ${CMAKE_SOURCE_DIR}) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. # The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh) diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 67402e213..bd849352e 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -214,12 +214,20 @@ Each of those steps: byte-exact, zero resets). The lock is what the **NPU phase** takes, below, because an NPU dispatch must not overlap another NPU dispatch. - **Heat.** It waits for Tctl to drop to 75 C, then runs under `~/lb/thermal-run.sh`, which freezes it at 78 C and kills it at 93 C. -- **Memory.** It runs under `scripts/mem-guard.sh` (kills the step below 12 GB MemAvailable). Before a step starts, `tools/mem_budget.py` estimates it. A step that would take GTT to 95 GiB or more (the engine#315 fault zone), or past MemAvailable, is refused. The gate then waits for memory to free up and retries every minute, up to ten times. +- **Memory.** It runs under `scripts/mem-guard.sh --term` (stops the step below 12 GB MemAvailable; `--term` because the guard's default SIGKILL ends `thermal-run` but leaves the step it started running, with its memory). Before a step starts, `tools/mem_budget.py` estimates it. A step that would take GTT to 95 GiB or more (the engine#315 fault zone), or past MemAvailable, is refused. The gate then waits for memory to free up and retries every minute, up to ten times (`HW_GATE_MEM_RETRIES`). It gives the GPU token back for each of those waits and takes it again before the next try, because the jobs holding the memory may be queued behind the token. Under the `gpu-hold` wrapper the ticket belongs to the wrapper and stays held. - **The PR-Agent model.** coder-llm (Vulkan, :8090) loses its device when long HRX dispatches starve it. The gate runs on `strixhalo-engine`, the runner that also runs the pr-agent jobs, and a runner takes one job at a time, so no review is generating during the gate. Before each step the gate also waits for coder-llm's slots to be idle. Afterwards it reports any new `ring ... timeout` in the kernel log. The build runs under thermal-run too, and resumes after a thermal kill. Build trees and ccache persist under `~/.cache/hw-gate`, so a PR that does not move the pins rebuilds in seconds. +The build has its own memory check (engine#1504: on 2026-11-08 both self-hosted runners were OOM-killed 76 s +after a restart handed them a compile). Before it compiles, the gate waits for 24 GB of `MemAvailable` +(`scripts/mem-wait.sh`, polled once a minute for up to 10 minutes; `HW_GATE_BUILD_MEM_FLOOR`, +`HW_GATE_BUILD_MEM_WAIT`). If memory does not free up, the build is not started and the gate exits 2 with the +reading. The compile runs under `scripts/mem-guard.sh --term`, which stops the whole `thermal-run` scope +below 12 GB (`HW_GATE_MEM_FLOOR`). A guard stop is exit 2 too and is not resumed: a re-run once memory frees +is the fix. `tests/hw_gate_mem_test.sh` (ctest `hw_gate_mem`) checks all of it with a fake meminfo. + ## Goldens: where they live - **kat goldens are in git:** `tests/golden/hw-gate/kat-*.json`. They are the text and per-token logprobs of every case, a few KB per model. diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 62f5c7f2d..337e388a0 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -126,6 +126,9 @@ token_held_by_ancestor() { } THERMAL=${HW_GATE_THERMAL:-$HOME/lb/thermal-run.sh} MEM_FLOOR=${HW_GATE_MEM_FLOOR:-12} +# the build (engine#1504): do not start a compile below this much MemAvailable; wait this many minutes for it +BUILD_MEM_FLOOR=${HW_GATE_BUILD_MEM_FLOOR:-24} +BUILD_MEM_WAIT=${HW_GATE_BUILD_MEM_WAIT:-10} CODER_URL=${HW_GATE_CODER_URL:-http://127.0.0.1:8090} CODER_WAIT=${HW_GATE_CODER_WAIT:-900} TOKEN=${HW_GATE_TOKEN:-$HOME/GPU-TOKEN} @@ -336,7 +339,9 @@ trap release_token EXIT trap 'exit 143' TERM INT HUP # so a killed run (timeout, CI cancel) still releases it # run "$@" on the GPU: hold the GPU token, wait for coder-llm, run it under mem-guard + thermal-run -# (no ~/npu.lock: engine#682, measured in engine#673) +# (no ~/npu.lock: engine#682, measured in engine#673). The guard is --term: its default SIGKILL ends +# thermal-run but not thermal-run's scope, so the step it "killed" kept running and kept its memory +# (engine#1504; tests/hw_gate_mem_test.sh). gpu() { if ! take_token; then result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" @@ -350,11 +355,20 @@ gpu() { export HW_GATE_STAMP=$OUT/.stamp rm -f "$HW_GATE_STAMP" while :; do - "$0" --cool --mem "${GPU_MODEL:--}" "$ROOT/scripts/mem-guard.sh" "$MEM_FLOOR" "$THERMAL" "$@" + "$0" --cool --mem "${GPU_MODEL:--}" "$ROOT/scripts/mem-guard.sh" --term "$MEM_FLOOR" "$THERMAL" "$@" rc=$? - # 76: the memory pre-check said no; nothing is held, so other work can finish first + # 76: the memory pre-check said no, and the step holds nothing. The gate still holds the GPU token, though, + # and the jobs whose memory it is waiting for may be queued behind that token: so the token is given back + # for the wait and taken again before the next try (review of engine#1936). Under the gpu-hold wrapper the + # ticket is the wrapper's and cannot be given back from here; release_token is then a no-op. [ $rc = 76 ] && [ $tries -lt "${HW_GATE_MEM_RETRIES:-10}" ] || break - tries=$((tries + 1)); sleep 60 + tries=$((tries + 1)) + release_token + sleep "${HW_GATE_MEM_RETRY_SLEEP:-60}" + if ! take_token; then + result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" + exit 2 + fi done [ $rc = 76 ] && { note "::error::memory pre-check refused the step ${tries} times"; infra=1; } t1=$(date +%s) @@ -429,7 +443,16 @@ if [ -z "$BIN" ]; then [ "$(cat "$build/.hip-flags" 2>/dev/null)" = "$hip_flags" ] || rm -f "$build/CMakeCache.txt" # the same configuration as cmake/hrx.cmake (llama_hrx), plus the two test tools : > "$OUT/build.log" - built=1 + built=1 guarded=0 + # Memory (engine#1504). On 2026-11-08 this job was handed to a just-restarted runner on a box with 0 swap + # free and both runners were OOM-killed 76 s later; the compile was the one step here with no memory check. + # So: no compile below BUILD_MEM_FLOOR GB MemAvailable (wait BUILD_MEM_WAIT min, then exit 2 with the + # reading), and the compile runs under mem-guard --term, which ends the whole thermal-run scope below + # MEM_FLOOR GB. A guard stop is not heat: the loop below does not resume it. + if ! "$ROOT/scripts/mem-wait.sh" "$BUILD_MEM_FLOOR" "$BUILD_MEM_WAIT" >> "$OUT/build.log" 2>&1; then + result build FAIL $(( $(date +%s) - t0 )) "not started: $(tail -n 1 "$OUT/build.log")" + exit 2 + fi # thermal-run kills a build at 93 C (the box is shared); ninja resumes where it stopped for attempt in 1 2 3 4 5 6; do cool @@ -441,7 +464,8 @@ if [ -z "$BIN" ]; then -DGGML_HRX_HIP_COMPILER="$TOOLCHAIN/bin/amdclang++" "-DGGML_HRX_HIP_FLAGS=$hip_flags" \ -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_TESTS=ON -DLLAMA_CURL=OFF >> "$OUT/build.log" 2>&1 && echo "$addon_pin" > "$build/.gpu-kernels-pin" && echo "$hip_flags" > "$build/.hip-flags"; }; } && - "$THERMAL" cmake --build "$build" -j "$JOBS" --target "${targets[@]}" >> "$OUT/build.log" 2>&1 && { built=0; break; } + "$ROOT/scripts/mem-guard.sh" --term "$MEM_FLOOR" "$THERMAL" cmake --build "$build" -j "$JOBS" --target "${targets[@]}" >> "$OUT/build.log" 2>&1 && { built=0; break; } + grep -q "^GUARD:" <(tail -n 40 "$OUT/build.log") && { guarded=1; break; } # out of memory, not heat grep -q "error:" <(tail -n 40 "$OUT/build.log") && break # a compile error, not heat note "build stopped (attempt $attempt, Tctl $(tctl) C); resuming" done @@ -455,6 +479,10 @@ if [ -z "$BIN" ]; then BUILT=1 result build PASS $(( $(date +%s) - t0 )) "$build" else + if [ $guarded = 1 ]; then + result build FAIL $(( $(date +%s) - t0 )) "stopped by the memory guard: $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)" + exit 2 + fi result build FAIL $(( $(date +%s) - t0 )) "see $OUT/build.log" # the compiler's error lines only, not its source excerpts: the job log is public and the # add-on's sources are not diff --git a/scripts/mem-guard.sh b/scripts/mem-guard.sh index 1a7e35e59..7b29a69a5 100755 --- a/scripts/mem-guard.sh +++ b/scripts/mem-guard.sh @@ -20,11 +20,14 @@ # --term sends SIGTERM first and SIGKILL 10 s later. Use it when cmd is ~/lb/thermal-run.sh: a # SIGKILL'd thermal-run cannot run its trap, so its scope (ninja and every compiler under it) keeps # running and keeps the memory; on SIGTERM it kills the whole scope (engine#1504). +# MEM_GUARD_MEMINFO names the file read instead of /proc/meminfo (the tests: "plenty, then starved" cannot be +# produced on demand on a shared box). +meminfo=${MEM_GUARD_MEMINFO:-/proc/meminfo} term=0; [ "$1" = --term ] && { term=1; shift; } floor=$1; shift "$@" & pid=$! while kill -0 $pid 2>/dev/null; do - avail=$(awk '/MemAvailable/ {print int($2/1048576)}' /proc/meminfo) + avail=$(awk '/MemAvailable/ {print int($2/1048576)}' "$meminfo") if [ "$avail" -lt "$floor" ]; then echo "GUARD: MemAvailable ${avail} GB < ${floor} GB, killing $pid" >&2 if [ $term = 1 ]; then diff --git a/scripts/mem-wait.sh b/scripts/mem-wait.sh index 00ce6df3b..a6c0f059f 100755 --- a/scripts/mem-wait.sh +++ b/scripts/mem-wait.sh @@ -19,15 +19,17 @@ # compile into an exhausted box is what turned one OOM into two on 2026-11-08 (engine#1504; both self-hosted # runners were OOM-killed 76 s after a restart, with 0 swap free). Each poll prints what the box looks like, so a # refusal says why. The floor is MemAvailable only: on this APU GTT is host RAM, and MemAvailable already counts it. +# MEM_GUARD_MEMINFO names the file read instead of /proc/meminfo, and MEM_WAIT_POLL the seconds between polls +# (default 60); both are for the tests. floor=${1:?usage: mem-wait.sh }; max=${2:?usage: mem-wait.sh } -meminfo() { awk -v k="$1" '$1 == k":" {print int($2/1048576)}' /proc/meminfo; } +meminfo() { awk -v k="$1" '$1 == k":" {print int($2/1048576)}' "${MEM_GUARD_MEMINFO:-/proc/meminfo}"; } for i in $(seq 0 "$max"); do avail=$(meminfo MemAvailable) line="MemAvailable ${avail} GB (floor ${floor} GB), swap free $(meminfo SwapFree) of $(meminfo SwapTotal) GB" if [ "$avail" -ge "$floor" ]; then echo "mem-wait: ok: $line"; exit 0; fi [ "$i" = "$max" ] && break echo "mem-wait: waiting ($((i + 1)) of $max min): $line" - sleep 60 + sleep "${MEM_WAIT_POLL:-60}" done echo "mem-wait: REFUSED after ${max} min: $line. The box is short of memory, so this job did not start its build;" \ "re-run it once memory frees (engine#1504)." >&2 diff --git a/tests/hw_gate_mem_test.sh b/tests/hw_gate_mem_test.sh new file mode 100755 index 000000000..9efe7957a --- /dev/null +++ b/tests/hw_gate_mem_test.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/hw_gate_mem_test.sh — the hardware gate's build does not start into a box that is short of memory, and +# a memory-guard stop really ends the job (engine#1504). +# +# What it pins, each one a thing that happened on 2026-11-08 when both self-hosted runners were OOM-killed: +# 1. preflight: below the build floor the compile is never started; the gate exits 2 ("the box could not run +# the gate") and its result row carries the MemAvailable reading; +# 2. guard: when memory falls during the compile the gate stops it once, does NOT resume it the way it resumes a +# thermal stop, exits 2 and names the guard; +# 3. the stop is real: the compile process is gone afterwards. mem-guard's default SIGKILL ends thermal-run but +# not what thermal-run started (a SIGKILL cannot run its trap), so the job kept running and kept its memory; +# 4. the same for a GPU step, whose guard is in hw-gate.sh's gpu(): it must be --term. +# +# Usage: hw_gate_mem_test.sh [repo root] +# +# No GPU, no compiler: the gate runs in a fake root with a stub cmake and a stub thermal-run that has the real +# one's property (the job is in its own session; TERM/INT/HUP kill it, SIGKILL orphans it). The meminfo is fake +# (MEM_GUARD_MEMINFO): "plenty, then starved" cannot be produced on demand on a shared box. +set -u +src=$(cd "${1:-$(dirname "$0")/..}" && pwd) +scratch=$(mktemp -d) +trap 'for f in "$scratch"/*.pid; do [ -f "$f" ] && kill -9 "$(cat "$f")" 2> /dev/null; done; rm -rf "$scratch"' EXIT +fails=0 +fail() { echo "FAIL $*"; fails=$((fails + 1)); } + +root=$scratch/root +mkdir -p "$root/scripts" "$root/third_party/llama.cpp" "$root/third_party/hrx-system" "$scratch/bin" +cp "$src"/scripts/hw-gate.sh "$src"/scripts/mem-guard.sh "$src"/scripts/mem-wait.sh "$src"/scripts/hrx-env.sh "$root/scripts/" +touch "$root/third_party/llama.cpp/CMakeLists.txt" "$root/third_party/hrx-system/CMakeLists.txt" + +sed 's/^MemAvailable:.*/MemAvailable: 900000000 kB/' /proc/meminfo > "$scratch/mem-high" +sed 's/^MemAvailable:.*/MemAvailable: 1048576 kB/' /proc/meminfo > "$scratch/mem-low" + +# stub thermal-run: the job in its own session, killed on TERM/INT/HUP only (as ~/lb/thermal-run.sh) +cat > "$scratch/thermal.sh" <<'STUB' +#!/usr/bin/env bash +setsid "$@" & +p=$! +trap 'kill -KILL -- -$p 2> /dev/null; exit 143' TERM INT HUP +wait $p +STUB +# stub cmake: configure writes the cache; --build counts itself, then finishes (FAKE_BUILD=ok) or hangs +cat > "$scratch/bin/cmake" <<'STUB' +#!/usr/bin/env bash +if [ "$1" = -S ]; then mkdir -p "$4" && touch "$4/CMakeCache.txt"; exit 0; fi +echo x >> "$FAKE_STATE/builds" +if [ "$FAKE_BUILD" = ok ]; then + mkdir -p "$2/bin" + for b in llama-server test-backend-ops llama-perplexity; do printf '#!/bin/sh\n' > "$2/bin/$b"; chmod +x "$2/bin/$b"; done + exit 0 +fi +echo $$ > "$FAKE_STATE/build.pid" +exec sleep 600 +STUB +chmod +x "$scratch/thermal.sh" "$scratch/bin/cmake" + +gate() { # gate [VAR=value...]: run the build phase; rc in $rc, seconds in $secs + local state=$1 mode=$2; shift 2 + mkdir -p "$state" + local t0; t0=$(date +%s) + env PATH="$scratch/bin:$PATH" FAKE_STATE="$state" FAKE_BUILD="$mode" \ + MEM_GUARD_MEMINFO="$scratch/meminfo" MEM_WAIT_POLL=1 \ + HW_GATE_GPU_HOLD=0 HW_GATE_TOKEN=none HW_GATE_PRIVATE=0 HW_GATE_COOL_C=1000 \ + HW_GATE_STORE="$state/store" HW_GATE_OUT="$state/out" HW_GATE_THERMAL="$scratch/thermal.sh" \ + ONEBIT_HRX_TOOLCHAIN="$state/no-toolchain" HRX_SVM=1 GITHUB_STEP_SUMMARY= "$@" \ + timeout 120 "$root/scripts/hw-gate.sh" --phases build > "$state/stdout" 2>&1 + rc=$?; secs=$(( $(date +%s) - t0 )) +} +builds() { if [ -f "$1/builds" ]; then wc -l < "$1/builds"; else echo 0; fi; } + +# 0. the harness: with memory to spare the stub build passes, so a refusal below is the memory check and not the fake root +cp "$scratch/mem-high" "$scratch/meminfo" +gate "$scratch/s0" ok +if [ "$rc" = 0 ] && [ "$(builds "$scratch/s0")" = 1 ]; then echo "ok with memory to spare the build runs and passes" +else fail "the harness: build phase rc=$rc, $(builds "$scratch/s0") compile(s), with memory to spare"; tail -n 5 "$scratch/s0/stdout"; fi + +# 1. preflight +cp "$scratch/mem-low" "$scratch/meminfo" +gate "$scratch/s1" ok HW_GATE_BUILD_MEM_WAIT=1 +if [ "$(builds "$scratch/s1")" != 0 ]; then fail "preflight: the compile started at 1 GB MemAvailable (rc=$rc)" +elif [ "$rc" != 2 ]; then fail "preflight: rc=$rc, want 2 (the box could not run the gate)" +elif ! grep -q "MemAvailable 1 GB (floor 24 GB)" "$scratch/s1/out/results.tsv"; then + fail "preflight: the result row does not carry the reading: $(cat "$scratch/s1/out/results.tsv")" +else echo "ok below the floor the compile is not started: rc=2 after ${secs}s, the row carries the reading"; fi + +# 2 + 3. the guard, during the compile +cp "$scratch/mem-high" "$scratch/meminfo" +( sleep 4; cp "$scratch/mem-low" "$scratch/meminfo" ) & +gate "$scratch/s2" hang +wait +pid=$(cat "$scratch/s2/build.pid" 2> /dev/null); cp "$scratch/s2/build.pid" "$scratch/s2.pid" 2> /dev/null +if [ "$rc" = 124 ]; then fail "guard: the compile ran to the 120 s bound: nothing stopped it when memory fell" +elif [ "$rc" != 2 ]; then fail "guard: rc=$rc, want 2" +elif [ "$(builds "$scratch/s2")" != 1 ]; then fail "guard: $(builds "$scratch/s2") compiles, want 1 (a guard stop was resumed like a thermal stop)" +elif ! grep -q "memory guard" "$scratch/s2/out/results.tsv"; then fail "guard: the result row does not name the guard: $(cat "$scratch/s2/out/results.tsv")" +else echo "ok memory fell during the compile: stopped once, rc=2 after ${secs}s, the row names the guard"; fi +sleep 1 +if [ -n "$pid" ] && kill -0 "$pid" 2> /dev/null; then fail "guard: the compile ($pid) is still running after the guard stopped it" +elif [ -n "$pid" ]; then echo "ok the stopped compile is gone"; fi + +# 4. a GPU step's guard: gpu() runs " --cool --mem - mem-guard thermal-run "; take the guard +# invocation from the script itself, so the test follows what gpu() really runs +guard=$(sed -n 's/.*"\$ROOT\/scripts\/mem-guard.sh" \(.*\)"\$MEM_FLOOR" "\$THERMAL" "\$@".*/\1/p' "$root/scripts/hw-gate.sh") +if [ "$(grep -c '"$ROOT/scripts/mem-guard.sh" .*"$MEM_FLOOR" "$THERMAL" "$@"' "$root/scripts/hw-gate.sh")" != 1 ]; then + fail "gpu step: cannot find gpu()'s one mem-guard call in hw-gate.sh" +else + cp "$scratch/mem-high" "$scratch/meminfo" + ( sleep 3; cp "$scratch/mem-low" "$scratch/meminfo" ) & + # shellcheck disable=SC2086 # $guard is the option word, or nothing + MEM_GUARD_MEMINFO="$scratch/meminfo" timeout 60 "$root/scripts/mem-guard.sh" $guard 12 "$scratch/thermal.sh" \ + bash -c 'echo $$ > "$0"; exec sleep 600' "$scratch/s4.pid" 2> /dev/null + wait; sleep 1 + pid=$(cat "$scratch/s4.pid" 2> /dev/null) + if [ -z "$pid" ]; then fail "gpu step: the step never started" + elif kill -0 "$pid" 2> /dev/null; then fail "gpu step: the step ($pid) is still running after the guard stopped it (mem-guard ${guard:-})" + else echo "ok a GPU step stopped by the guard is gone (mem-guard $guard)"; fi +fi + +# 5. the GPU token is not held while gpu() waits for memory. gpu(), take_token() and release_token() are taken from +# the script as they are and run with a real token file; the pre-check ("$0 --cool --mem") is this harness, which +# refuses twice (76) and records who holds the token each time it is asked and during each wait. +h=$scratch/s5; mkdir -p "$h/out" +for f in take_token release_token gpu; do + awk -v f="$f" '$0 ~ "^" f "\\(\\) \\{" { on = 1 } on { print } on && /^}/ { exit }' "$root/scripts/hw-gate.sh" +done > "$h/funcs.sh" +cat > "$h/harness.sh" << 'EOS' +#!/usr/bin/env bash +h=$(dirname "$0") +held() { if [ -e "$h/token" ]; then echo held; else echo free; fi; } +if [ "${1:-}" = --cool ]; then + n=$(cat "$h/asked" 2> /dev/null || echo 0); echo $((n + 1)) > "$h/asked" + echo "precheck $(held)" >> "$h/log" + [ "$n" -lt 2 ] && exit 76 + exit 0 +fi +TOKEN=$h/token TOKEN_WAIT=30 TOKEN_HELD="" OUT=$h/out ROOT=/nonexistent MEM_FLOOR=1 THERMAL=true infra=0 +note() { :; }; result() { echo "result $*" >> "$h/log"; }; wait_coder_idle() { :; }; ring_timeouts() { echo 0; } +sleep() { echo "wait $(held)" >> "$h/log"; } +. "$h/funcs.sh" +gpu true; echo "rc $? token-after $(held)" >> "$h/log" +EOS +if [ "$(grep -c '^gpu() {' "$h/funcs.sh")" != 1 ] || ! grep -q '^take_token() {' "$h/funcs.sh" || ! grep -q '^release_token() {' "$h/funcs.sh"; then + fail "token: cannot find gpu(), take_token() and release_token() in hw-gate.sh" +else + chmod +x "$h/harness.sh" # gpu() runs "$0 --cool ..." + HW_GATE_MEM_RETRY_SLEEP=0 timeout 60 "$h/harness.sh" > "$h/stdout" 2>&1 + want='precheck held +wait free +precheck held +wait free +precheck held +rc 0 token-after held' + if [ "$(cat "$h/log" 2> /dev/null)" = "$want" ]; then + echo "ok the GPU token is given back for each memory wait and held again for each try" + else + fail "token: gpu() held the GPU token while waiting for memory (or lost it for a try); the sequence was:" + sed 's/^/ /' "$h/log" 2> /dev/null; sed 's/^/ | /' "$h/stdout" | head -n 5 + fi +fi + +[ "$fails" = 0 ] || { echo "$fails check(s) failed"; exit 1; } +echo "ok hw-gate memory checks"