Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,9 @@ add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.s
# engine#1945: scripts/bump-hrx-sync.sh tells AMD's own commit from a line AMD rebased apart from a commit that
# is not AMD's. Local git repositories only: no network, no GPU.
add_test(NAME bump_hrx_sync COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_sync_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-sync.sh)
# engine#1945: the body `bump-hrx` files when a scheduled sync fails -- the step name, the conflicting paths
# the log names, and the remedy. Fixture logs only: no network, no git.
add_test(NAME bump_hrx_report COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_report_test.sh)
add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $<TARGET_FILE:onebit>)
# serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU.
add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $<TARGET_FILE:onebit>)
Expand Down
128 changes: 128 additions & 0 deletions scripts/bump-hrx-report.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The body of the issue `bump-hrx` files when a scheduled sync fails (engine#1945).
#
# The workflow already notices a failure and files one issue per episode (#332, #350, #372, #388, #406,
# #628, #1945). What it cannot do inline is *read the log*: the body it posts today says "Look at the
# sync steps", so the person who opens the issue starts by opening a run page and scrolling. This
# script turns the failing step's log into the three things that step actually needs — which step
# failed, which files are in conflict (the log names them), and what to do about it.
#
# scripts/bump-hrx-report.sh --run-id 42839424667 [--repo OWNER/REPO] [--log FILE] [--step NAME]
#
# --log defaults to stdin; --step defaults to the step named in the log's own `##[error]` context. The
# body goes to stdout, so the caller decides whether it becomes an issue or a comment.
#
# Exit: 0 the body was produced; 2 the log named no failing step (a report about nothing is worse than
# no report: the caller then posts the workflow's plain body rather than a confident-looking empty one).

set -uo pipefail

run_id=""
repo="${GITHUB_REPOSITORY:-1bit-MONSTER/engine}"
log=""
step=""

while [ $# -gt 0 ]; do
case "$1" in
--run-id) run_id="${2:-}"; shift 2 ;;
--repo) repo="${2:-}"; shift 2 ;;
--log) log="${2:-}"; shift 2 ;;
--step) step="${2:-}"; shift 2 ;;
-h|--help) sed -n '19,33p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
*) printf 'bump-hrx-report: unknown argument: %s\n' "$1" >&2; exit 2 ;;
esac
done

text=""
if [ -n "$log" ]; then
[ -r "$log" ] || { printf 'bump-hrx-report: cannot read %s\n' "$log" >&2; exit 2; }
text=$(cat -- "$log")
else
text=$(cat)
fi

# The step name, in the order GitHub prints it: the `##[group]Run …`/`##[error]` wrapper first, then a bare
# banner, then a `Run <script>` line. Last one wins, because a log can name several steps and the failure
# belongs to the last that started.
# GitHub's own log format, in preference order: `##[section]Starting: <step name>` names the step;
# `##[group]Run <command>` names the command; the `##[error]Process completed with exit code N` trailer
# names neither. Taking the last `##[error]` line first (as this script did before its test existed) reports
# "Process completed with exit code 1." as the failing step, which is the one line the reader already knows.
if [ -z "$step" ]; then
step=$(printf '%s\n' "$text" | grep -E '^##\[section\]Starting: ' | tail -1 | sed -E 's/^##\[section\]Starting: //')
fi
if [ -z "$step" ]; then
step=$(printf '%s\n' "$text" | grep -E '^##\[group\]Run ' | tail -1 | sed -E 's/^##\[group\](Run )?//')
fi
if [ -z "$step" ]; then
step=$(printf '%s\n' "$text" | grep -E '^##\[error\]' | grep -v 'Process completed with exit code' | tail -1 |
sed -E 's/^##\[error\]//')
fi
if [ -z "$step" ] && [ -n "$(printf '%s\n' "$text" | tr -d '[:space:]')" ]; then
# A log with content but no recognisable step banner: report the exit status line if there is one, else refuse.
step=$(printf '%s\n' "$text" | grep -E 'exit code|exit status|Killed|Traceback' | tail -1 |
sed -E 's/^##\[[a-z]+\]//; s/^[[:space:]]+//')
fi
if [ -z "$step" ]; then
printf 'bump-hrx-report: the log names no failing step, so there is nothing specific to report\n' >&2
exit 2
fi

# The exit status, when the log carries one: useful next to the step name, and never a substitute for it.
status=$(printf '%s\n' "$text" | grep -oE 'Process completed with exit code [0-9]+' | tail -1 | grep -oE '[0-9]+$')
status_line=""
if [ -n "$status" ]; then
status_line=" (exit $status)"
fi

# Conflicting paths. git names them in four shapes, and all four appear in this workflow's history:
# CONFLICT (content): Merge conflict in <path>
# CONFLICT (modify/delete): <path> deleted in <ref> and modified in <ref>
# CONFLICT (submodule): Merge conflict in <path>
# error: Merging is not possible because you have unmerged files. + git ls-files output shape
files=$(
printf '%s\n' "$text" |
sed -nE \
-e 's/^.*CONFLICT \([a-z\/-]+\): Merge conflict in (.*)$/\1/p' \
-e 's/^.*CONFLICT \([a-z\/-]+\): (.*) deleted in .*$/\1/p' \
-e 's/^.*CONFLICT \([a-z\/-]+\): (.*) created in .*$/\1/p' |
sed -E 's/[[:space:]]+$//' | sort -u
)

{
if [ -n "$run_id" ]; then
printf 'The bump-hrx run %s failed: https://github.com/%s/actions/runs/%s\n\n' "$run_id" "$repo" "$run_id"
else
printf 'A scheduled bump-hrx run failed.\n\n'
fi
printf '**The failing step:** `%s`%s\n\n' "$step" "$status_line"

if [ -n "$files" ]; then
printf '**Conflicting files** (named in the log):\n\n```\n%s\n```\n\n' "$files"
printf 'Resolve them by hand on `1bit/hrx-vulkan-patched` (for `third_party/llama.cpp`) or on '
printf '`1bit-MONSTER/hrx-system` `1bit/main` (for `third_party/hrx-system`), then re-run this workflow.\n\n'
else
printf '**No conflicting files were named in the log.** This run failed for a reason other than a merge '
printf 'conflict, so the step above is where to start: the log for that step is the record, and this body\n'
printf 'deliberately does not guess at a cause it did not see.\n\n'
fi

printf 'The HRX pin is what this workflow moves, so until it is green the pin is frozen at whatever it was.\n'
printf 'Why this step exists: bump-hrx failed on every scheduled run from 2026-10-01 to 2026-10-05 without\n'
printf 'anyone noticing, and the HRX pin silently froze.\n'
}
113 changes: 113 additions & 0 deletions tests/bump_hrx_report_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# tests/bump_hrx_report_test.sh -- scripts/bump-hrx-report.sh against logs in the shapes this workflow
# has actually failed with (engine#1945). No network, no git: fixtures only.

set -uo pipefail

root=$(cd -- "$(dirname -- "$0")/.." && pwd)
script="$root/scripts/bump-hrx-report.sh"
fails=0
tmp=$(mktemp -d)
trap 'rm -rf -- "$tmp"' EXIT

check() { # check <ok|fail> <what>
if [ "$1" = ok ]; then
printf 'ok %s\n' "$2"
else
printf 'FAIL %s\n' "$2"
fails=$((fails + 1))
fi
}

has() { # has <file> <literal>
grep -qF -- "$2" "$1"
}

# 1. a merge conflict in the llama.cpp submodule: the step and the path, both from the log
cat > "$tmp/conflict.log" <<'LOG'
##[section]Starting: Set up job
##[section]Starting: Sync the fork and merge AMD's pin into our line
##[group]Run git merge --no-edit origin/1bit/hrx-vulkan-patched
Auto-merging ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp
CONFLICT (content): Merge conflict in ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp
Automatic merge failed; fix conflicts and then commit the result.
##[error]Process completed with exit code 1.
LOG
"$script" --run-id 42839424667 --repo 1bit-MONSTER/engine --log "$tmp/conflict.log" > "$tmp/out1.txt"
check $([ $? -eq 0 ] && echo ok || echo fail) "conflict log: a body is produced"
check $(has "$tmp/out1.txt" "42839424667" && has "$tmp/out1.txt" "actions/runs/42839424667" && echo ok || echo fail) \
"conflict log: the run id and its URL are in the body"
check $(has "$tmp/out1.txt" "ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp" && echo ok || echo fail) \
"conflict log: the conflicting path is NAMED (the whole point of reading the log)"
check $(has "$tmp/out1.txt" "Resolve them by hand on \`1bit/hrx-vulkan-patched\`" && echo ok || echo fail) \
"conflict log: the remedy names where to resolve it"
check $(has "$tmp/out1.txt" "**The failing step:** \`Sync the fork and merge AMD's pin into our line\`" && echo ok || echo fail) \
"conflict log: the failing step is the SECTION name, not the exit-code trailer"
check $(has "$tmp/out1.txt" '**The failing step:** `Process completed with exit code' && echo fail || echo ok) \
"conflict log: the exit-code trailer is not reported AS the step"
check $(has "$tmp/out1.txt" '(exit 1)' && echo ok || echo fail) \
"conflict log: the exit status rides beside the step name, as context"

# 2. a modify/delete conflict, the second shape git prints
cat > "$tmp/modify-delete.log" <<'LOG'
##[section]Starting: Merge AMD's hrx-system pin into our line
CONFLICT (modify/delete): third_party/hrx-system/README.md deleted in 1bit/main and modified in HEAD. Version HEAD of third_party/hrx-system/README.md left in tree.
##[error]Process completed with exit code 1.
LOG
"$script" --run-id 7 --log "$tmp/modify-delete.log" > "$tmp/out2.txt"
check $(has "$tmp/out2.txt" "third_party/hrx-system/README.md" && echo ok || echo fail) \
"modify/delete log: the path is named, not just 'a conflict'"

# 3. two different paths in one log: both, deduplicated
cat > "$tmp/two.log" <<'LOG'
##[section]Starting: Sync
CONFLICT (content): Merge conflict in a/b.cpp
CONFLICT (content): Merge conflict in c/d.cpp
CONFLICT (content): Merge conflict in a/b.cpp
##[error]Process completed with exit code 1.
LOG
"$script" --run-id 8 --log "$tmp/two.log" > "$tmp/out3.txt"
check $([ "$(grep -c 'a/b.cpp' "$tmp/out3.txt")" -eq 1 ] && echo ok || echo fail) \
"two conflicts: a repeated path appears once"
check $(has "$tmp/out3.txt" "c/d.cpp" && echo ok || echo fail) "two conflicts: both paths are named"

# 4. a failure that is NOT a conflict: the body says so instead of inventing a cause
cat > "$tmp/generic.log" <<'LOG'
##[section]Starting: Sync the fork and merge AMD's pin into our line
##[group]Run git fetch
fatal: unable to access 'https://github.com/AMD/...': Could not resolve host
##[error]Process completed with exit code 128.
LOG
"$script" --run-id 9 --log "$tmp/generic.log" > "$tmp/out4.txt"
check $(has "$tmp/out4.txt" "No conflicting files were named in the log" && echo ok || echo fail) \
"generic log: the body refuses to name a conflict it did not see"
check $(has "$tmp/out4.txt" "(exit 128)" && echo ok || echo fail) \
"generic log: the step line carries the exit status"
check $(has "$tmp/out4.txt" "a/b.cpp" && echo fail || echo ok) "generic log: no path is invented"

# 5. an empty log: refuse (exit 2) rather than post a confident-looking empty body
: > "$tmp/empty.log"
"$script" --run-id 10 --log "$tmp/empty.log" > "$tmp/out5.txt" 2> "$tmp/err5.txt"
check $([ $? -eq 2 ] && echo ok || echo fail) "empty log: exits 2"
check $([ ! -s "$tmp/out5.txt" ] && echo ok || echo fail) "empty log: prints no body"
check $(has "$tmp/err5.txt" "names no failing step" && echo ok || echo fail) "empty log: says why on stderr"

# 6. --step wins over what the log says, so a caller that knows better can say so
"$script" --run-id 11 --log "$tmp/generic.log" --step 'Sync the fork and merge AMD'"'"'s pin into our line' > "$tmp/out6.txt"
check $(has "$tmp/out6.txt" "Sync the fork and merge AMD's pin into our line" && echo ok || echo fail) \
"--step: the caller's step name is used"

# 7. stdin works (the workflow pipes a step log), and no --run-id still produces a body
cat "$tmp/conflict.log" | "$script" > "$tmp/out7.txt"
check $([ $? -eq 0 ] && echo ok || echo fail) "stdin: a body is produced without --log"
check $(has "$tmp/out7.txt" "A scheduled bump-hrx run failed." && echo ok || echo fail) \
"stdin: with no run id the body says so rather than printing an empty link"

if [ "$fails" -ne 0 ]; then
printf 'bump_hrx_report: %d failure(s)\n' "$fails"
exit 1
fi
printf 'bump_hrx_report: all cases ok\n'
Loading