#!/bin/bash
# Applies the two GLM-5.3-Flash over RPC fixes to a llama.cpp source tree
# (see GLM5-RPC-FIXES.md for details). Idempotent: already-applied fixes are skipped.
#
# Usage: ./apply_glm_rpc_fix.sh [path-to-source-tree]
# Default tree: the directory containing this script.
set -euo pipefail
TREE="${1:-$(cd "$(dirname "$0")" && pwd)}"
python3 - "$TREE" <<'EOF'
import sys
tree = sys.argv[1]
import re
FIXES = [
{
"file": "ggml/src/ggml-alloc.c",
"title": "zero-sized tensors: place at chunk 0 instead of creating size-0 chunks",
"marker": "zero-sized tensors still need a valid address",
# (indent) captured so the insertion keeps the surrounding style
"regex": (
r"static struct buffer_address ggml_dyn_tallocr_alloc"
r"\(struct ggml_dyn_tallocr \* alloc, size_t size, const struct ggml_tensor \* tensor\) \{\n"
r" size = aligned_offset\(NULL, size, alloc->alignment\);\n"
),
"replacement": (
"static struct buffer_address ggml_dyn_tallocr_alloc"
"(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) {\n"
" size = aligned_offset(NULL, size, alloc->alignment);\n"
"\n"
" // zero-sized tensors still need a valid address. place them at offset 0 of the first chunk\n"
" // instead of allocating memory, so they do not create size-0 chunks that the realloc logic\n"
" // cannot track (such chunks have no valid base on remote buffers)\n"
" if (size == 0) {\n"
" if (alloc->n_chunks == 0) {\n"
" ggml_dyn_tallocr_new_chunk(alloc, alloc->alignment);\n"
" alloc->chunks[0]->max_size = alloc->alignment;\n"
" }\n"
" return (struct buffer_address) { .chunk = 0, .offset = 0 };\n"
" }\n"
),
},
{
"file": "src/models/glm5-next.cpp",
"title": "DSA gather path: fold head dim into N to avoid degenerate N=1 batched GEMM",
"marker": "fold the head dim into N",
"regex": (
r"(?P<ind> )ggml_tensor \* v_t = ggml_cont\(ctx0, ggml_transpose\(ctx0, k_g\)\);\s*"
r"// \[n_sel, kv_lora_rank, 1, n_tokens\]\n"
r" ggml_tensor \* kqv = ggml_mul_mat\(ctx0, v_t, kq\);\s*"
r"// \[kv_lora_rank, 1, n_head, n_tokens\]\n"
),
"replacement": (
" ggml_tensor * v_t = ggml_cont(ctx0, ggml_transpose(ctx0, k_g)); // [n_sel, kv_lora_rank, 1, n_tokens]\n"
" // fold the head dim into N so this is not a strided-batched GEMM with N=1, which some\n"
" // BLAS backends have no kernel for; kq is contiguous, the token batch stays as-is\n"
" ggml_tensor * kqv = ggml_mul_mat(ctx0, v_t, ggml_reshape_4d(ctx0, kq, kq->ne[0], kq->ne[2], 1, kq->ne[3]));\n"
" kqv = ggml_reshape_4d(ctx0, kqv, kv_lora_rank, 1, kq->ne[2], kq->ne[3]); // [kv_lora_rank, 1, n_head, n_tokens]\n"
),
},
]
rc = 0
for fix in FIXES:
path = f"{tree}/{fix['file']}"
try:
with open(path, encoding="utf-8") as f:
src = f.read()
except FileNotFoundError:
print(f"SKIP {fix['file']}: not found (not needed on this tree)")
continue
if fix["marker"] in src:
print(f"OK {fix['file']}: already applied")
continue
m = re.search(fix["regex"], src)
if m is None:
print(f"FAIL {fix['file']}: anchor not found, tree differs from expected")
rc = 1
continue
if len(re.findall(fix["regex"], src)) != 1:
print(f"FAIL {fix['file']}: anchor found multiple times, aborting to stay safe")
rc = 1
continue
src = src[:m.start()] + fix["replacement"] + src[m.end():]
with open(path, "w", encoding="utf-8") as f:
f.write(src)
print(f"APPLIED {fix['file']}: {fix['title']}")
sys.exit(rc)
EOF
if [ $? -eq 0 ]; then
echo "Done."
# suggest the rebuild command for each existing cmake build dir in the tree
found=0
for d in "$TREE"/build*; do
[ -f "$d/CMakeCache.txt" ] || continue
found=1
if grep -q "GGML_HIP:.*=ON" "$d/CMakeCache.txt" 2>/dev/null; then
# rocm includes are not in the cmake cache of these trees, export them
echo " rebuild: cd $d && env CPLUS_INCLUDE_PATH=/opt/rocm/include C_INCLUDE_PATH=/opt/rocm/include LIBRARY_PATH=/opt/rocm/lib cmake --build . --config Release -j\$(nproc)"
else
echo " rebuild: cd $d && cmake --build . --config Release -j\$(nproc)"
fi
done
if [ $found -eq 0 ]; then
echo " no build dir found; configure one first (see GLM5-RPC-FIXES.md, Build section)"
fi
else
echo "One or more fixes failed, see above." >&2
fi
Name and Version
PR from timkhronos #27773 in RPC
./llama-server --version
version: 0.3.0-dev (build 0, commit unknown)
built with GNU 16.2.1 for Linux x86_64
7.2.2-1-cachyos
Operating systems
Linux
GGML backends
RPC
Hardware
Machine 1 with 4x 9060 XT (GFX1200) and Machine 2 with Strix-Halo (GFX1151) + 7600 XT (GFX1102).
Models
I tried the Quant Q3_XL-3.86bpw:
https://huggingface.co/avar6/GLM-5.3-Flash-BF16-gguf/tree/main/Q3_XL-3.86bpw
Problem description & steps to reproduce
I could not run the GLM-5.3-Flash Model with the PR from timkhronos #27773 in RPC when the -b and -ub is above 128. Identical with Vulkan and ROCM.
timkhronos suggested after looking at my fix, that i write a issue at least for the ggml related part.
Works not:
./llama-server -m /home/lunarbuntu/Downloads/hf/GLM-5.3-Flash-Q4/Q3_XL-3.86bpw/GLM-5.3-Flash-Q3_XL-3.86.gguf -ngl 99 --rpc 192.168.10.100:50053 -c 327680 --no-mmap -sm layer --device RPC1,ROCM0,ROCM1,ROCM2,ROCM3,RPC0 -ts 60,7,6,6,6,12 --host 0.0.0.0 --port 5001 --timeout 7200 -to 7200 -cram 32768 -ncmoe 0 --jinja --chat-template /home/lunarbuntu/Downloads/glm5.3.jinja --reasoning-preserve -b 1024 -ub 1024
Works:
./llama-server -m /home/lunarbuntu/Downloads/hf/GLM-5.3-Flash-Q4/Q3_XL-3.86bpw/GLM-5.3-Flash-Q3_XL-3.86.gguf -ngl 99 --rpc 192.168.10.100:50053 -c 327680 --no-mmap -sm layer --device RPC1,ROCM0,ROCM1,ROCM2,ROCM3,RPC0 -ts 60,7,6,6,6,12 --host 0.0.0.0 --port 5001 --timeout 7200 -to 7200 -cram 32768 -ncmoe 0 --jinja --chat-template /home/lunarbuntu/Downloads/glm5.3.jinja --reasoning-preserve -b 128 -ub 128
First Bad Commit
Because PR, there is no previous Version.
Relevant log output
I have no logs but let my local AI write a fix for me.
apply_glm_rpc_fix.sh
Could not upload files GLM5-RPC-FIXES.md is missing here with the AI explanation.