diff --git a/CMakeLists.txt b/CMakeLists.txt index e20a86864..420dc7928 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -342,6 +342,12 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) +# engine #1750: the carried amdxdna series applies in order to the pinned driver, and the context-restart code it +# produces fails a context's jobs (and logs it) when the restart fails, instead of leaving them parked or running +# them at close on a half-built firmware context. The driver's own functions in a user-space harness: no NPU, no +# kernel headers. Skips (77) where the driver source is not checked out (third_party/xdna-driver or ONEBIT_XDNA_SRC). +add_test(NAME amdxdna_restart COMMAND bash ${CMAKE_SOURCE_DIR}/tests/amdxdna_restart_test.sh ${CMAKE_SOURCE_DIR}) +set_tests_properties(amdxdna_restart PROPERTIES SKIP_RETURN_CODE 77) # engine #713: the pinned HRX toolchain (config/hrx-toolchain.config, scripts/fetch-rocm-sdk.sh) -- the # pin file's refusals, the completeness check against a fabricated tree, and that cmake and # weekly.sh both use the pin instead of the box's /opt/rocm-therock. No network, no SDK, no GPU. diff --git a/tests/amdxdna_restart_harness.c b/tests/amdxdna_restart_harness.c new file mode 100644 index 000000000..4540f621c --- /dev/null +++ b/tests/amdxdna_restart_harness.c @@ -0,0 +1,227 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// tests/amdxdna_restart_harness.c - runs the amdxdna driver's own context-restart code in user space +// (engine#1750, engine#1855). tests/amdxdna_restart_test.sh applies third_party/patches/amdxdna to the pinned +// driver, cuts aie2_hwctx_job_pause(), aie2_hwctx_stop(), aie2_hwctx_restart() and aie2_hwctx_job_unpause() out +// of the resulting aie2_ctx.c, and compiles them into this file as RESTART_FUNCS. Everything they call is a stub +// here. The stubs keep the two properties the checks depend on, both read from the driver: +// - aie2_destroy_context() clears the mailbox channel, aie2_create_context() sets it only on success; +// - aie2_job_run() refuses a job with -ENODEV when the context has no mailbox channel, and otherwise sends it. +// What is real is the control flow under test: when the queue is unpaused, and what state the context is in then. +#include +#include +#include +#include +#include +#include +#include + +typedef uint64_t u64; +struct list_head { struct list_head *next, *prev; }; +static void INIT_LIST_HEAD(struct list_head *h) { h->next = h->prev = h; } +static void list_add_tail(struct list_head *n, struct list_head *h) { n->prev = h->prev; n->next = h; h->prev->next = n; h->prev = n; } +static void list_del(struct list_head *n) { n->prev->next = n->next; n->next->prev = n->prev; n->next = n->prev = n; } +static size_t list_count_nodes(struct list_head *h) { size_t c = 0; for (struct list_head *p = h->next; p != h; p = p->next) c++; return c; } +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +#define list_for_each_entry_safe(pos, n, head, member) \ + for (pos = container_of((head)->next, __typeof__(*pos), member), \ + n = container_of(pos->member.next, __typeof__(*pos), member); \ + &pos->member != (head); pos = n, n = container_of(n->member.next, __typeof__(*n), member)) + +#define AMDXDNA_INVALID_ADDR (~0ULL) +struct amdxdna_gem_obj { u64 addr; struct { size_t size; } mem; }; +struct amdxdna_dev { void *dev_handle; }; +struct amdxdna_client { struct amdxdna_dev *xdna; int dev_heap_xa; }; +struct amdxdna_hwctx_priv { + struct amdxdna_gem_obj *heap; + void *mbox_chann; + void *cached_health; + int io_lock; + struct list_head paused_job_list; + bool job_paused; +}; +struct amdxdna_hwctx { + struct amdxdna_client *client; + struct amdxdna_hwctx_priv *priv; + const char *name; + int fw_ctx_id; + unsigned long last_attached_heap; +}; +enum job_state { PARKED, SENT, FAILED }; +struct amdxdna_sched_job { + struct amdxdna_hwctx *hwctx; + struct list_head aie2_job_paused_list; + enum job_state state; + int err; + bool sent_to_unconfigured; // sent while the firmware context was not fully set up +}; + +// ---- the fake firmware ----------------------------------------------------------------------------- +static struct { int create, map, config; } fail; // what each restart step returns +static bool fw_ctx, fw_mapped, fw_configured; // what firmware has for the context +static int destroys; +static char logbuf[4096]; +static void logf_(char lvl, const char *fmt, ...) +{ + size_t n = strlen(logbuf); + va_list ap; + va_start(ap, fmt); + n += (size_t)snprintf(logbuf + n, sizeof(logbuf) - n, "%c ", lvl); + n += (size_t)vsnprintf(logbuf + n, sizeof(logbuf) - n, fmt, ap); + snprintf(logbuf + n, sizeof(logbuf) - n, "\n"); + va_end(ap); +} +#define XDNA_DBG(xdna, fmt, ...) ((void)(xdna), logf_('D', fmt, ##__VA_ARGS__)) +#define XDNA_ERR(xdna, fmt, ...) ((void)(xdna), logf_('E', fmt, ##__VA_ARGS__)) +#define XDNA_WARN(xdna, fmt, ...) ((void)(xdna), logf_('W', fmt, ##__VA_ARGS__)) +#define mutex_lock(m) ((void)(m)) +#define mutex_unlock(m) ((void)(m)) +#define kfree(p) ((void)(p)) +// no extra heaps: the loop body (aie2_add_host_buf) is compiled and never entered +#define xa_for_each_range(xa, index, entry, first, last) for ((index) = (first); (void)(xa), (void)(entry), (index) < (first); (index)++) + +static u64 amdxdna_obj_dma_addr(struct amdxdna_gem_obj *o) { return o->addr; } +static int aie2_create_context(void *ndev, struct amdxdna_hwctx *hwctx) +{ + (void)ndev; + if (fail.create) return fail.create; + hwctx->priv->mbox_chann = hwctx; hwctx->fw_ctx_id = 7; fw_ctx = true; fw_mapped = fw_configured = false; + return 0; +} +static int aie2_destroy_context(void *ndev, struct amdxdna_hwctx *hwctx) +{ + (void)ndev; + if (!hwctx->priv->mbox_chann) return 0; + hwctx->priv->mbox_chann = NULL; hwctx->fw_ctx_id = -1; fw_ctx = fw_mapped = fw_configured = false; destroys++; + return 0; +} +static int aie2_map_host_buf(void *ndev, int ctx, u64 addr, size_t size) +{ + (void)ndev; (void)ctx; (void)addr; (void)size; + if (fail.map) return fail.map; + fw_mapped = true; + return 0; +} +static int aie2_add_host_buf(void *ndev, int ctx, u64 addr, size_t size) { (void)ndev; (void)ctx; (void)addr; (void)size; return 0; } +static int aie2_config_cu(struct amdxdna_hwctx *hwctx, void *cb) +{ + (void)hwctx; (void)cb; + if (fail.config) return fail.config; + fw_configured = true; + return 0; +} +static void aie2_job_run(struct amdxdna_sched_job *job) +{ + if (!job->hwctx->priv->mbox_chann) { job->state = FAILED; job->err = -ENODEV; return; } + job->state = SENT; + job->sent_to_unconfigured = !(fw_ctx && fw_mapped && fw_configured); +} + +void aie2_hwctx_job_pause(struct amdxdna_hwctx *hwctx); +void aie2_hwctx_job_unpause(struct amdxdna_hwctx *hwctx); +int aie2_hwctx_stop(struct amdxdna_hwctx *hwctx); +int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx); +#include RESTART_FUNCS + +// ---- the scenarios --------------------------------------------------------------------------------- +static struct amdxdna_dev dev; +static struct amdxdna_client client = { .xdna = &dev }; +static struct amdxdna_gem_obj heap; +static struct amdxdna_hwctx_priv priv; +static struct amdxdna_hwctx ctx; +static int fails; + +static void submit(struct amdxdna_sched_job *job) // aie2_cmd_submit()'s last step +{ + memset(job, 0, sizeof(*job)); + job->hwctx = &ctx; job->state = PARKED; + INIT_LIST_HEAD(&job->aie2_job_paused_list); + if (ctx.priv->job_paused) list_add_tail(&job->aie2_job_paused_list, &ctx.priv->paused_job_list); + else aie2_job_run(job); +} +// a running context, stopped as a TDR or a suspend stops it, with one job submitted while it is stopped +static void stopped_context(struct amdxdna_sched_job *job) +{ + memset(&fail, 0, sizeof(fail)); memset(&priv, 0, sizeof(priv)); memset(&ctx, 0, sizeof(ctx)); + logbuf[0] = 0; destroys = 0; + heap.addr = 0x1000; heap.mem.size = 4096; + priv.heap = &heap; INIT_LIST_HEAD(&priv.paused_job_list); + ctx.client = &client; ctx.priv = &priv; ctx.name = "hwctx.test.1"; + aie2_create_context(NULL, &ctx); fw_mapped = fw_configured = true; + aie2_hwctx_stop(&ctx); + destroys = 0; + submit(job); +} +static void check(const char *scenario, bool ok, const char *what) +{ + printf("%s %s: %s\n", ok ? "ok " : "FAIL", scenario, what); + if (!ok) fails++; +} +static bool logged(char lvl, const char *needle) +{ + for (char *l = logbuf; l && *l; l = strchr(l, '\n') ? strchr(l, '\n') + 1 : NULL) { + char *e = strchr(l, '\n'), *hit = strstr(l, needle); + if (l[0] == lvl && hit && (!e || hit < e)) return true; + } + return false; +} +// what must hold after a restart that failed with the context still owned by a live client +static void failed_restart(const char *s, int ret, int want, struct amdxdna_sched_job *job) +{ + struct amdxdna_sched_job later; + check(s, ret == want, "the restart returns the step's error"); + check(s, job->state == FAILED && job->err == -ENODEV, "the parked job is failed with -ENODEV by the restart, not held until close"); + check(s, !fw_ctx && !ctx.priv->mbox_chann, "firmware is left with no context for it"); + check(s, logged('E', "restart failed") && logged('E', ctx.name), "the failure is logged at error level with the context's name"); + check(s, !logged('D', "restarted") && !logged('E', "restarted"), "nothing logs it as restarted"); + submit(&later); + check(s, later.state == FAILED && later.err == -ENODEV, "a later submission fails with -ENODEV at once"); + aie2_hwctx_job_unpause(&ctx); // aie2_hwctx_fini() does this unconditionally + check(s, !job->sent_to_unconfigured && !later.sent_to_unconfigured && job->state != SENT && later.state != SENT, + "close sends nothing to firmware"); +} + +int main(void) +{ + struct amdxdna_sched_job job; + int ret; + + stopped_context(&job); + ret = aie2_hwctx_restart(&dev, &ctx); + check("restart succeeds", ret == 0 && job.state == SENT && !job.sent_to_unconfigured && !ctx.priv->job_paused, + "the parked job is sent to the restarted, configured context"); + + stopped_context(&job); fail.config = -EIO; + failed_restart("config CU fails", aie2_hwctx_restart(&dev, &ctx), -EIO, &job); + + stopped_context(&job); fail.map = -EINVAL; + failed_restart("map host buffer fails", aie2_hwctx_restart(&dev, &ctx), -EINVAL, &job); + + stopped_context(&job); fail.create = -ETIMEDOUT; + failed_restart("create context fails", aie2_hwctx_restart(&dev, &ctx), -ETIMEDOUT, &job); + + // a context whose owner is closing: -ESRCH is what the TDR walk and the device resume key on (0004, 0008) + stopped_context(&job); heap.addr = AMDXDNA_INVALID_ADDR; + ret = aie2_hwctx_restart(&dev, &ctx); + check("owner is closing", ret == -ESRCH, "the restart still returns -ESRCH"); + check("owner is closing", !fw_ctx, "no firmware context is created for it"); + aie2_hwctx_job_unpause(&ctx); + check("owner is closing", job.state == FAILED && job.err == -ENODEV, "close fails the parked job with -ENODEV"); + + if (fails) { printf("%d check(s) failed\n", fails); return 1; } + printf("ok amdxdna restart\n"); + return 0; +} diff --git a/tests/amdxdna_restart_test.sh b/tests/amdxdna_restart_test.sh new file mode 100755 index 000000000..94fe2ac1e --- /dev/null +++ b/tests/amdxdna_restart_test.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/amdxdna_restart_test.sh — the carried amdxdna series applies, in order, to the pinned driver, and the +# context-restart code it produces fails a context's jobs when the restart fails (engine#1750, engine#1855). +# +# Usage: amdxdna_restart_test.sh [repo root] +# +# No NPU, no kernel headers, no module: the series is applied to a copy of the driver source and the four +# functions under test are compiled into a user-space harness (tests/amdxdna_restart_harness.c says what is +# stubbed). The driver source is the engine's submodule (third_party/xdna-driver) or the git checkout +# ONEBIT_XDNA_SRC names; with neither at the pin, the test skips (77). +set -u +root=$(cd "${1:-$(dirname "$0")/..}" && pwd) +series=$root/third_party/patches/amdxdna +pin=$(git -C "$root" ls-tree HEAD third_party/xdna-driver 2> /dev/null | awk '{print $3}') +[ -n "$pin" ] || { echo "skip: cannot read the third_party/xdna-driver pin (not a git checkout)"; exit 77; } +src="" +for cand in "${ONEBIT_XDNA_SRC:-}" "$root/third_party/xdna-driver"; do + [ -n "$cand" ] && git -C "$cand" cat-file -e "$pin^{commit}" 2> /dev/null && { src=$cand; break; } +done +[ -n "$src" ] || { echo "skip: no driver source at the pin ${pin:0:12} (check out third_party/xdna-driver, or set ONEBIT_XDNA_SRC)"; exit 77; } +cc=${CC:-cc} +command -v "$cc" > /dev/null || { echo "skip: no C compiler"; exit 77; } + +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir "$scratch/tree" +git -C "$src" archive "$pin" drivers/accel/amdxdna | tar -x -C "$scratch/tree" || { echo "FAIL cannot export the driver at ${pin:0:12}"; exit 1; } +git -C "$scratch/tree" init -q +n=0 +for p in "$series"/[0-9][0-9][0-9][0-9]-*.patch; do + if ! git -C "$scratch/tree" apply "$p" 2> "$scratch/apply.err"; then + echo "FAIL $(basename "$p") does not apply on the pin ${pin:0:12} after the $n patch(es) before it:" + sed 's/^/ /' "$scratch/apply.err" | head -n 5 + exit 1 + fi + n=$((n + 1)) +done +echo "ok $n patches apply in order on ${pin:0:12}" + +# one function definition: from its header line (column 0, not a prototype) to the first closing brace in column 0 +for f in aie2_hwctx_job_pause aie2_hwctx_stop aie2_hwctx_restart aie2_hwctx_job_unpause; do + awk -v f="$f" '!on && $0 ~ "^[a-z].*[ *]" f "\\(" && $0 !~ /;[ \t]*$/ { on = 1 } on { print } on && /^}/ { exit }' \ + "$scratch/tree/drivers/accel/amdxdna/aie2_ctx.c" > "$scratch/$f.c" + [ "$(tail -n 1 "$scratch/$f.c")" = "}" ] || { echo "FAIL cannot find $f() in the patched aie2_ctx.c"; exit 1; } + cat "$scratch/$f.c" +done > "$scratch/funcs.inc" + +if ! "$cc" -std=gnu11 -O1 -Wall -Wno-unused-function -DRESTART_FUNCS="\"$scratch/funcs.inc\"" \ + -o "$scratch/harness" "$root/tests/amdxdna_restart_harness.c" 2> "$scratch/cc.err"; then + echo "FAIL the restart code no longer compiles into the harness (a new callee needs a stub?):" + sed 's/^/ /' "$scratch/cc.err" | head -n 20 + exit 1 +fi +"$scratch/harness" diff --git a/third_party/patches/amdxdna/0013-accel-amdxdna-a-context-that-failed-to-restart-fails-its-jobs.patch b/third_party/patches/amdxdna/0013-accel-amdxdna-a-context-that-failed-to-restart-fails-its-jobs.patch new file mode 100644 index 000000000..cc8b36152 --- /dev/null +++ b/third_party/patches/amdxdna/0013-accel-amdxdna-a-context-that-failed-to-restart-fails-its-jobs.patch @@ -0,0 +1,129 @@ +From 135f6a890220ccecb283997e046ff3417f98915e Mon Sep 17 00:00:00 2001 +From: bong-water-water-bong +Date: Tue, 10 Nov 2026 10:47:01 -0400 +Subject: [PATCH] accel/amdxdna: a context that failed to restart fails its + jobs + +aie2_hwctx_restart() left a context whose restart failed paused. That +kept its queue off a context firmware does not have, but it did not +finish the job: + +- A failure after aie2_create_context() succeeded (map host buffer, + add heap, config CU) left the new firmware context and its started + mailbox channel in place, without a host buffer or a CU + configuration. aie2_hwctx_fini() unpauses unconditionally, so at + close the parked jobs were run on that context. +- A parked job was never failed. Its submitter waited until its own + timeout or until the context was closed. + +On failure, destroy the firmware context again (a no-op when the +create itself failed) and then unpause. aie2_job_run() already refuses +a job on a context with no mailbox channel: it sets -ENODEV on the +fence, signals it and frees the job. So the parked jobs and every +later submission on the context fail at once, and nothing is sent to a +context firmware does not have or has only half of. + +The failure is logged once at error level with the context name, the +error, the teardown result and the number of parked jobs failed. The +-ESRCH return for a context that is being torn down is unchanged, and +so is the TDR resume callback returning 0 for every context. +--- + drivers/accel/amdxdna/aie2_ctx.c | 50 ++++++++++++++++++++++++++------ + drivers/accel/amdxdna/aie2_tdr.c | 9 ++++-- + 2 files changed, 47 insertions(+), 12 deletions(-) + +diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c +index 75bc573..100a07d 100644 +--- a/drivers/accel/amdxdna/aie2_ctx.c ++++ b/drivers/accel/amdxdna/aie2_ctx.c +@@ -137,6 +137,8 @@ int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) + { + struct amdxdna_gem_obj *heap = hwctx->priv->heap; + unsigned long heap_id; ++ int destroy_ret; ++ size_t parked; + int ret; + + /* +@@ -204,19 +206,49 @@ int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) + } + + out: +- /* +- * Only a context that actually came back may take work. Unpausing on the +- * failure path submitted jobs to a context firmware no longer has, and the +- * only visible symptom was an NPU that had "restarted" and then silently +- * stopped answering (engine#1750). A context that failed stays paused, so +- * its jobs get a clean, attributable failure instead of a wedge. +- */ + if (!ret) { + XDNA_DBG(xdna, "%s restarted", hwctx->name); + aie2_hwctx_job_unpause(hwctx); +- } else { +- XDNA_DBG(xdna, "%s restart failed, left paused, ret %d", hwctx->name, ret); ++ return 0; + } ++ ++ /* ++ * -ESRCH: the owner is tearing the context down. It has no firmware ++ * context (aie2_hwctx_stop() destroyed it) and aie2_hwctx_fini() fails ++ * whatever is parked on it, so there is nothing to do here. ++ */ ++ if (ret == -ESRCH) ++ return ret; ++ ++ /* ++ * The restart failed. Two things were wrong with leaving it at that ++ * (engine#1750, engine#1855): ++ * ++ * - A failure after aie2_create_context() succeeded (map host buffer, ++ * add heap, config CU) left a firmware context and a started mailbox ++ * channel behind with no host buffer or no CU configuration. ++ * Whoever unpaused the queue next - aie2_hwctx_fini() does, ++ * unconditionally - ran the parked jobs on that context. ++ * - A parked job was never failed: its submitter waited until its own ++ * timeout or until the context was closed, which is the "NPU ++ * recovered and then stopped answering" symptom. ++ * ++ * So take the firmware context away again (a no-op when the create ++ * itself failed), and then unpause: aie2_job_run() refuses a job on a ++ * context with no mailbox channel, sets -ENODEV on its fence, signals ++ * it and frees the job. The parked jobs and every later submission on ++ * this context get that error at once, and nothing is ever sent to a ++ * context firmware does not have or has only half of. A later restart ++ * that succeeds (the next device resume) makes the context usable again. ++ */ ++ mutex_lock(&hwctx->priv->io_lock); ++ parked = list_count_nodes(&hwctx->priv->paused_job_list); ++ mutex_unlock(&hwctx->priv->io_lock); ++ ++ destroy_ret = aie2_destroy_context(xdna->dev_handle, hwctx); ++ XDNA_ERR(xdna, "%s restart failed, ret %d: no firmware context (teardown ret %d), failing %zu parked job(s) and later submissions with -ENODEV", ++ hwctx->name, ret, destroy_ret, parked); ++ aie2_hwctx_job_unpause(hwctx); + return ret; + } + +diff --git a/drivers/accel/amdxdna/aie2_tdr.c b/drivers/accel/amdxdna/aie2_tdr.c +index fbe9a86..1615839 100644 +--- a/drivers/accel/amdxdna/aie2_tdr.c ++++ b/drivers/accel/amdxdna/aie2_tdr.c +@@ -135,9 +135,12 @@ static int aie2_tdr_resume_hwctx(struct amdxdna_hwctx *hwctx, void *arg) + */ + rarg->skipped++; + } else if (ret) { +- /* aie2_hwctx_restart() left it paused when it failed: it is counted as +- * failed here, and must not be unpaused by a retry either. */ +- XDNA_ERR(rarg->xdna, "%s restart failed, left paused, ret %d", hwctx->name, ret); ++ /* ++ * aie2_hwctx_restart() has already taken the firmware context ++ * away, failed the parked jobs and logged it; this context ++ * refuses work with -ENODEV. It is counted as failed here. ++ */ ++ XDNA_ERR(rarg->xdna, "%s restart failed, ret %d", hwctx->name, ret); + rarg->failed = true; + } + +-- +2.53.0 + diff --git a/third_party/patches/amdxdna/README.md b/third_party/patches/amdxdna/README.md index 9b9e7f571..cadcfa514 100644 --- a/third_party/patches/amdxdna/README.md +++ b/third_party/patches/amdxdna/README.md @@ -50,9 +50,10 @@ two-client shape (3/3 wedges on `646FFC30…`) now refuses the second client `co | 0008 | `accel/amdxdna: a dying context must not fail the device resume` (2 files, +33/−4) | `aie2_hwctx_resume_cb()` failed the whole walk on the `-ESRCH` that `aie2_hwctx_restart()` returns for a context whose heap address is gone (a client closing while the device is runtime-suspended), and `aie2_hw_resume()` then `break`s out of the **client** loop: every context of every later client stayed destroyed with no firmware context at all, and `pm_runtime_resume_and_get()` reported failure for a resume that had in fact started the hardware, so `amdxdna_pm_resume_get()` called `pm_runtime_set_suspended()` on a started device | the c29 take's own log: `amdxdna_pm_resume_get: Resume failed: -3` then `amdxdna_drm_close: Resume failed (-3), pid 11374 restore data not reclaimed`, followed immediately by `device is already started` — the resume ran, the dying context made it report failure (engine#1051) | | 0009 | `accel/amdxdna: keep the array awake while a context exists` (1 file, +35/−0) | a runtime suspend destroys every firmware context and the resume recreates them; on this platform a recreated context is created and host-buffer-mapped with the same messages, the same fw ctx id and the same address, `aie2_hwctx_restart()` returns 0 — and its first dispatch still never reaches the DPU (`DPU PC 0xffffffff`, `TXN OP ID 0xffffffff`, `timed out sub command 0`), after which the array wedges until a power-cycle. Take a runtime-PM usage reference for the life of each hw_context (`pm_runtime_get_noresume()` in `aie2_hwctx_init()`, released with `pm_runtime_put_autosuspend()` in `aie2_hwctx_fini()`), so the array cannot enter that state under any client; a device with no contexts still autosuspends at 5 s | the engine#1051 ladder on strixhalo 2026-11-06: **3/3 wedge** with `power/control=auto` on the shipping module, each log showing `Destroyed fw ctx 15` → suspend → `Created fw ctx 15` + host buffer → `restarted, ret 0` → wedge, while the same request into a context created after the resume succeeded and XRT's own validate passed 5/5 across resumes. This is the driver-level form of the c30 keep-awake policy, so it protects every client, not only the payload that polls GET_INFO | -| 0010 | `accel/amdxdna: a context that failed to restart stays paused` (2 files, +16/−3) | `aie2_hwctx_restart()` called `aie2_hwctx_job_unpause()` from its shared `out:` label, so a context whose restart had just **failed** was unpaused anyway, and `aie2_tdr_resume_hwctx()` logged it as restarted either way: the queue then drained onto a `fw_ctx_id` firmware no longer has, and the only visible symptom is an NPU that says it recovered and then silently stops answering (engine#1750). The unpause now happens only on success. **What this does NOT deliver** (engine#1855): `aie2_hwctx_fini()` still calls `aie2_hwctx_job_unpause()` unconditionally -- this patch touches one call site, not fini -- and `aie2_hwctx_job_unpause()` *runs* the parked jobs (`list_del` + `aie2_job_run`) rather than failing them, so a failed context's parked jobs are executed at teardown. That is strictly better than draining them onto a `fw_ctx_id` firmware no longer has, but it is not the clean, attributable failure this row used to claim: the residual is a context whose restart failed doing work at destroy time. Failing those jobs at fini is a second patch, not a doc change, and is not carried here. One change fixes both callers — the TDR resume callback and `aie2_hwctx_resume_cb()` on the pm-resume path, which had the same defect; `aie2_hwctx_init()` keeps its own unpause and is untouched | **carried, NOT measured** — no TDR run yet, and the issue's own first question (does firmware reject a command on a destroyed `fw_ctx_id` cleanly?) is the run that would settle bug vs robustness smell. Verified here instead: it applies **on top of `0001`–`0009`** (`git apply --check` against the series result) and **conflicts on the bare pin**, so it must be applied in series order; and the module **compiles** against `7.4.0-rc1-1bit`'s build tree (`amdxdna.ko`, 14,033,392 B, 0 errors, `aie2_ctx.o` and `aie2_tdr.o` rebuilt), never loaded — `/lib/modules` untouched, no `modprobe`, no parameter changed | +| 0010 | `accel/amdxdna: a context that failed to restart stays paused` (2 files, +16/−3) | `aie2_hwctx_restart()` called `aie2_hwctx_job_unpause()` from its shared `out:` label, so a context whose restart had just **failed** was unpaused anyway, and `aie2_tdr_resume_hwctx()` logged it as restarted either way: the queue then drained onto a `fw_ctx_id` firmware no longer has, and the only visible symptom is an NPU that says it recovered and then silently stops answering (engine#1750). The unpause now happens only on success. **What this does NOT deliver** (engine#1855): `aie2_hwctx_fini()` still calls `aie2_hwctx_job_unpause()` unconditionally -- this patch touches one call site, not fini -- and `aie2_hwctx_job_unpause()` *runs* the parked jobs (`list_del` + `aie2_job_run`) rather than failing them, so a failed context's parked jobs are executed at teardown. That is strictly better than draining them onto a `fw_ctx_id` firmware no longer has, but it is not the clean, attributable failure this row used to claim: the residual is a context whose restart failed doing work at destroy time. Failing those jobs is a second patch, not a doc change: it is `0013`, which also replaces this patch's "stays paused" outcome on the failure path. One change fixes both callers — the TDR resume callback and `aie2_hwctx_resume_cb()` on the pm-resume path, which had the same defect; `aie2_hwctx_init()` keeps its own unpause and is untouched | **carried, NOT measured** — no TDR run yet, and the issue's own first question (does firmware reject a command on a destroyed `fw_ctx_id` cleanly?) is the run that would settle bug vs robustness smell. Verified here instead: it applies **on top of `0001`–`0009`** (`git apply --check` against the series result) and **conflicts on the bare pin**, so it must be applied in series order; and the module **compiles** against `7.4.0-rc1-1bit`'s build tree (`amdxdna.ko`, 14,033,392 B, 0 errors, `aie2_ctx.o` and `aie2_tdr.o` rebuilt), never loaded — `/lib/modules` untouched, no `modprobe`, no parameter changed | | 0011 | `accel/amdxdna: document trusted mailbox ring geometry and drop the power-of-two check` (1 file, +7/−5) — **carried, NOT installed** | `xdna_mailbox_start_channel()` refused any channel whose firmware-supplied `rb_size` was not a power of two (`-EINVAL`), which turns a ring geometry AMD's signed firmware handed us into a channel that cannot start at all. Upstream drops the check and documents that the geometry, `rb_start_addr` and `rb_size` come from signed firmware and are trusted by design | **carried, NOT measured** — no run has needed a non-power-of-two ring here, so this is a port of an upstream decision rather than a fix we have watched. Verified: the file is the upstream change unmodified, and it `git apply --check`s **both** on the bare pin (`526c6081`) and on top of `0001`–`0010`, so it is order-independent in the series. Never loaded: `/lib/modules` untouched, no `modprobe`, no parameter changed | | 0012 | `accel/amdxdna: Reject device heap BO when dev_mem_size is not set` (1 file, +5) — **carried, NOT installed** | a heap-BO request on a device with no heap hit a `WARN_ON` and a bogus alignment test instead of one quiet refusal; upstream returns `-EOPNOTSUPP` from the allocation path. Ours warns today (`amdxdna_gem.c`), so this removes a warning from a hot path rather than changing behaviour anyone has depended on | **carried, NOT measured** — the warn path has not been hit on this box in the fleet's runs. Verified the same way: unmodified upstream change, applies on the bare pin and on `0001`–`0011`. Not loaded | +| 0013 | `accel/amdxdna: a context that failed to restart fails its jobs` (2 files, +47/−12) — **carried, NOT installed** | the half of engine#1750 that `0010` left (engine#1855). Read at the pin: `aie2_destroy_context()` clears the context's mailbox channel, `aie2_create_context()` sets it only on success, and `aie2_job_run()` already fails a job on a context with no channel (`-ENODEV` on its fence, signalled, freed). So after `0010` two cases were still wrong. (1) A restart that fails **after** the create succeeded (map host buffer, add heap, config CU) left a firmware context and a started channel with no host buffer or CU configuration, and `aie2_hwctx_fini()`, which unpauses unconditionally, ran the parked jobs on it at close. (2) In every failure case a parked job was never failed: its submitter waited until its own timeout or until close. On failure the patch destroys the firmware context again (a no-op when the create itself failed) and then unpauses, so the parked jobs and every later submission on that context fail with `-ENODEV` at once through the driver's existing path, and nothing is sent to a context firmware lacks or has half of. The failure is logged once at error level by `aie2_hwctx_restart()` itself (context name, error, teardown result, parked-job count), so the device-resume path, which logged nothing, is covered as well as the TDR path. Unchanged: `-ESRCH` for a context that is being torn down (`0004`, `0008` key on it), and the TDR resume callback returning 0 for every context | **carried, NOT measured on the NPU** — no TDR has been provoked against it, and it has never been loaded (`/lib/modules` untouched, no `modprobe`). Verified without the device: `tests/amdxdna_restart_test.sh` (ctest `amdxdna_restart`) applies the whole series to the pin and runs the patched `aie2_hwctx_restart()` / `aie2_hwctx_job_unpause()` text in a user-space harness with stubbed callees — 13 of its 26 checks fail on `0001`–`0012` and all pass with `0013`; and the module **compiles** against `7.4.0-rc1-1bit`'s build tree with `0001`–`0013` applied (`amdxdna.ko` 14,038,208 B, 0 errors, 0 warnings). The harness proves the control flow, not what firmware does with a destroy request on a half-built context: that is the TDR run | **Numbering note (2026-11-06):** the kernel lane's `~/briefs/TKERNEL-74-STATE.md` uses its own series names (`0007`-`0012`) for what is `0004`/`0005`/`0003` here. The numbers in *this* directory are the ones the install order and @@ -64,9 +65,9 @@ reproduces, byte for byte, the sources each shipped build was built from (checke `drivers/accel/amdxdna/*.c` and `*.h`), and rebuilding that tree reproduces that build's fingerprint — see *Expected fingerprints* below. -**`0010` is outside that reproduced set.** A tree carrying it reproduces **no** shipped fingerprint, because nothing has been -built from it: it is carried and unmeasured (see its row), so a fingerprint comparison including it must fail and that failure is -the honest result. `0001`–`0009` remain the set whose rebuild matches a shipped build. +**`0010` and `0013` are outside that reproduced set.** A tree carrying either reproduces **no** shipped fingerprint, because nothing +has been built from them for a box: they are carried and unmeasured (see their rows), so a fingerprint comparison including them must fail +and that failure is the honest result. `0001`–`0009` remain the set whose rebuild matches a shipped build. ## Building, signing, installing, loading