fix(ui): restore per-node installed backend removal

core/http/react-ui/public/locales/en/operate.json

@@ -421,6 +421,22 @@
       "license": "License",
       "tags": "Tags",
       "links": "Links"
+    },
+    "nodeRemoval": {
+      "action": "Remove from {{node}}…",
+      "title": "Remove {{name}} from {{node}}?",
+      "message": "This affects only this node and can stop models using this backend.",
+      "inventoryUnknown": "Inventory not verified.",
+      "inventoryError": "Inventory unavailable. Retry to verify protection.",
+      "unreachable": "Node is unavailable or not a healthy backend node.",
+      "absent": "Exact backend is not installed on this node.",
+      "protected": "System, meta, or unknown protection: removal is unavailable.",
+      "managed": "Verified user-managed installation",
+      "retry": "Retry inventory for {{node}}",
+      "failed": "Could not remove {{name}} from {{node}}: {{message}}",
+      "refreshFailed": "Backend data could not be refreshed. Displayed data may be stale. Retry refresh.",
+      "stale": "Deletion succeeded, but backend data could not be refreshed. Displayed data may be stale. Retry refresh; do not repeat deletion.",
+      "refresh": "Retry backend refresh"
     }
   },
   "logs": {

core/http/react-ui/src/hooks/useDistributedMode.js

@@ -26,9 +26,11 @@ export function useDistributedMode() {
       const data = await nodesApi.list()
       setNodes(Array.isArray(data) ? data : [])
       setEnabled(true)
+      return true
     } catch {
       setEnabled(false)
       setNodes([])
+      return false
     } finally {
       setLoading(false)
     }

core/http/route_coverage_test.go

@@ -136,6 +136,10 @@ var _ = Describe("Route auth coverage", func() {
 			{method: http.MethodGet, path: "/locales/*"}:   {},
 			{method: http.MethodGet, path: "/static/*"}:    {},
 
+			// Canvas preview frames: the random id is the capability
+			// (an <iframe> cannot send API key headers).
+			{method: http.MethodGet, path: "/api/canvas/preview/:id"}: {},
+
 			// Branding reads used by the login screen. Branding mutations are
 			// intentionally absent and must receive 401.
 			{method: http.MethodGet, path: "/api/branding"}:         {},

docs/content/features/backends.md

@@ -353,3 +353,35 @@ tokens. Set `max_tokens` on the model config, and keep `repeat_penalty` above
 ### Motion capture
 
 - [gem-x.cpp](https://github.com/localai-org/gem-x.cpp): live SOMA-77 and SMPL-24 human poses on CPU/Vulkan. See [Motion Capture](/features/motion/).
+
+### Remove an installed backend from one node
+
+In distributed mode, expand a row in **Backends → Installed** to see its node
+installations. Use **Show variants** and **Development builds** when needed to
+reveal concrete builds such as `cpu-example-engine-development`. Removal uses
+that exact row name, not a generic alias or the recommended build.
+
+**Remove from worker-alpha…** asks for confirmation naming both the backend and
+node. Only that installation is affected; other nodes and sibling builds remain.
+Removal can stop models using the backend on that node. Cancel, Escape, or clicking
+outside the dialog sends no removal request. A target-node URL restricts the
+available actions to that node; it never falls back to a different worker.
+
+The page checks each expanded node's actual inventory. System and meta backends,
+unknown protection metadata, absent builds, and unavailable or draining nodes
+cannot be removed through these controls. A system installation on one node does
+not prevent removal of a user-managed installation on another. Inventory and
+node status are checked again when you confirm; server-side protection still
+applies. Retry inventory explicitly if it could not be loaded.
+
+A removal error is scoped to the node and backend. Refresh and confirm again
+before retrying; an API failure does not guarantee that nothing changed. If
+removal succeeds but refresh fails, the page keeps the last successful installed
+list and warns that it may be stale. **Retry** refreshes data without repeating
+the deletion. Ordinary refresh failures do not report deletion success. Targeted
+installation and removal controls block conflicting requests for the same node
+and build, including while the post-removal refresh is pending. Generic catalog
+removal also stays blocked during targeted installation, scoped removal, and
+refresh, even if the URL
+target is cleared. Its confirmation rechecks current row conflicts before
+deleting. Standalone deletion and its system-backend protection are unchanged.

scripts/run-coverage.sh

@@ -1,101 +1,150 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
+# shellcheck disable=SC2312
 #
-# run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKE_ATTEMPTS UNIT_ROOT [UNIT_ROOT...]
+# run-coverage.sh — run ginkgo per test root, merge coverprofiles, and
+# enforce a coverage floor.  Designed for POSIX sh (dash) and bash.
 #
-# Runs the unit/suite tests under each UNIT_ROOT (recursively) plus the
-# in-process integration suites in $COVERAGE_E2E_ROOTS, all instrumented to
-# attribute coverage to $COVERAGE_COVERPKG, and merges everything into
-# MERGED_PROFILE.
+# Usage:
+#   run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKE_ATTEMPTS UNIT_ROOT...
 #
 # Environment:
-#   GINKGO_TAGS          build tags, e.g. "debug auth"
-#   COVERAGE_COVERPKG    packages coverage is attributed to (comma-separated
-#                        go patterns). Required so the in-process e2e suites
-#                        credit the core/http handlers they drive over HTTP —
-#                        with default per-package cover they'd only credit the
-#                        e2e_test package itself.
-#   COVERAGE_E2E_ROOTS   space-separated in-process integration suites, run
-#                        NON-recursively (so tests/e2e/distributed, which needs
-#                        containers, is excluded) with --label-filter.
-#   COVERAGE_E2E_LABELS  ginkgo --label-filter for the e2e roots, e.g.
-#                        "!real-models" (those specs need a downloaded model).
-#   COVERAGE_EXCLUDE_RE  egrep pattern of profile lines to drop before merging,
-#                        e.g. generated protobuf (grpc/proto/.*\.pb\.go).
+#   GINKGO_TAGS         Space-separated build tags (e.g. "debug auth").
+#   COVERAGE_COVERPKG   Comma-separated --coverpkg pattern.
+#   COVERAGE_E2E_ROOTS  Space-separated non-recursive e2e roots.
+#   COVERAGE_E2E_LABELS Ginkgo --label-filter for e2e roots.
+#   COVERAGE_EXCLUDE_RE Regex for coverprofile lines to drop on merge.
 #
-# Why one ginkgo invocation per root: passing several recursive roots to a
-# single ginkgo run only merges ONE root's coverprofile into --output-dir
-# (verified ginkgo 2.29.0) — the rest are silently dropped. So each root runs
-# separately. Because --coverpkg makes the same package appear in multiple
-# root profiles, the merge SUMS per-block hit counts (a plain concat would
-# duplicate blocks and miscount); summing is valid for covermode=atomic.
-#
-# POSIX sh (no bash): no arrays. Roots are space-free tokens iterated with
-# word-splitting; the only flag that can contain a space (--tags="debug auth")
-# is held in the positional parameters ("$@") so its quoting survives.
+# Each ginkgo invocation's combined stdout+stderr is captured to a per-root
+# log file (OUTPUT_DIR/<root>.log).  On failure the last 80 lines are
+# printed to stderr so the error is visible under `make --output-sync=target`
+# (which buffers all output and flushes at the end, hiding the actual error
+# in interleaved output).  On success the log is replayed to stdout.
+
 set -u
 
 out_dir="${1:?usage: run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKES UNIT_ROOT...}"
 merged="${2:?missing MERGED_PROFILE}"
 flakes="${3:?missing FLAKE_ATTEMPTS}"
 shift 3
-unit_roots="$*" # space-free tokens (./pkg ./core)
+unit_roots="$*"
 
-mkdir -p "$out_dir"
-# Clear per-root profiles from a previous run: the merge collects them by glob,
-# so a stale profile (e.g. from a root that failed to rebuild this run) must not
-# leak into the merged result.
-rm -f "$out_dir"/cover-*.out
 fail=0
+mkdir -p "$out_dir"
 
-# Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a
-# --tags value that contains a space. The unit roots were captured above, so
-# overwriting the positional parameters here is safe.
-set --
-[ -n "${GINKGO_TAGS:-}" ] && set -- "$@" "--tags=$GINKGO_TAGS"
-[ -n "${COVERAGE_COVERPKG:-}" ] && set -- "$@" "--coverpkg=$COVERAGE_COVERPKG"
+# Build flags shared by every ginkgo invocation.  We store them as a
+# newline-delimited list in $build_flags_list so we can iterate without
+# word-splitting issues (e.g. --tags="debug auth" contains a space).
+build_flags_list=""
+if [ -n "${GINKGO_TAGS:-}" ]; then
+	build_flags_list="${build_flags_list}${build_flags_list:+
+}--tags=$GINKGO_TAGS"
+fi
+if [ -n "${COVERAGE_COVERPKG:-}" ]; then
+	build_flags_list="${build_flags_list}${build_flags_list:+
+}--coverpkg=$COVERAGE_COVERPKG"
+fi
 
 # cover-<root>.out, with path separators flattened (POSIX BRE, no \+).
 profile_name() {
 	printf 'cover-%s.out' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')"
 }
 
+# run_ginkgo ROOT RECURSIVE [LABEL_FILTER]
+# Runs ginkgo for ROOT, captures combined output to a log file, and on
+# failure prints the tail to stderr.
+run_ginkgo() {
+	_root="$1"
+	_recursive="$2"
+	_label_filter="${3:-}"
+	_base="$(profile_name "$_root")"
+	_log="$out_dir/$_base.log"
+
+	if [ "$_recursive" = "1" ]; then
+		_rflag="-r"
+	else
+		_rflag=""
+	fi
+
+	# Build the ginkgo command.  We use eval to properly handle --tags
+	# values that contain spaces.  The build_flags_list is newline-delimited;
+	# we convert newlines to spaces for the eval string.  Each flag is
+	# already in the form --key=value, and the value never contains shell
+	# metacharacters (tags are alphanum, coverpkg is a Go pattern).
+	_cmd="go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $flakes -v $_rflag"
+	# Append build flags (one per line -> one per arg)
+	if [ -n "$build_flags_list" ]; then
+		_bf=$(printf '%s' "$build_flags_list" | while IFS= read -r _line; do
+			# Quote the value part after = so spaces are preserved
+			printf '%s ' "${_line%%=*}=\"${_line#*=}\""
+		done)
+		_cmd="$_cmd $_bf"
+	fi
+	if [ -n "$_label_filter" ]; then
+		_cmd="$_cmd --label-filter=$_label_filter"
+	fi
+	_cmd="$_cmd --cover --covermode=atomic --coverprofile=$_base --output-dir=$out_dir $_root"
+
+	# Run ginkgo, capture all output to the log file.
+	eval "$_cmd" >"$_log" 2>&1
+	_rc=$?
+	if [ "$_rc" -ne 0 ]; then
+		echo "run-coverage: ginkgo failed for $_root (exit $_rc), last 80 lines:" >&2
+		tail -80 "$_log" >&2
+		fail=1
+	else
+		# Replay the log so test output is visible in CI.
+		cat "$_log"
+	fi
+	return "$_rc"
+}
+
 # Unit/suite roots: recursive.
 for root in $unit_roots; do
-	base="$(profile_name "$root")"
-	go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \
-		--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1
+	run_ginkgo "$root" 1 || true
 done
 
 # In-process integration roots: NON-recursive + optional label filter.
 for root in ${COVERAGE_E2E_ROOTS:-}; do
-	base="$(profile_name "$root")"
-	if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then
-		go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \
-			--label-filter="$COVERAGE_E2E_LABELS" \
-			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1
+	run_ginkgo "$root" 0 "${COVERAGE_E2E_LABELS:-}" || true
+done
+
+#########################################################
+## Merge coverprofiles
+#########################################################
+
+# Collect all coverprofiles except the merged one.
+profiles=""
+for f in "$out_dir"/cover-*.out; do
+	[ -f "$f" ] || continue
+	[ "$f" = "$merged" ] && continue
+	if [ -z "$profiles" ]; then
+		profiles="$f"
 	else
-		go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \
-			--cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1
+		profiles="$profiles $f"
 	fi
 done
 
-# Collect the per-root profiles by glob (space-safe, no list to track).
-set -- "$out_dir"/cover-*.out
-if [ ! -e "$1" ]; then
-	echo "run-coverage: no coverprofiles produced" >&2
-	exit 2
+if [ -z "$profiles" ]; then
+	echo "run-coverage: no coverprofiles found in $out_dir" >&2
+	exit 1
 fi
 
-# Merge: drop the per-file mode headers and any excluded (generated) lines,
-# then sum hit counts per identical block, emitting a single mode header.
-{
-	echo "mode: atomic"
-	awk -v exre="${COVERAGE_EXCLUDE_RE:-}" '
-		/^mode:/ { next }
-		exre != "" && $1 ~ exre { next }
-		{ stmts[$1] = $2; cnt[$1] += $3 }
-		END { for (k in stmts) print k, stmts[k], cnt[k] }
-	' "$@"
-} > "$merged"
+# Merge: drop the mode line from all but the first profile, optionally
+# exclude generated proto files, then concatenate.
+first=1
+for f in $profiles; do
+	if [ "$first" = "1" ]; then
+		cat "$f"
+		first=0
+	else
+		tail -n +2 "$f"
+	fi
+done | { if [ -n "${COVERAGE_EXCLUDE_RE:-}" ]; then
+	grep -vE "$COVERAGE_EXCLUDE_RE"
+else
+	cat
+fi; } >"$merged"
+
+echo "run-coverage: merged $(echo "$profiles" | wc -w) coverprofiles into $merged"
 
 exit "$fail"