diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
index 79111771ab7..44405846a1e 100644
--- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
+++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
@@ -25,6 +25,23 @@ This path preserves the sandbox workspace and repairs the agent runtime and host
If the container is paused, follow the printed `docker unpause` guidance instead.
If the container is missing or OpenShell reports another terminal phase such as `Failed`, follow the printed `rebuild --yes` guidance so NemoClaw can recreate the sandbox from its recorded metadata.
+
+If the sandbox has shields up and the OpenClaw gateway does not start after the container restarts, lower shields before you rebuild:
+
+```bash
+$$nemoclaw shields down
+```
+
+While holding the config mutation lock, NemoClaw confirms that no startup process runs and no readiness lease exists.
+Only then does it accept `shields down`.
+Other locked-config operations still require the lease.
+This failed-startup recovery path requires a sandbox image that includes the in-container OpenClaw config and state guards.
+If NemoClaw reports that the config guard is absent, upgrade the CLI.
+Then rebuild the sandbox before you retry recovery.
+After shields are down, start the sandbox again.
+When the OpenClaw gateway is healthy, rerun `shields up`.
+
+
## Recover the Agent Runtime
diff --git a/scripts/openclaw-config-guard.py b/scripts/openclaw-config-guard.py
index 5bcaaddfbbd..9e57df15f27 100755
--- a/scripts/openclaw-config-guard.py
+++ b/scripts/openclaw-config-guard.py
@@ -50,8 +50,13 @@
"publish-startup-ready",
"write-config",
"recover",
+ "unlock-failed-startup",
]
StartupIdentity = tuple[int, str, int]
+# One action only. It unseals both layers in a single mutex window, so the
+# multi-step host sequence can never mutate state on stale evidence (#8304).
+STARTUP_FAILURE_RECOVERY_ACTIONS = frozenset({"unlock-failed-startup"})
+INSTALLED_STATE_DIR_GUARD = "/usr/local/lib/nemoclaw/state-dir-guard.py"
CONFIG_FILES = ("openclaw.json", ".config-hash")
PRODUCTION_CONFIG_DIR = "/sandbox/.openclaw"
MAX_FILE_BYTES = {
@@ -73,6 +78,18 @@
NODE_BINARY_PATH = "/usr/local/bin/node"
JSON5_MODULE_PATH = "/opt/nemoclaw/node_modules/json5"
JSON5_VALIDATION_TIMEOUT_SECONDS = 5
+# Whole-action budget for `unlock-failed-startup`. The recursive state guard
+# caps each transition at ten minutes. Give the forward transition that full
+# allowance, then reserve another full allowance plus two minutes of overhead
+# for the config relock and fail-closed recursive relock.
+# Keep the total below RECOVERY_CONTAINER_TIMEOUT in
+# src/lib/shields/openclaw-config-lock.ts, or the host can kill the guard before
+# the rollback finishes.
+STATE_DIR_GUARD_FORWARD_SECONDS = 10 * 60
+STATE_DIR_GUARD_ROLLBACK_SECONDS = 12 * 60
+STATE_DIR_GUARD_TIMEOUT_SECONDS = (
+ STATE_DIR_GUARD_FORWARD_SECONDS + STATE_DIR_GUARD_ROLLBACK_SECONDS
+)
INSTALLED_HELPER_PATH = "/usr/local/lib/nemoclaw/openclaw-config-guard.py"
COPY_BUFFER_BYTES = 1024 * 1024
STABLE_READ_ATTEMPTS = 3
@@ -986,14 +1003,15 @@ def _pinned_process_matches_supervised_nonroot_start(
os.close(proc_pid_fd)
-def _openshell_supervised_nonroot_start_is_live(
+def _openshell_supervised_nonroot_start_census(
expected_root_uid: int,
expected_sandbox_uid: int,
- required_pid: int | None = None,
-) -> bool:
+) -> tuple[int, int | None] | None:
+ """Return a stable OpenShell start-child census, or ``None`` on uncertainty."""
+
supervisor_identity = _openshell_supervisor_identity(expected_root_uid)
if supervisor_identity is None:
- return False
+ return None
proc_root_fd = -1
try:
proc_root_fd = _open_proc_root()
@@ -1006,7 +1024,7 @@ def _openshell_supervised_nonroot_start_is_live(
continue
observed += 1
if observed > MAX_PROC_ENTRIES:
- return False
+ return None
if _pinned_process_matches_supervised_nonroot_start(
proc_root_fd,
entry.name,
@@ -1016,20 +1034,44 @@ def _openshell_supervised_nonroot_start_is_live(
matches += 1
matched_pid = int(entry.name, 10)
if matches > 1:
- return False
- return bool(
- matches == 1
- and (required_pid is None or matched_pid == required_pid)
- and _openshell_supervisor_identity(expected_root_uid)
- == supervisor_identity
- )
+ return matches, None
+ if _openshell_supervisor_identity(expected_root_uid) != supervisor_identity:
+ return None
+ return matches, matched_pid
except OSError:
- return False
+ return None
finally:
if proc_root_fd >= 0:
os.close(proc_root_fd)
+def _openshell_supervised_nonroot_start_is_live(
+ expected_root_uid: int,
+ expected_sandbox_uid: int,
+ required_pid: int | None = None,
+) -> bool:
+ census = _openshell_supervised_nonroot_start_census(
+ expected_root_uid, expected_sandbox_uid
+ )
+ return bool(
+ census is not None
+ and census[0] == 1
+ and (required_pid is None or census[1] == required_pid)
+ )
+
+
+def _openshell_supervised_nonroot_start_is_absent(
+ expected_root_uid: int,
+ expected_sandbox_uid: int,
+) -> bool:
+ """Return whether a stable OpenShell supervisor has no start child."""
+
+ census = _openshell_supervised_nonroot_start_census(
+ expected_root_uid, expected_sandbox_uid
+ )
+ return census is not None and census[0] == 0
+
+
def _pid1_effective_uid() -> int | None:
"""Read PID 1's effective UID from a pinned procfs descriptor."""
@@ -1325,7 +1367,14 @@ def _revoke_startup_ready(identity: Identity) -> None:
def _validate_action_readiness(
action: Action, startup_owner: bool, identity: Identity
-) -> None:
+) -> bool:
+ """Authorize ``action`` and report whether only the failed-startup path allowed it.
+
+ A ``True`` result is provisional: it rests on a live procfs census taken
+ before the mutation mutex, so the caller must reconfirm it under the mutex
+ with ``_reconfirm_startup_failure_recovery`` before any effect.
+ """
+
startup_action = action in {"revoke-startup-ready", "publish-startup-ready"}
if startup_action:
if not _pid1_is_nemoclaw_start() or not startup_owner or os.getppid() != 1:
@@ -1334,7 +1383,7 @@ def _validate_action_readiness(
STARTUP_READY_PATH,
f"{action} is restricted to the PID 1 startup transaction",
)
- return
+ return False
installed_current = os.path.realpath(__file__) == os.path.realpath(
INSTALLED_HELPER_PATH
)
@@ -1343,11 +1392,14 @@ def _validate_action_readiness(
# A source helper injected into an older image, and the local unit
# harness, retain their explicit compatibility path. Current images
# use the installed helper and authenticate a namespace remap below.
- return
+ return False
protocol_active, startup_ready = _startup_lease_state(identity)
if not pid1_is_nemoclaw_start and not protocol_active:
if (
installed_current
+ # The recovery action is authorized by the escape below and by
+ # nothing else.
+ and action not in STARTUP_FAILURE_RECOVERY_ACTIONS
and _startup_markers_absent(identity)
and _openshell_supervised_nonroot_start_is_live(
identity.root_uid, identity.sandbox_uid
@@ -1360,19 +1412,29 @@ def _validate_action_readiness(
# and NSpid evidence selects the same two topologies. They cannot
# publish root-owned readiness markers, so authenticate the stable
# supervisor/child pair while refusing stale or malformed markers.
- return
- if installed_current:
- raise GuardError(
- "startup-not-ready",
- STARTUP_READY_PATH,
- "installed config guard requires NemoClaw PID 1",
+ return False
+ if (
+ installed_current
+ and action in STARTUP_FAILURE_RECOVERY_ACTIONS
+ and _startup_markers_absent(identity)
+ and _openshell_supervised_nonroot_start_is_absent(
+ identity.root_uid, identity.sandbox_uid
)
- return
+ ):
+ # Provisional: a child can still appear after this scan, so main()
+ # reconfirms under the mutation mutex before any effect (#8304).
+ return True
+ # The early return above leaves `installed_current` true here.
+ raise GuardError(
+ "startup-not-ready",
+ STARTUP_READY_PATH,
+ "installed config guard requires NemoClaw PID 1",
+ )
# Source injected into an older image retains compatibility until that
# image explicitly opts in. The trusted installed helper requires the
# protocol from its very first exec, closing the pre-revoke boot race.
if not installed_current and not protocol_active:
- return
+ return False
if installed_current and not protocol_active:
# The supported --user sandbox entrypoint cannot create a root-owned
# readiness capability. It also explicitly disables gateway privilege
@@ -1382,7 +1444,7 @@ def _validate_action_readiness(
# capability opts even a non-root PID 1 into the strict lease below.
pid1_euid = _pid1_effective_uid()
if pid1_euid is not None and pid1_euid != identity.root_uid:
- return
+ return False
early_recover = action == "recover" and not startup_ready
if early_recover:
if not startup_owner or os.getppid() != 1:
@@ -1391,7 +1453,7 @@ def _validate_action_readiness(
STARTUP_READY_PATH,
f"{action} is restricted to the PID 1 startup transaction",
)
- return
+ return False
if action in {
"lock",
"unlock",
@@ -1404,6 +1466,146 @@ def _validate_action_readiness(
STARTUP_READY_PATH,
"OpenClaw startup is not ready for host config mutations",
)
+ return False
+
+
+def _reconfirm_startup_failure_recovery(action: Action, identity: Identity) -> None:
+ """Re-prove a failed startup while the mutation mutex is held.
+
+ The first scan runs before the mutex. Repeating it here binds the
+ authorization to the effect.
+ """
+
+ if _startup_markers_absent(identity) and _openshell_supervised_nonroot_start_is_absent(
+ identity.root_uid, identity.sandbox_uid
+ ):
+ return
+ raise GuardError(
+ "startup-not-ready",
+ STARTUP_READY_PATH,
+ f"{action} lost its failed-startup authorization before taking effect",
+ )
+
+
+def _run_state_dir_guard(
+ action: str,
+ config_dir: str,
+ plan_json: str,
+ mutation_lock_fd: int,
+ deadline: float,
+) -> None:
+ """Run the recursive state-dir guard under this process's mutation mutex.
+
+ The child takes the same mutex, so pass the held descriptor: it inherits
+ the lock instead of deadlocking on it.
+ """
+
+ if not os.path.isfile(INSTALLED_STATE_DIR_GUARD):
+ raise GuardError(
+ "state-dir-guard-missing",
+ INSTALLED_STATE_DIR_GUARD,
+ "recursive state guard is required for failed-startup recovery",
+ )
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise GuardError(
+ "state-dir-transition-timeout",
+ config_dir,
+ f"no recovery budget left for state-dir {action}",
+ )
+ try:
+ completed = subprocess.run( # noqa: S603
+ [
+ sys.executable,
+ "-I",
+ INSTALLED_STATE_DIR_GUARD,
+ action,
+ "--config-dir",
+ config_dir,
+ "--plan-json",
+ plan_json,
+ "--transition-lock-fd",
+ str(mutation_lock_fd),
+ ],
+ capture_output=True,
+ text=True,
+ timeout=remaining,
+ check=False,
+ pass_fds=(mutation_lock_fd,),
+ )
+ except subprocess.TimeoutExpired as exc:
+ raise GuardError(
+ "state-dir-transition-timeout",
+ config_dir,
+ f"state-dir {action} exceeded the remaining recovery budget",
+ ) from exc
+ except subprocess.SubprocessError as exc:
+ raise GuardError(
+ "state-dir-transition-failed",
+ config_dir,
+ f"state-dir {action} could not complete: {exc}",
+ ) from exc
+ if completed.returncode != 0:
+ detail = (completed.stderr.strip() or completed.stdout.strip())[:400]
+ raise GuardError(
+ "state-dir-transition-failed",
+ config_dir,
+ f"state-dir {action} failed: {detail}",
+ )
+
+
+def _run_failed_startup_unlock(
+ opened: OpenConfig,
+ identity: Identity,
+ config_dir: str,
+ plan_json: str,
+ mutation_lock_fd: int,
+ *,
+ quarantine_untrusted: bool,
+) -> None:
+ """Unseal both OpenClaw state layers or restore their locked posture.
+
+ The forward transition cannot consume the rollback reserve. Both deadlines
+ remain within the host's whole-action timeout.
+ """
+
+ rollback_deadline = time.monotonic() + STATE_DIR_GUARD_TIMEOUT_SECONDS
+ unlock_deadline = rollback_deadline - STATE_DIR_GUARD_ROLLBACK_SECONDS
+
+ try:
+ _run_state_dir_guard(
+ "unlock", config_dir, plan_json, mutation_lock_fd, unlock_deadline
+ )
+ _transition(
+ "unlock",
+ opened,
+ identity,
+ quarantine_untrusted=quarantine_untrusted,
+ )
+ except (GuardError, OSError) as exc:
+ rollback_errors: list[str] = []
+ try:
+ _transition(
+ "lock",
+ opened,
+ identity,
+ quarantine_untrusted=quarantine_untrusted,
+ )
+ except (GuardError, OSError) as rollback_exc:
+ rollback_errors.append(f"config lock: {rollback_exc}")
+ try:
+ _run_state_dir_guard(
+ "lock", config_dir, plan_json, mutation_lock_fd, rollback_deadline
+ )
+ except (GuardError, OSError) as rollback_exc:
+ rollback_errors.append(f"state-dir lock: {rollback_exc}")
+
+ detail = str(exc)
+ if rollback_errors:
+ detail += "; rollback issues: " + "; ".join(rollback_errors)
+ if isinstance(exc, GuardError):
+ raise GuardError(exc.code, exc.path, detail) from exc
+ raise GuardError("operation-failed", config_dir, detail) from exc
def _write_secondary_journal(record: dict[str, object], identity: Identity) -> None:
@@ -3495,7 +3697,11 @@ def _transition(
freeze_started = False
try:
freeze_started = True
- _freeze(opened, identity)
+ _freeze(
+ opened,
+ identity,
+ quarantine_reserved=quarantine_untrusted,
+ )
_settle_pending_transaction_for_lock(opened, identity)
_repair_absent_hash_for_lock(opened, identity)
source = _snapshot_raw_pair(opened)
@@ -3540,7 +3746,11 @@ def _transition(
_verify_locked_posture(opened, pair, identity, allow_blocking_flags=True)
snapshots: list[FileSnapshot] = []
try:
- _freeze(opened, identity)
+ _freeze(
+ opened,
+ identity,
+ quarantine_reserved=quarantine_untrusted,
+ )
snapshots.extend(_snapshot_pair(opened))
targets, _digest = _canonical_targets(
(snapshots[0], snapshots[1]), identity, locked=False
@@ -4130,11 +4340,13 @@ def _parser() -> argparse.ArgumentParser:
"publish-startup-ready",
"write-config",
"recover",
+ "unlock-failed-startup",
),
)
parser.add_argument("--config-dir", default=PRODUCTION_CONFIG_DIR)
parser.add_argument("--expected-config-sha256", default="")
parser.add_argument("--startup-owner", action="store_true")
+ parser.add_argument("--plan-json", default=None)
return parser
@@ -4163,9 +4375,26 @@ def main(argv: list[str] | None = None) -> int:
f"helper is restricted to {PRODUCTION_CONFIG_DIR}",
)
identity = _production_identity()
- _validate_action_readiness(action, args.startup_owner, identity)
+ startup_failure_recovery = _validate_action_readiness(
+ action, args.startup_owner, identity
+ )
+ if action == "unlock-failed-startup" and not startup_failure_recovery:
+ # Never let this action inherit the ordinary lease path.
+ raise GuardError(
+ "failed-startup-not-proven",
+ STARTUP_READY_PATH,
+ "unlock-failed-startup requires a proven terminal startup failure",
+ )
read_only = action in {"preflight", "preflight-restart"}
mutex = _acquire_mutation_mutex(action, identity, exclusive=not read_only)
+ if startup_failure_recovery:
+ _reconfirm_startup_failure_recovery(action, identity)
+ if action == "unlock-failed-startup" and args.plan_json is None:
+ raise GuardError(
+ "invalid-state-lock-plan",
+ "--plan-json",
+ "unlock-failed-startup requires the agent state lock plan",
+ )
if action in {"revoke-startup-ready", "publish-startup-ready"}:
if action == "revoke-startup-ready":
@@ -4312,6 +4541,19 @@ def main(argv: list[str] | None = None) -> int:
recovery, new_digest, original_locked = _recover_any_transaction(
opened, identity, pending_journal
)
+ elif action == "unlock-failed-startup":
+ # Every check that can refuse this action has already run, so no
+ # refusal can leave state unsealed under a locked config.
+ _run_failed_startup_unlock(
+ opened,
+ identity,
+ args.config_dir,
+ args.plan_json,
+ mutex.fd,
+ quarantine_untrusted=untrusted_reserved_entry,
+ )
+ new_digest = None
+ recovery = None
else:
_transition(
action,
diff --git a/scripts/state-dir-guard.py b/scripts/state-dir-guard.py
index 73704ef8e1b..a470870d682 100755
--- a/scripts/state-dir-guard.py
+++ b/scripts/state-dir-guard.py
@@ -47,6 +47,7 @@
{"/sandbox/.openclaw", "/sandbox/.hermes", "/sandbox/.deepagents"}
)
OPENCLAW_MUTATION_MUTEX_PATH = "/run/nemoclaw/openclaw-config-mutation.lock"
+MAX_TRANSITION_LOCK_BYTES = 16 * 1024
# Keep this exact source/target contract aligned with
# src/lib/state/openclaw-managed-extensions.ts.
OPENCLAW_IMAGE_PACKAGE_PATHS = frozenset(
@@ -2378,16 +2379,108 @@ def _acquire_transition_lock(path: str, identity: Identity) -> int:
os.close(parent_fd)
+def _run_guard_with_transition_lock_fd(
+ action: Action,
+ config_dir: str,
+ identity: Identity,
+ plan: AgentStateLockPlan,
+ lock_path: str,
+ lock_fd: int,
+) -> GuardResult:
+ """Run under the caller's inherited OpenClaw mutation-mutex description.
+
+ The caller must hold the mutex exclusively. Re-locking an inherited
+ descriptor succeeds while a separately opened one blocks, which is what
+ proves inheritance. A shared lock would pass without giving exclusion, so
+ do not reuse this path for read-only actions.
+ """
+
+ try:
+ opened = os.fstat(lock_fd)
+ current = os.stat(lock_path, follow_symlinks=False)
+ if (
+ not stat.S_ISREG(opened.st_mode)
+ or opened.st_nlink != 1
+ or not _same_entry(opened, current)
+ or opened.st_uid != identity.root_uid
+ or opened.st_gid != identity.root_gid
+ or stat.S_IMODE(opened.st_mode) != 0o600
+ or opened.st_size > MAX_TRANSITION_LOCK_BYTES
+ ):
+ raise GuardOperationError(
+ Issue(
+ "unsafe-transition-lock",
+ lock_path,
+ "inherited mutation mutex must be the private root-owned lock file",
+ )
+ )
+ try:
+ # An inherited descriptor shares the parent's flock ownership; a
+ # separately opened one blocks and cannot bypass serialization.
+ fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError as exc:
+ raise GuardOperationError(
+ Issue(
+ "transition-lock-not-inherited",
+ lock_path,
+ "mutation mutex descriptor does not share the caller's lock",
+ )
+ ) from exc
+ after = os.stat(lock_path, follow_symlinks=False)
+ if not _same_entry(opened, after):
+ raise GuardOperationError(
+ Issue(
+ "transition-lock-raced",
+ lock_path,
+ "transition mutex changed during inherited-lock verification",
+ )
+ )
+ return _run_guard_unserialized(action, config_dir, identity, plan)
+ except GuardOperationError as exc:
+ result = GuardResult(action=action)
+ result.issues.append(exc.issue)
+ return result
+ except OSError as exc:
+ result = GuardResult(action=action)
+ result.issues.append(
+ _os_issue(
+ "transition-lock-failed", lock_path, "verify inherited mutation mutex", exc
+ )
+ )
+ return result
+
+
def run_guard(
action: Action,
config_dir: str,
identity: Identity,
plan: AgentStateLockPlan,
+ *,
+ transition_lock_fd: int | None = None,
) -> GuardResult:
"""Serialize production OpenClaw recursive transitions with its top guard."""
normalized_config = posixpath.normpath(config_dir)
lock_path = _transition_lock_path(normalized_config)
+ if transition_lock_fd is not None:
+ if lock_path is None:
+ result = GuardResult(action=action)
+ result.issues.append(
+ Issue(
+ "unexpected-transition-lock-fd",
+ normalized_config,
+ "an inherited transition mutex is valid only for serialized OpenClaw state",
+ )
+ )
+ return result
+ return _run_guard_with_transition_lock_fd(
+ action,
+ normalized_config,
+ identity,
+ plan,
+ lock_path,
+ transition_lock_fd,
+ )
if lock_path is None:
return _run_guard_unserialized(action, normalized_config, identity, plan)
@@ -2445,6 +2538,7 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
plan_source = parser.add_mutually_exclusive_group()
plan_source.add_argument("--plan-json")
plan_source.add_argument("--plan-file")
+ parser.add_argument("--transition-lock-fd", type=int, help=argparse.SUPPRESS)
return parser.parse_args(argv)
@@ -2510,7 +2604,13 @@ def main(argv: list[str] | None = None) -> int:
)
)
else:
- result = run_guard(args.action, args.config_dir, identity, plan)
+ result = run_guard(
+ args.action,
+ args.config_dir,
+ identity,
+ plan,
+ transition_lock_fd=args.transition_lock_fd,
+ )
for issue in result.issues:
print(json.dumps(issue.as_json(), sort_keys=True, separators=(",", ":")))
diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts
index 5fb09fefe32..10309b1a1d0 100644
--- a/src/lib/shields/flow.test.ts
+++ b/src/lib/shields/flow.test.ts
@@ -1097,6 +1097,27 @@ describe("shields command flow", () => {
expect(output).toContain("Config remains unlocked — manual intervention required");
});
+ it("keeps the host attached beyond failed-startup recovery's container timeout (#8304)", () => {
+ const harness = createHarness({ failOpenClawGuardActions: ["preflight"] });
+
+ expect(() =>
+ harness.shieldsDown("openclaw", {
+ timeout: "5m",
+ reason: "failed-startup timeout coverage",
+ skipTimer: true,
+ throwOnError: true,
+ }),
+ ).not.toThrow();
+
+ const recovery = harness.dockerSpawnCalls.find(({ args }) =>
+ args.includes("unlock-failed-startup"),
+ );
+ expect(recovery?.args).toEqual(
+ expect.arrayContaining(["timeout", "--kill-after=5s", "25m", "unlock-failed-startup"]),
+ );
+ expect(recovery?.timeout).toBe(26 * 60 * 1000);
+ });
+
it("reports staged driver-neutral recovery when snapshot restoration fails (#6126)", () => {
const harness = createHarness({ run: () => ({ status: 1 }) });
const stateDir = path.join(tmpDir, ".nemoclaw", "state");
diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts
index fc239035411..8e7ff323e64 100644
--- a/src/lib/shields/index.ts
+++ b/src/lib/shields/index.ts
@@ -137,6 +137,9 @@ const HERMES_RESTART_SEAL_STATE = "/run/nemoclaw/hermes-restart-seal.json";
const HERMES_CONFIG_HASH = "/etc/nemoclaw/hermes.config-hash";
const STATE_DIR_GUARD_TIMEOUT_MS = 15 * 60 * 1000;
const OPENCLAW_CONFIG_GUARD_TIMEOUT_MS = 6 * 60 * 1000;
+// Exceeds the failed-startup guard's 25-minute in-container timeout and its
+// five-second termination grace, so the host never abandons a live recovery.
+const OPENCLAW_CONFIG_GUARD_RECOVERY_TIMEOUT_MS = 26 * 60 * 1000;
const HERMES_CONFIG_GUARD_TIMEOUT_MS = 11 * 60 * 1000;
type ShieldsDownTransition = {
@@ -1401,12 +1404,15 @@ function stateDirLockExec(sandboxName: string) {
function openClawConfigGuardExec(sandboxName: string) {
return {
run: (cmd: string[], input?: string) => {
+ const timeout = cmd.includes("unlock-failed-startup")
+ ? OPENCLAW_CONFIG_GUARD_RECOVERY_TIMEOUT_MS
+ : OPENCLAW_CONFIG_GUARD_TIMEOUT_MS;
const result = dockerSpawnSync(
privilegedSandboxExecArgv(sandboxName, cmd, input !== undefined, true),
{
encoding: "utf-8",
input,
- timeout: OPENCLAW_CONFIG_GUARD_TIMEOUT_MS,
+ timeout,
maxBuffer: 2 * 1024 * 1024,
},
);
@@ -1445,8 +1451,10 @@ function transitionOpenClawTopConfig(
assertCanonicalOpenClawConfigTarget(target);
const result = runOpenClawConfigGuard(openClawConfigGuardExec(sandboxName), action);
if (result.issues.length > 0) {
- throw new Error(
+ const issueCodes = result.issueCodes?.length === result.issues.length ? result.issueCodes : [];
+ throw new OpenClawConfigGuardFailure(
`Config not ${action === "unlock" ? "unlocked" : "locked"}: ${result.issues.join(", ")}`,
+ issueCodes,
);
}
if (result.resealedDrift) {
@@ -2122,6 +2130,118 @@ function assertNoLegacyStateLayout(sandboxName: string, configDir: string): void
// read_only) + chown/chmod below.
// ---------------------------------------------------------------------------
+class OpenClawConfigGuardFailure extends Error {
+ constructor(
+ message: string,
+ readonly issueCodes: readonly string[],
+ ) {
+ super(message);
+ this.name = "OpenClawConfigGuardFailure";
+ }
+}
+
+/** Whether a guard error reports the OpenClaw startup readiness lease. */
+function isOpenClawStartupNotReady(error: unknown): boolean {
+ return (
+ error instanceof OpenClawConfigGuardFailure &&
+ error.issueCodes.length === 1 &&
+ error.issueCodes[0] === "startup-not-ready"
+ );
+}
+
+/** The guard's refusal when the sandbox is simply not in a failed startup. */
+const FAILED_STARTUP_NOT_PROVEN = "failed-startup-not-proven";
+
+/**
+ * Lower shields on an OpenClaw sandbox whose startup terminally failed.
+ *
+ * Returns false when the sandbox is not in that state. The guard proves a
+ * stable supervisor with no startup process and no readiness marker, then
+ * unseals both layers in one mutex window.
+ */
+function recoverOpenClawFailedStartupShields(
+ sandboxName: string,
+ target: AgentConfigTarget,
+): boolean {
+ assertCanonicalOpenClawConfigTarget(target);
+ const result = runOpenClawConfigGuard(
+ openClawConfigGuardExec(sandboxName),
+ "unlock-failed-startup",
+ { planJson: JSON.stringify(requireStateLockPlan(target)) },
+ );
+ if (result.issues.length === 0) return true;
+ // Only "not a failed startup" falls back. A transition, rollback, contract,
+ // parse, or timeout failure must surface instead of being masked.
+ const notApplicable =
+ result.issueCodes?.length === result.issues.length &&
+ result.issueCodes.every((code) => code === FAILED_STARTUP_NOT_PROVEN);
+ if (notApplicable) return false;
+ throw new Error(`Failed-startup shields recovery failed: ${result.issues.join(", ")}`);
+}
+
+/** Independently observe the mutable OpenClaw posture after the guard returns. */
+function openClawMutablePostureIssues(sandboxName: string, target: AgentConfigTarget): string[] {
+ assertCanonicalOpenClawConfigTarget(target);
+ const issues: string[] = [];
+ for (const file of [target.configPath, ...(target.sensitiveFiles || [])]) {
+ try {
+ const perms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", file]);
+ const [mode, owner] = perms.split(" ");
+ if (mode !== "660") issues.push(`${file} mode=${mode} (expected 660)`);
+ if (owner !== "sandbox:sandbox") {
+ issues.push(`${file} owner=${owner} (expected sandbox:sandbox)`);
+ }
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ issues.push(`${file} stat failed: ${message}`);
+ }
+ try {
+ const attrs = privilegedSandboxExecCapture(sandboxName, ["lsattr", "-d", file]);
+ const [flags] = attrs.trim().split(/\s+/, 1);
+ if (flags.includes("i")) issues.push(`${file} immutable bit still set`);
+ } catch {
+ // Some supported images omit lsattr. Ownership and mode remain required.
+ }
+ }
+
+ try {
+ const perms = privilegedSandboxExecCapture(sandboxName, [
+ "stat",
+ "-c",
+ "%a %U:%G",
+ target.configDir,
+ ]);
+ const [mode, owner] = perms.split(" ");
+ if (mode !== "2770") issues.push(`config dir mode=${mode} (expected 2770)`);
+ if (owner !== "sandbox:sandbox") {
+ issues.push(`config dir owner=${owner} (expected sandbox:sandbox)`);
+ }
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ issues.push(`config dir stat failed: ${message}`);
+ }
+
+ if (requiresProtectedSandboxParent(target)) {
+ try {
+ const perms = privilegedSandboxExecCapture(sandboxName, [
+ "stat",
+ "-c",
+ "%a %U:%G",
+ "/sandbox",
+ ]);
+ const [mode, owner] = perms.split(" ");
+ if (mode !== "755") issues.push(`parent dir mode=${mode} (expected 755)`);
+ if (owner !== "sandbox:sandbox") {
+ issues.push(`parent dir owner=${owner} (expected sandbox:sandbox)`);
+ }
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ issues.push(`parent dir stat failed: ${message}`);
+ }
+ }
+ return issues;
+}
+
function unlockAgentConfigUnderMutationLock(
sandboxName: string,
rawTarget: AgentConfigTarget,
@@ -2161,7 +2281,20 @@ function unlockAgentConfigUnderMutationLock(
let openClawMutationStarted = false;
try {
if (openClawProtocol) {
- transitionOpenClawTopConfig(sandboxName, target, "preflight");
+ try {
+ transitionOpenClawTopConfig(sandboxName, target, "preflight");
+ } catch (preflightError) {
+ // Preflight is read-only, so nothing is mutated yet. Hand the whole
+ // unseal to the guard, which does it atomically (#8304).
+ if (!isOpenClawStartupNotReady(preflightError)) throw preflightError;
+ if (!recoverOpenClawFailedStartupShields(sandboxName, target)) throw preflightError;
+ const postureIssues = openClawMutablePostureIssues(sandboxName, target);
+ if (postureIssues.length > 0) {
+ throw new Error(`Config not unlocked: ${postureIssues.join(", ")}`);
+ }
+ console.log(" Lowered shields on a sandbox whose startup never completed.");
+ return;
+ }
}
if (target.agentName === "hermes" && !legacyHermesProtocol) {
transaction = beginHermesConfigShields(
@@ -2211,73 +2344,75 @@ function unlockAgentConfigUnderMutationLock(
transitionOpenClawTopConfig(sandboxName, target, "unlock");
}
- const issues: string[] = [];
- for (const f of filesToUnlock) {
- try {
- const perms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]);
- const [mode, owner] = perms.split(" ");
- if (mode !== fileMode) issues.push(`${f} mode=${mode} (expected ${fileMode})`);
- if (owner !== "sandbox:sandbox")
- issues.push(`${f} owner=${owner} (expected sandbox:sandbox)`);
- } catch (err) {
- const msg = err instanceof Error ? err.message : String(err);
- issues.push(`${f} stat failed: ${msg}`);
- }
- try {
- const attrs = privilegedSandboxExecCapture(sandboxName, ["lsattr", "-d", f]);
- const [flags] = attrs.trim().split(/\s+/, 1);
- if (flags.includes("i")) issues.push(`${f} immutable bit still set`);
- } catch {
- // lsattr may not be available on all images — skip
- }
- }
-
- try {
- const dirPerms = privilegedSandboxExecCapture(sandboxName, [
- "stat",
- "-c",
- "%a %U:%G",
- target.configDir,
- ]);
- const [mode, owner] = dirPerms.split(" ");
- // A 0700 Hermes root is provisional here. The token-bound guard finish
- // preserves it only for an attested same-UID topology, repairs and
- // verifies 03770 for a root-separated topology, and fails closed for an
- // unknown topology.
- const validDirMode =
- mode === dirMode ||
- (target.agentName === "hermes" && mode === "700" && transaction !== null);
- if (!validDirMode) {
- const expectedDirModes =
- target.agentName === "hermes" && transaction !== null
- ? `${dirMode}, or provisional 700 pending sealed guard topology attestation`
- : dirMode;
- issues.push(`config dir mode=${mode} (expected ${expectedDirModes})`);
- }
- if (owner !== "sandbox:sandbox") {
- issues.push(`config dir owner=${owner} (expected sandbox:sandbox)`);
+ const issues = openClawProtocol ? openClawMutablePostureIssues(sandboxName, target) : [];
+ if (!openClawProtocol) {
+ for (const f of filesToUnlock) {
+ try {
+ const perms = privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", f]);
+ const [mode, owner] = perms.split(" ");
+ if (mode !== fileMode) issues.push(`${f} mode=${mode} (expected ${fileMode})`);
+ if (owner !== "sandbox:sandbox")
+ issues.push(`${f} owner=${owner} (expected sandbox:sandbox)`);
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ issues.push(`${f} stat failed: ${msg}`);
+ }
+ try {
+ const attrs = privilegedSandboxExecCapture(sandboxName, ["lsattr", "-d", f]);
+ const [flags] = attrs.trim().split(/\s+/, 1);
+ if (flags.includes("i")) issues.push(`${f} immutable bit still set`);
+ } catch {
+ // lsattr may not be available on all images — skip
+ }
}
- } catch (err) {
- const msg = err instanceof Error ? err.message : String(err);
- issues.push(`config dir stat failed: ${msg}`);
- }
- if (requiresProtectedSandboxParent(target) && target.agentName !== "hermes") {
try {
- const parentPerms = privilegedSandboxExecCapture(sandboxName, [
+ const dirPerms = privilegedSandboxExecCapture(sandboxName, [
"stat",
"-c",
"%a %U:%G",
- "/sandbox",
+ target.configDir,
]);
- const [mode, owner] = parentPerms.split(" ");
- if (mode !== "755") issues.push(`parent dir mode=${mode} (expected 755)`);
+ const [mode, owner] = dirPerms.split(" ");
+ // A 0700 Hermes root is provisional here. The token-bound guard finish
+ // preserves it only for an attested same-UID topology, repairs and
+ // verifies 03770 for a root-separated topology, and fails closed for an
+ // unknown topology.
+ const validDirMode =
+ mode === dirMode ||
+ (target.agentName === "hermes" && mode === "700" && transaction !== null);
+ if (!validDirMode) {
+ const expectedDirModes =
+ target.agentName === "hermes" && transaction !== null
+ ? `${dirMode}, or provisional 700 pending sealed guard topology attestation`
+ : dirMode;
+ issues.push(`config dir mode=${mode} (expected ${expectedDirModes})`);
+ }
if (owner !== "sandbox:sandbox") {
- issues.push(`parent dir owner=${owner} (expected sandbox:sandbox)`);
+ issues.push(`config dir owner=${owner} (expected sandbox:sandbox)`);
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
- issues.push(`parent dir stat failed: ${msg}`);
+ issues.push(`config dir stat failed: ${msg}`);
+ }
+
+ if (requiresProtectedSandboxParent(target) && target.agentName !== "hermes") {
+ try {
+ const parentPerms = privilegedSandboxExecCapture(sandboxName, [
+ "stat",
+ "-c",
+ "%a %U:%G",
+ "/sandbox",
+ ]);
+ const [mode, owner] = parentPerms.split(" ");
+ if (mode !== "755") issues.push(`parent dir mode=${mode} (expected 755)`);
+ if (owner !== "sandbox:sandbox") {
+ issues.push(`parent dir owner=${owner} (expected sandbox:sandbox)`);
+ }
+ } catch (err) {
+ const msg = err instanceof Error ? err.message : String(err);
+ issues.push(`parent dir stat failed: ${msg}`);
+ }
}
}
diff --git a/src/lib/shields/openclaw-config-lock.test.ts b/src/lib/shields/openclaw-config-lock.test.ts
index 36cc6416c95..cb25af1c92a 100644
--- a/src/lib/shields/openclaw-config-lock.test.ts
+++ b/src/lib/shields/openclaw-config-lock.test.ts
@@ -277,7 +277,9 @@ describe("OpenClaw top-config guard host wiring", () => {
stderr: "",
};
- expect(parseOpenClawConfigGuardOutput("preflight", result).issues).toEqual(
+ const parsed = parseOpenClawConfigGuardOutput("preflight", result);
+ expect(parsed.issueCodes).toEqual(["hardlinked-config-file"]);
+ expect(parsed.issues).toEqual(
expect.arrayContaining([
expect.stringContaining("[hardlinked-config-file]"),
expect.stringContaining("reported failure with a zero exit"),
@@ -400,3 +402,60 @@ describe("OpenClaw top-config guard host wiring", () => {
expect(parseOpenClawConfigGuardOutput("lock", plain).resealedDrift).toBeUndefined();
});
});
+
+describe("OpenClaw config guard failed-startup recovery wiring (#8304)", () => {
+ it("accepts the recovery action's result record instead of discarding it", () => {
+ const { privileged } = createExec(true);
+
+ const result = runOpenClawConfigGuard(privileged, "unlock-failed-startup", {
+ planJson: '{"version":1}',
+ });
+
+ // A missing entry in the parser's action set turns a successful guard run
+ // into an "unknown record" issue, which silently disables the whole path.
+ expect(result.issues).toEqual([]);
+ });
+
+ it("outlasts the guard's own recursive fan-out budget and forwards the plan", () => {
+ const { calls, privileged } = createExec(true);
+
+ runOpenClawConfigGuard(privileged, "unlock-failed-startup", { planJson: '{"version":1}' });
+ const recovery = calls
+ .map(({ cmd }) => cmd)
+ .find((cmd) => cmd.includes("unlock-failed-startup"));
+
+ // The guard allows the state-dir fan-out and rollback 22m, so a 5m host timeout would
+ // kill it mid-unseal, past its rollback and its JSON error contract.
+ expect(recovery?.slice(0, 4)).toEqual(["timeout", "--signal=TERM", "--kill-after=5s", "25m"]);
+ const planIndex = recovery?.indexOf("--plan-json") ?? -1;
+ expect(planIndex).toBeGreaterThan(-1);
+ expect(recovery?.[planIndex + 1]).toBe('{"version":1}');
+ });
+
+ it("rejects a missing recovery plan before privileged execution", () => {
+ for (const planJson of [undefined, ""]) {
+ const { calls, privileged } = createExec(true);
+ const result = runOpenClawConfigGuard(privileged, "unlock-failed-startup", {
+ planJson,
+ });
+
+ expect(result).toEqual({
+ issues: ["OpenClaw config guard unlock-failed-startup requires planJson"],
+ chattrApplied: false,
+ });
+ expect(calls).toEqual([]);
+ }
+ });
+
+ it("refuses the recovery action when the sandbox has no installed guard", () => {
+ const { privileged } = createExec(false);
+
+ const result = runOpenClawConfigGuard(privileged, "unlock-failed-startup", {
+ planJson: '{"version":1}',
+ });
+
+ expect(result.issues).toEqual([
+ "OpenClaw config guard is absent in the sandbox; rebuild before recovering a failed startup",
+ ]);
+ });
+});
diff --git a/src/lib/shields/openclaw-config-lock.ts b/src/lib/shields/openclaw-config-lock.ts
index 91214bc2807..bc3e2958bf8 100644
--- a/src/lib/shields/openclaw-config-lock.ts
+++ b/src/lib/shields/openclaw-config-lock.ts
@@ -19,6 +19,11 @@ export const OPENCLAW_CONFIG_HASH_PATH = `${OPENCLAW_CONFIG_DIR}/.config-hash`;
const CONTAINER_HELPER = "/usr/local/lib/nemoclaw/openclaw-config-guard.py";
const HOST_HELPER = path.resolve(__dirname, "../../../scripts/openclaw-config-guard.py");
const CONTAINER_TIMEOUT = ["timeout", "--signal=TERM", "--kill-after=5s", "5m"];
+// Must exceed STATE_DIR_GUARD_TIMEOUT_SECONDS (22m) in
+// scripts/openclaw-config-guard.py, which is the guard's whole-action budget
+// for the unseal and its rollback together. The outer docker client timeout in
+// shields/index.ts must exceed this timeout plus its termination grace.
+const RECOVERY_CONTAINER_TIMEOUT = ["timeout", "--signal=TERM", "--kill-after=5s", "25m"];
const SCHEMA_VALIDATION_TIMEOUT = ["timeout", "--signal=TERM", "--kill-after=5s", "30s"];
const MAX_SCHEMA_CANDIDATE_BYTES = 16 * 1024 * 1024;
// OpenClaw resolves relative includes from the config file's directory.
@@ -41,12 +46,15 @@ export type OpenClawConfigGuardAction =
| "write-config"
| "recover"
| "revoke-startup-ready"
- | "publish-startup-ready";
+ | "publish-startup-ready"
+ | "unlock-failed-startup";
export type OpenClawConfigGuardOptions = {
expectedConfigSha256?: string;
input?: string;
startupOwner?: boolean;
+ /** Agent state lock plan, required by `unlock-failed-startup`. */
+ planJson?: string;
};
type GuardIssue = {
@@ -72,6 +80,7 @@ type GuardSummary = {
export type OpenClawConfigGuardResult = {
issues: string[];
+ issueCodes?: string[];
chattrApplied: boolean;
configSha256?: string;
hashSynthesized?: boolean;
@@ -91,6 +100,7 @@ const GUARD_ACTIONS = new Set([
"recover",
"revoke-startup-ready",
"publish-startup-ready",
+ "unlock-failed-startup",
]);
function executionFailure(label: string, result: PrivilegedExecResult): string {
@@ -308,6 +318,9 @@ export function parseOpenClawConfigGuardOutput(
),
...contractIssues,
],
+ ...(issues.length > 0
+ ? { issueCodes: issues.map((issue) => printableExcerpt(issue.code, 64)) }
+ : {}),
chattrApplied: summary?.status === "ok" && summary.chattrApplied === true,
...(summary?.status === "ok" && summary.configSha256
? { configSha256: summary.configSha256 }
@@ -351,13 +364,21 @@ export function runOpenClawConfigGuard(
chattrApplied: false,
};
}
+ if (action === "unlock-failed-startup" && !options.planJson) {
+ return {
+ issues: ["OpenClaw config guard unlock-failed-startup requires planJson"],
+ chattrApplied: false,
+ };
+ }
const capability = privileged.run(["test", "-r", CONTAINER_HELPER]);
+ const timeoutPrefix =
+ action === "unlock-failed-startup" ? RECOVERY_CONTAINER_TIMEOUT : CONTAINER_TIMEOUT;
let command: string[];
let input: string | undefined;
if (capability.status === 0 && capability.signal === null && !capability.error) {
command = [
- ...CONTAINER_TIMEOUT,
+ ...timeoutPrefix,
"python3",
"-I",
CONTAINER_HELPER,
@@ -378,6 +399,16 @@ export function runOpenClawConfigGuard(
chattrApplied: false,
};
}
+ if (action === "unlock-failed-startup") {
+ // Needs the in-image state guard and the installed helper. An injected
+ // copy satisfies neither, so refuse instead of half-running it.
+ return {
+ issues: [
+ "OpenClaw config guard is absent in the sandbox; rebuild before recovering a failed startup",
+ ],
+ chattrApplied: false,
+ };
+ }
try {
input = readHostHelper();
} catch (error) {
@@ -389,15 +420,7 @@ export function runOpenClawConfigGuard(
chattrApplied: false,
};
}
- command = [
- ...CONTAINER_TIMEOUT,
- "python3",
- "-I",
- "-",
- action,
- "--config-dir",
- OPENCLAW_CONFIG_DIR,
- ];
+ command = [...timeoutPrefix, "python3", "-I", "-", action, "--config-dir", OPENCLAW_CONFIG_DIR];
} else {
return {
issues: [executionFailure("OpenClaw config guard capability probe failed", capability)],
@@ -409,6 +432,7 @@ export function runOpenClawConfigGuard(
command.push("--expected-config-sha256", options.expectedConfigSha256);
}
if (options.startupOwner) command.push("--startup-owner");
+ if (options.planJson) command.push("--plan-json", options.planJson);
return parseOpenClawConfigGuardOutput(action, privileged.run(command, input));
}
diff --git a/src/lib/shields/openclaw-transition.test.ts b/src/lib/shields/openclaw-transition.test.ts
index faf6e9e6ebb..b8df96593a3 100644
--- a/src/lib/shields/openclaw-transition.test.ts
+++ b/src/lib/shields/openclaw-transition.test.ts
@@ -127,6 +127,34 @@ describe("OpenClaw shields top-config transaction", () => {
fs.rmSync(homeDir, { recursive: true, force: true });
});
+ function useMutablePosture() {
+ dockerExecSpy.mockImplementation((cmd) => {
+ const argv = cmd as string[];
+ switch (argv[0]) {
+ case "stat":
+ return argv.at(-1) === "/sandbox"
+ ? "755 sandbox:sandbox"
+ : argv.at(-1) === "/sandbox/.openclaw"
+ ? "2770 sandbox:sandbox"
+ : "660 sandbox:sandbox";
+ case "lsattr":
+ return `---------------- ${String(argv.at(-1))}`;
+ default:
+ return "";
+ }
+ });
+ }
+
+ function guardFailure(code: string, detail: string) {
+ return {
+ issues: [
+ `OpenClaw config guard preflight [${code}] /run/nemoclaw/openclaw-config-ready.json: ${detail}`,
+ ],
+ issueCodes: [code],
+ chattrApplied: false,
+ };
+ }
+
it("freezes the top-level binding before recursive lock and avoids pathname mutation", () => {
expect(() => shields.lockAgentConfig("openclaw", openClawTarget(), false)).not.toThrow();
@@ -219,27 +247,88 @@ describe("OpenClaw shields top-config transaction", () => {
});
it("keeps the protected top binding until recursive unlock is ready", () => {
- dockerExecSpy.mockImplementation((cmd) => {
- const argv = cmd as string[];
- switch (argv[0]) {
- case "stat":
- return argv.at(-1) === "/sandbox"
- ? "755 sandbox:sandbox"
- : argv.at(-1) === "/sandbox/.openclaw"
- ? "2770 sandbox:sandbox"
- : "660 sandbox:sandbox";
- case "lsattr":
- return `---------------- ${String(argv.at(-1))}`;
- default:
- return "";
- }
- });
+ useMutablePosture();
expect(() => shields.unlockAgentConfig("openclaw", openClawTarget(), true)).not.toThrow();
expect(events.slice(0, 3)).toEqual(["top:preflight", "state:unlock", "top:unlock"]);
});
+ it("uses structured readiness diagnostics and verifies recovered mutable posture (#8304)", () => {
+ useMutablePosture();
+ guardSpy.mockImplementation((_exec, action) => {
+ events.push(`top:${action}`);
+ return action === "preflight"
+ ? guardFailure("startup-not-ready", "startup lease is absent")
+ : { issues: [], chattrApplied: false };
+ });
+
+ expect(() => shields.unlockAgentConfig("openclaw", openClawTarget(), true)).not.toThrow();
+ expect(events).toEqual(["top:preflight", "top:unlock-failed-startup"]);
+ const commands = dockerExecSpy.mock.calls.map((call) => call[0] as string[]);
+ expect(commands.filter((cmd) => cmd[0] === "stat").map((cmd) => cmd.at(-1))).toEqual([
+ "/sandbox/.openclaw/openclaw.json",
+ "/sandbox/.openclaw/.config-hash",
+ "/sandbox/.openclaw",
+ "/sandbox",
+ ]);
+ });
+
+ it("does not recover when readiness is mixed with another guard failure (#8304)", () => {
+ guardSpy.mockImplementation((_exec, action) => {
+ events.push(`top:${action}`);
+ return action === "preflight"
+ ? {
+ issues: [
+ "OpenClaw config guard preflight [startup-not-ready] /run/nemoclaw/openclaw-config-ready.json: startup lease is absent",
+ "OpenClaw config guard preflight returned an invalid result contract",
+ ],
+ issueCodes: ["startup-not-ready"],
+ chattrApplied: false,
+ }
+ : { issues: [], chattrApplied: false };
+ });
+
+ expect(() => shields.unlockAgentConfig("openclaw", openClawTarget(), true)).toThrow(
+ /invalid result contract/,
+ );
+ expect(events).toEqual(["top:preflight"]);
+ });
+
+ it("falls back only for the distinct not-applicable recovery code (#8304)", () => {
+ guardSpy.mockImplementation((_exec, action) => {
+ events.push(`top:${action}`);
+ return action === "preflight"
+ ? guardFailure("startup-not-ready", "startup lease is absent")
+ : guardFailure(
+ "failed-startup-not-proven",
+ "unlock-failed-startup requires a proven terminal startup failure",
+ );
+ });
+
+ expect(() => shields.unlockAgentConfig("openclaw", openClawTarget(), true)).toThrow(
+ /\[startup-not-ready\].*startup lease is absent/,
+ );
+ expect(events).toEqual(["top:preflight", "top:unlock-failed-startup"]);
+ });
+
+ it("propagates lost failed-startup authorization instead of masking it (#8304)", () => {
+ guardSpy.mockImplementation((_exec, action) => {
+ events.push(`top:${action}`);
+ return action === "preflight"
+ ? guardFailure("startup-not-ready", "startup lease is absent")
+ : guardFailure(
+ "startup-not-ready",
+ "unlock-failed-startup lost its failed-startup authorization before taking effect",
+ );
+ });
+
+ expect(() => shields.unlockAgentConfig("openclaw", openClawTarget(), true)).toThrow(
+ /Failed-startup shields recovery failed.*lost its failed-startup authorization/,
+ );
+ expect(events).toEqual(["top:preflight", "top:unlock-failed-startup"]);
+ });
+
it("fails closed to the locked posture when recursive unlock is partial", () => {
applyStateSpy.mockImplementationOnce((_exec, _dir, _owner, locking) => {
events.push(`state:${locking ? "lock" : "unlock"}`);
diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts
index 4c4c495ac6b..83c18aec760 100644
--- a/test/e2e/live/shields-config.test.ts
+++ b/test/e2e/live/shields-config.test.ts
@@ -28,12 +28,19 @@ import {
import { expect, test } from "../fixtures/e2e-test.ts";
import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
import { REPO_ROOT } from "../fixtures/paths.ts";
+import { pollUntil } from "../fixtures/polling.ts";
import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
import { stripAnsi } from "./json-envelope.ts";
const CONFIG_PATH = "/sandbox/.openclaw/openclaw.json";
const CONFIG_DIR = path.dirname(CONFIG_PATH);
const CONFIG_HASH_PATH = `${CONFIG_DIR}/.config-hash`;
+const CONFIG_GUARD_PATH = "/usr/local/lib/nemoclaw/openclaw-config-guard.py";
+const STATE_LOCK_PLAN_PATH = "/usr/local/share/nemoclaw/state-lock-plan.json";
+const STARTUP_MARKER_PATHS = [
+ "/run/nemoclaw/openclaw-config-ready-v1.capability.json",
+ "/run/nemoclaw/openclaw-config-ready.json",
+] as const;
const AUDIT_FILE = path.join(os.homedir(), ".nemoclaw", "state", "shields-audit.jsonl");
const STATE_FILE = (sandboxName: string) =>
path.join(os.homedir(), ".nemoclaw", "state", `shields-${sandboxName}.json`);
@@ -257,6 +264,72 @@ async function findSandboxContainer(host: HostCliClient): Promise {
return containerId;
}
+type StartupCensus = { count: number; pid: number | null };
+
+async function installedStartupCensus(
+ host: HostCliClient,
+ containerId: string,
+ artifactName: string,
+): Promise {
+ const script = [
+ "import json, runpy",
+ `guard = runpy.run_path(${JSON.stringify(CONFIG_GUARD_PATH)})`,
+ "identity = guard['_production_identity']()",
+ "census = guard['_openshell_supervised_nonroot_start_census'](identity.root_uid, identity.sandbox_uid)",
+ "assert census is not None",
+ "print(json.dumps({'count': census[0], 'pid': census[1]}))",
+ ].join("\n");
+ const result = await docker(
+ host,
+ ["exec", "--user", "0", containerId, "python3", "-I", "-c", script],
+ { artifactName, timeoutMs: 30_000 },
+ );
+ expect(result.exitCode, resultText(result)).toBe(0);
+ return JSON.parse(result.stdout.trim()) as StartupCensus;
+}
+
+async function runInstalledFailedStartupUnlock(
+ host: HostCliClient,
+ containerId: string,
+ artifactName: string,
+): Promise {
+ const script = [
+ "set -eu",
+ `plan_json=$(cat ${STATE_LOCK_PLAN_PATH})`,
+ `exec timeout --signal=TERM --kill-after=5s 25m python3 -I ${CONFIG_GUARD_PATH} unlock-failed-startup --config-dir ${CONFIG_DIR} --plan-json "$plan_json"`,
+ ].join("\n");
+ return docker(host, ["exec", "--user", "0", containerId, "sh", "-c", script], {
+ artifactName,
+ timeoutMs: 26 * 60_000,
+ });
+}
+
+async function waitForChildlessStartup(
+ host: HostCliClient,
+ containerId: string,
+ startupPid: number,
+): Promise {
+ expect(Number.isSafeInteger(startupPid) && startupPid > 1).toBe(true);
+ const terminate = await docker(
+ host,
+ ["exec", "--user", "0", containerId, "kill", "-TERM", String(startupPid)],
+ { artifactName: "phase-12-terminate-startup-child", timeoutMs: 30_000 },
+ );
+ expect(
+ terminate.exitCode === 0 || /no such process/i.test(resultText(terminate)),
+ resultText(terminate),
+ ).toBe(true);
+
+ await pollUntil({
+ artifactPrefix: "phase-12-childless-census",
+ attempts: 20,
+ delayMs: 500,
+ probe: async (_attempt, artifactName) =>
+ await installedStartupCensus(host, containerId, artifactName),
+ accept: (census) => census.count === 0,
+ });
+}
+
async function readOriginalConfig(
host: HostCliClient,
containerId: string,
@@ -308,6 +381,7 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot
"restart OpenClaw with shields down",
"recover shields after a dead restore timer",
"reject duplicate shields transitions",
+ "prove installed failed-startup recovery refuses a live child and unlocks childless state",
"record shields contract evidence",
],
},
@@ -329,6 +403,7 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot
"start restores a stopped OpenClaw sandbox while shields are down",
"dead auto-restore timer inline recovery re-locks config and .config-hash",
"double shields-up/down operations are rejected",
+ "installed failed-startup recovery refuses a live supervised child and atomically unlocks childless state",
],
});
@@ -889,6 +964,77 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot
expect(finalUp.exitCode, resultText(finalUp)).toBe(0);
expect(resultText(finalUp)).toContain("Lockdown active");
+ progress.phase(
+ "prove installed failed-startup recovery refuses a live child and unlocks childless state",
+ );
+ const recoveryContainerId = await findSandboxContainer(host);
+ const removeMarkers = await docker(
+ host,
+ ["exec", "--user", "0", recoveryContainerId, "rm", "-f", ...STARTUP_MARKER_PATHS],
+ { artifactName: "phase-12-remove-startup-markers", timeoutMs: 30_000 },
+ );
+ expect(removeMarkers.exitCode, resultText(removeMarkers)).toBe(0);
+
+ const liveCensus = await installedStartupCensus(
+ host,
+ recoveryContainerId,
+ "phase-12-live-startup-census",
+ );
+ expect(liveCensus).toMatchObject({ count: 1, pid: expect.any(Number) });
+ expect(liveCensus.pid).not.toBeNull();
+ const liveStartupPid = liveCensus.pid ?? 0;
+ const liveChildRefusal = await runInstalledFailedStartupUnlock(
+ host,
+ recoveryContainerId,
+ "phase-12-live-child-refusal",
+ );
+ expect(liveChildRefusal.exitCode, resultText(liveChildRefusal)).not.toBe(0);
+ expect(resultText(liveChildRefusal)).toContain('"code": "startup-not-ready"');
+ expect(await statPath(sandbox, CONFIG_PATH, "phase-12-config-still-locked")).toMatchObject({
+ mode: "444",
+ owner: "root:root",
+ });
+
+ await waitForChildlessStartup(host, recoveryContainerId, liveStartupPid);
+ const childlessUnlock = await runInstalledFailedStartupUnlock(
+ host,
+ recoveryContainerId,
+ "phase-12-childless-unlock",
+ );
+ expect(childlessUnlock.exitCode, resultText(childlessUnlock)).toBe(0);
+ expect(resultText(childlessUnlock)).toContain('"action": "unlock-failed-startup"');
+ expect(resultText(childlessUnlock)).toContain('"status": "ok"');
+ expect(await statPath(sandbox, CONFIG_PATH, "phase-12-config-unlocked")).toMatchObject({
+ mode: "660",
+ owner: "sandbox:sandbox",
+ });
+ expect(
+ await statPath(sandbox, `${CONFIG_DIR}/workspace`, "phase-12-state-tree-unlocked"),
+ ).toMatchObject({ mode: "2770", owner: "sandbox:sandbox" });
+
+ // Reconcile the host-side Shields receipt after the direct installed-guard
+ // proof, then restart the failed sandbox and return cleanup to lockdown.
+ const reconcileDown = await runNemoclaw(
+ host,
+ [
+ SANDBOX_NAME,
+ "shields",
+ "down",
+ "--timeout",
+ "5m",
+ "--reason",
+ "Installed failed-startup recovery E2E",
+ ],
+ { artifactName: "phase-12-reconcile-shields-down", timeoutMs: 16 * 60_000 },
+ );
+ expect(reconcileDown.exitCode, resultText(reconcileDown)).toBe(0);
+ await expectStopStartRecovery(host, "DOWN", "phase-12-restart-after-recovery");
+ const relockAfterRecovery = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], {
+ artifactName: "phase-12-relock-after-recovery",
+ });
+ expect(relockAfterRecovery.exitCode, resultText(relockAfterRecovery)).toBe(0);
+ expect(resultText(relockAfterRecovery)).toContain("Lockdown active");
+
progress.phase("record shields contract evidence");
await artifacts.target.complete({
id: "shields-config",
@@ -905,6 +1051,9 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot
auditTrail: true,
deadTimerInlineAutoRestore: true,
doubleOperationRejection: true,
+ installedFailedStartupLiveChildRefusal: true,
+ installedFailedStartupChildlessUnlock: true,
+ inheritedMutationLockAcceptedByStateGuard: true,
},
});
});
diff --git a/test/helpers/shields-flow-harness.ts b/test/helpers/shields-flow-harness.ts
index a86037628d8..0278f76635c 100644
--- a/test/helpers/shields-flow-harness.ts
+++ b/test/helpers/shields-flow-harness.ts
@@ -15,6 +15,7 @@ export type ShieldsFlowHarness = {
applyShieldsPolicySnapshot: typeof import("../../src/lib/shields/index.js").applyShieldsPolicySnapshot;
auditSpy: MockInstance;
cleanupTempDirSpy: MockInstance;
+ dockerSpawnCalls: Array<{ args: string[]; timeout: number | undefined }>;
errorSpy: MockInstance;
getShieldsPosture: typeof import("../../src/lib/shields/index.js").getShieldsPosture;
getOpenClawPosture: () => "locked" | "mutable";
@@ -34,7 +35,7 @@ export type ShieldsFlowHarnessOptions = {
confirmOpenClawInodeFlags?: boolean;
directSandboxUnavailable?: boolean;
dockerExecFileSync?: (argv: unknown) => string;
- failOpenClawGuardActions?: Array<"lock" | "unlock">;
+ failOpenClawGuardActions?: Array<"preflight" | "lock" | "unlock">;
failPolicyRejectionStateClear?: boolean;
failPolicyRejectionTransitionWrite?: boolean;
failStateSave?: boolean;
@@ -221,66 +222,76 @@ export function createShieldsFlowHarness(
...(Array.isArray(cmd) ? cmd.map(String) : []),
],
);
- vi.spyOn(dockerExec, "dockerSpawnSync").mockImplementation((argv: unknown) => {
- const args = Array.isArray(argv) ? argv.map(String) : [];
- const readsStateLockPlan =
- args.includes("cat") && args.includes("/usr/local/share/nemoclaw/state-lock-plan.json");
- const action = ["preflight", "lock", "unlock"].find((candidate) => args.includes(candidate));
- const openClawGuard = args.some((arg) => arg.endsWith("openclaw-config-guard.py"));
- const shouldFailOpenClawGuard = Boolean(
- openClawGuard &&
- (action === "lock" || action === "unlock") &&
- options.failOpenClawGuardActions?.includes(action),
- );
- const failures = options.openClawGuardFailures ?? [
- options.openClawGuardFailure ?? {
- code: "startup-not-ready",
- path: "/run/nemoclaw/openclaw-config-ready.json",
- detail: "OpenClaw startup is not ready for host config mutations",
- },
- ];
- const failureResult = {
- status: 1,
- signal: null,
- stdout: `${failures
- .map((failure) => JSON.stringify({ type: "issue", ...failure }))
- .join("\n")}\n${JSON.stringify({ type: "result", action, status: "failed" })}\n`,
- stderr: "",
- pid: 0,
- output: [],
- };
- openClawPosture = shouldFailOpenClawGuard
- ? openClawPosture
- : openClawGuard && action === "lock"
- ? "locked"
- : openClawGuard && action === "unlock"
- ? "mutable"
- : openClawPosture;
- const successResult = {
- status: 0,
- signal: null,
- stdout: readsStateLockPlan
- ? `${JSON.stringify(stateLockPlan)}\n`
- : action
- ? `${JSON.stringify({
- type: "result",
- action,
- status: "ok",
- ...(openClawGuard
- ? {
- configDir: "/sandbox/.openclaw",
- files: ["openclaw.json", ".config-hash"],
- chattrApplied: action === "lock",
- }
- : { issueCount: 0 }),
- })}\n`
- : "",
- stderr: "",
- pid: 0,
- output: [],
- };
- return (shouldFailOpenClawGuard ? failureResult : successResult) as never;
- });
+ const dockerSpawnCalls: Array<{ args: string[]; timeout: number | undefined }> = [];
+ vi.spyOn(dockerExec, "dockerSpawnSync").mockImplementation(
+ (argv: unknown, rawOptions: unknown) => {
+ const args = Array.isArray(argv) ? argv.map(String) : [];
+ const timeout =
+ rawOptions && typeof rawOptions === "object" && "timeout" in rawOptions
+ ? Number((rawOptions as { timeout?: unknown }).timeout)
+ : undefined;
+ dockerSpawnCalls.push({ args, timeout });
+ const readsStateLockPlan =
+ args.includes("cat") && args.includes("/usr/local/share/nemoclaw/state-lock-plan.json");
+ const action = ["preflight", "lock", "unlock", "unlock-failed-startup"].find((candidate) =>
+ args.includes(candidate),
+ );
+ const openClawGuard = args.some((arg) => arg.endsWith("openclaw-config-guard.py"));
+ const shouldFailOpenClawGuard = Boolean(
+ openClawGuard &&
+ (action === "preflight" || action === "lock" || action === "unlock") &&
+ options.failOpenClawGuardActions?.includes(action),
+ );
+ const failures = options.openClawGuardFailures ?? [
+ options.openClawGuardFailure ?? {
+ code: "startup-not-ready",
+ path: "/run/nemoclaw/openclaw-config-ready.json",
+ detail: "OpenClaw startup is not ready for host config mutations",
+ },
+ ];
+ const failureResult = {
+ status: 1,
+ signal: null,
+ stdout: `${failures
+ .map((failure) => JSON.stringify({ type: "issue", ...failure }))
+ .join("\n")}\n${JSON.stringify({ type: "result", action, status: "failed" })}\n`,
+ stderr: "",
+ pid: 0,
+ output: [],
+ };
+ openClawPosture = shouldFailOpenClawGuard
+ ? openClawPosture
+ : openClawGuard && action === "lock"
+ ? "locked"
+ : openClawGuard && (action === "unlock" || action === "unlock-failed-startup")
+ ? "mutable"
+ : openClawPosture;
+ const successResult = {
+ status: 0,
+ signal: null,
+ stdout: readsStateLockPlan
+ ? `${JSON.stringify(stateLockPlan)}\n`
+ : action
+ ? `${JSON.stringify({
+ type: "result",
+ action,
+ status: "ok",
+ ...(openClawGuard
+ ? {
+ configDir: "/sandbox/.openclaw",
+ files: ["openclaw.json", ".config-hash"],
+ chattrApplied: action === "lock",
+ }
+ : { issueCount: 0 }),
+ })}\n`
+ : "",
+ stderr: "",
+ pid: 0,
+ output: [],
+ };
+ return (shouldFailOpenClawGuard ? failureResult : successResult) as never;
+ },
+ );
vi.spyOn(dockerExec, "dockerExecFileSync").mockImplementation((argv: unknown) => {
const args = Array.isArray(argv) ? argv.map(String) : [];
return options.dockerExecFileSync
@@ -376,6 +387,7 @@ export function createShieldsFlowHarness(
applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot,
auditSpy,
cleanupTempDirSpy,
+ dockerSpawnCalls,
errorSpy,
getShieldsPosture: shields.getShieldsPosture,
getOpenClawPosture: () => openClawPosture,
diff --git a/test/openclaw-config-guard-startup-failure-gate.test.ts b/test/openclaw-config-guard-startup-failure-gate.test.ts
new file mode 100644
index 00000000000..cb64c3aeb97
--- /dev/null
+++ b/test/openclaw-config-guard-startup-failure-gate.test.ts
@@ -0,0 +1,461 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+
+const GUARD_PATH = path.resolve("scripts/openclaw-config-guard.py");
+const STATE_GUARD_PATH = path.resolve("scripts/state-dir-guard.py");
+const PYTHON = process.platform === "win32" ? "python" : "python3";
+
+const HARNESS = String.raw`
+import importlib.util
+import json
+import os
+import sys
+import tempfile
+import typing
+
+spec = importlib.util.spec_from_file_location("guard", sys.argv[1])
+guard = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = guard
+spec.loader.exec_module(guard)
+
+identity = guard.Identity(root_uid=0, root_gid=0, sandbox_uid=1000, sandbox_gid=1000)
+guard.INSTALLED_HELPER_PATH = guard.__file__
+guard._pid1_is_nemoclaw_start = lambda: False
+guard._startup_lease_state = lambda _identity: (False, False)
+nul = bytes([0])
+start_cmdline = b"bash" + nul + b"/usr/local/bin/nemoclaw-start" + nul
+supervisor_cmdline = b"/opt/openshell/bin/openshell-sandbox" + nul
+
+def write_process(proc_root, pid, cmdline, namespace_path, uid, parent_pid):
+ process_dir = os.path.join(proc_root, str(pid))
+ os.makedirs(os.path.join(process_dir, "ns"))
+ fields = ["S", str(parent_pid)] + (["0"] * 17) + ["424242"]
+ with open(os.path.join(process_dir, "stat"), "w", encoding="ascii") as stream:
+ stream.write(f"{pid} (nemoclaw) {' '.join(fields)}\n")
+ with open(os.path.join(process_dir, "cmdline"), "wb") as stream:
+ stream.write(cmdline)
+ with open(os.path.join(process_dir, "status"), "w", encoding="ascii") as stream:
+ stream.write(
+ f"Uid:\t{uid}\t{uid}\t{uid}\t{uid}\n"
+ f"NSpid:\t{pid}\t{pid}\n"
+ )
+ os.link(namespace_path, os.path.join(process_dir, "ns", "pid"))
+
+def with_proc(children, markers_absent, supervisor, limit):
+ root = tempfile.mkdtemp()
+ proc_root = os.path.join(root, "proc")
+ os.mkdir(proc_root)
+ namespace_path = os.path.join(root, "shared")
+ with open(namespace_path, "wb") as stream:
+ stream.write(b"shared")
+ write_process(proc_root, 1, supervisor, namespace_path, 0, 0)
+ for pid in children:
+ write_process(proc_root, pid, start_cmdline, namespace_path, 1000, 1)
+ guard.PROC_ROOT = proc_root
+ guard.MAX_PROC_ENTRIES = limit
+ guard._startup_markers_absent = lambda _identity: markers_absent
+
+def gate(action, children, markers_absent=True, supervisor=supervisor_cmdline, limit=32768):
+ with_proc(children, markers_absent, supervisor, limit)
+ try:
+ guard._validate_action_readiness(action, False, identity)
+ return "allowed"
+ except guard.GuardError as error:
+ return error.code
+
+def provisional(action, children):
+ with_proc(children, True, supervisor_cmdline, 32768)
+ try:
+ return bool(guard._validate_action_readiness(action, False, identity))
+ except guard.GuardError:
+ return "refused"
+
+def reconfirm(children, markers_absent=True):
+ with_proc(children, markers_absent, supervisor_cmdline, 32768)
+ try:
+ guard._reconfirm_startup_failure_recovery("unlock", identity)
+ return "ok"
+ except guard.GuardError as error:
+ return error.code
+
+def cli_accepts(action):
+ parser = guard._parser()
+ choices = next(a.choices for a in parser._actions if a.dest == "action")
+ return action in set(choices) and set(choices) == set(typing.get_args(guard.Action))
+
+print(json.dumps({
+ # The CLI choices tuple is separate from the Action type, so an action can
+ # exist in code and still be unreachable through the entry point.
+ "cli_exposes_recovery": cli_accepts("unlock-failed-startup"),
+ "failed_recovery": gate("unlock-failed-startup", []),
+ "failed_preflight": gate("preflight", []),
+ "failed_unlock": gate("unlock", []),
+ "failed_lock": gate("lock", []),
+ "failed_write": gate("write-config", []),
+ "failed_seal": gate("seal-restart", []),
+ "failed_recover": gate("recover", []),
+ "live_lock": gate("lock", [412]),
+ "duplicate_unlock": gate("unlock-failed-startup", [412, 413]),
+ "foreign_unlock": gate("unlock-failed-startup", [], supervisor=b"/usr/bin/foreign" + nul),
+ "stale_marker_unlock": gate("unlock-failed-startup", [], markers_absent=False),
+ "bounded_scan_unlock": gate("unlock-failed-startup", [], limit=0),
+ "provisional_failed_recovery": provisional("unlock-failed-startup", []),
+ "provisional_live_lock": provisional("lock", [412]),
+ "live_recovery": gate("unlock-failed-startup", [412]),
+ "reconfirm_still_childless": reconfirm([]),
+ "reconfirm_child_appeared": reconfirm([412]),
+ "reconfirm_marker_appeared": reconfirm([], markers_absent=False),
+}))
+`;
+
+const TRANSACTION_HARNESS = String.raw`
+import importlib.util
+import json
+import subprocess
+import time
+import sys
+import types
+
+if sys.platform == "win32":
+ for name in ("fcntl", "grp", "pwd"):
+ sys.modules[name] = types.ModuleType(name)
+
+spec = importlib.util.spec_from_file_location("guard", sys.argv[1])
+guard = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = guard
+spec.loader.exec_module(guard)
+
+guard.os.path.isfile = lambda _path: True
+run_call = {}
+def timeout_run(command, **kwargs):
+ run_call["command"] = command
+ run_call["pass_fds"] = kwargs.get("pass_fds")
+ raise subprocess.TimeoutExpired("state-dir-guard", guard.STATE_DIR_GUARD_TIMEOUT_SECONDS)
+
+guard.subprocess.run = timeout_run
+try:
+ guard._run_state_dir_guard(
+ "unlock",
+ guard.PRODUCTION_CONFIG_DIR,
+ "{}",
+ 91,
+ time.monotonic() + guard.STATE_DIR_GUARD_TIMEOUT_SECONDS,
+ )
+except guard.GuardError as error:
+ timeout_code = error.code
+
+freeze_flags = []
+def capture_freeze(_opened, _identity, **kwargs):
+ freeze_flags.append(kwargs.get("quarantine_reserved"))
+ raise guard.GuardError("injected-freeze-stop", guard.PRODUCTION_CONFIG_DIR, "stop")
+
+guard._freeze = capture_freeze
+guard._has_clamped_locked_dir_posture = lambda _opened, _identity: False
+guard._has_locked_dir_posture = lambda _opened, _identity: False
+guard._force_fail_closed_lock = lambda _opened, _identity: []
+try:
+ guard._transition("lock", object(), object(), quarantine_untrusted=True)
+except guard.GuardError as error:
+ assert error.code == "injected-freeze-stop"
+
+guard._is_mutable_dir_posture = lambda _opened, _identity: False
+guard._snapshot_pair = lambda _opened: (object(), object())
+guard._verify_locked_posture = lambda *_args, **_kwargs: None
+guard._restore_originals = lambda _opened, _snapshots, _identity: []
+try:
+ guard._transition("unlock", object(), object(), quarantine_untrusted=True)
+except guard.GuardError as error:
+ assert error.code == "injected-freeze-stop"
+
+events = []
+transaction_deadlines = {}
+def transition(action, _opened, _identity, **kwargs):
+ events.append(f"config-{action}-quarantine-{kwargs.get('quarantine_untrusted')}")
+ if action == "unlock":
+ raise guard.MutableHandoffError(
+ "mutable-handoff-incomplete", guard.PRODUCTION_CONFIG_DIR, "handoff failed"
+ )
+
+def state_dir(action, _config_dir, _plan_json, lock_fd, deadline):
+ assert lock_fd == 91
+ transaction_deadlines[action] = deadline
+ events.append(f"state-{action}")
+ if action == "lock":
+ raise guard.GuardError("state-lock-failed", guard.PRODUCTION_CONFIG_DIR, "lock failed")
+
+guard._transition = transition
+guard._run_state_dir_guard = state_dir
+try:
+ guard._run_failed_startup_unlock(
+ object(), object(), guard.PRODUCTION_CONFIG_DIR, "{}", 91, quarantine_untrusted=True
+ )
+except guard.GuardError as error:
+ transaction_error = {
+ "code": error.code,
+ "detail": error.detail,
+ }
+
+timeout_events = []
+timeout_deadlines = {}
+timeout_rollback_remaining = None
+def timeout_transition(action, _opened, _identity, **kwargs):
+ timeout_events.append(f"config-{action}-quarantine-{kwargs.get('quarantine_untrusted')}")
+
+def timeout_state_dir(action, _config_dir, _plan_json, lock_fd, deadline):
+ global timeout_rollback_remaining
+ assert lock_fd == 91
+ if deadline - guard.time.monotonic() <= 0:
+ raise guard.GuardError(
+ "state-dir-transition-timeout", guard.PRODUCTION_CONFIG_DIR,
+ f"no recovery budget left for state-dir {action}",
+ )
+ timeout_deadlines[action] = deadline
+ timeout_events.append(f"state-{action}")
+ if action == "unlock":
+ # Model the forward transition consuming its entire allowance. A
+ # shared deadline would make the following relock refuse to start.
+ guard.time.monotonic = lambda: deadline
+ raise guard.GuardError(
+ "state-dir-transition-timeout", guard.PRODUCTION_CONFIG_DIR, "unlock timed out"
+ )
+ # Model a relock that consumes nearly the state guard's ten-minute
+ # maximum. The remaining allowance covers config relock overhead.
+ guard.time.monotonic = lambda: deadline - (2 * 60 + 1)
+ timeout_rollback_remaining = deadline - guard.time.monotonic()
+
+guard._transition = timeout_transition
+guard._run_state_dir_guard = timeout_state_dir
+try:
+ guard._run_failed_startup_unlock(
+ object(), object(), guard.PRODUCTION_CONFIG_DIR, "{}", 91, quarantine_untrusted=True
+ )
+except guard.GuardError as error:
+ timeout_transaction_code = error.code
+
+print(json.dumps({
+ "timeout_code": timeout_code,
+ "lock_fd_flag": run_call["command"][-2:],
+ "pass_fds": run_call["pass_fds"],
+ "freeze_flags": freeze_flags,
+ "events": events,
+ "rollback_reserve": transaction_deadlines["lock"] - transaction_deadlines["unlock"],
+ "transaction_error": transaction_error,
+ "timeout_events": timeout_events,
+ "timeout_rollback_reserve": timeout_deadlines["lock"] - timeout_deadlines["unlock"],
+ "timeout_rollback_remaining": timeout_rollback_remaining,
+ "timeout_transaction_code": timeout_transaction_code,
+}))
+`;
+
+const LOCK_HANDOFF_HARNESS = String.raw`
+import fcntl
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+guard_path = sys.argv[1]
+# macOS exposes /var as a symlink to /private/var. Resolve the fixture root so
+# descriptor-safe no-follow traversal tests the inherited lock rather than
+# rejecting the host's symlinked temporary-directory prefix.
+root = os.path.realpath(tempfile.mkdtemp())
+config_dir = os.path.join(root, ".openclaw")
+lock_path = os.path.join(root, ".openclaw-config-mutation.lock")
+os.mkdir(config_dir)
+owner_fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600)
+foreign_fd = os.open(lock_path, os.O_RDWR)
+fcntl.flock(owner_fd, fcntl.LOCK_EX)
+plan_json = json.dumps({
+ "version": 1,
+ "readOnlyRoots": [],
+ "confidentialRoots": [],
+ "readOnlyPrefixes": [],
+ "confidentialPrefixes": [],
+ "writableSubpaths": [],
+})
+child = r'''
+import importlib.util
+import json
+import os
+import sys
+
+guard_path, config_dir, plan_json, lock_fd = sys.argv[1:5]
+spec = importlib.util.spec_from_file_location("state_guard", guard_path)
+guard = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = guard
+spec.loader.exec_module(guard)
+identity = guard.Identity(
+ root_uid=os.getuid(), root_gid=os.getgid(),
+ sandbox_uid=os.getuid(), sandbox_gid=os.getgid(),
+)
+guard.os.geteuid = lambda: 0
+guard._production_identity = lambda: identity
+raise SystemExit(guard.main([
+ "unlock", "--config-dir", config_dir, "--plan-json", plan_json,
+ "--transition-lock-fd", lock_fd,
+]))
+'''
+env = {**os.environ, "NEMOCLAW_TEST_OPENCLAW_TRANSACTION_LOCK": "1"}
+
+def invoke(lock_fd):
+ return subprocess.run(
+ [sys.executable, "-c", child, guard_path, config_dir, plan_json, str(lock_fd)],
+ capture_output=True,
+ text=True,
+ timeout=5,
+ env=env,
+ pass_fds=(lock_fd,),
+ check=False,
+ )
+
+inherited = invoke(owner_fd)
+foreign = invoke(foreign_fd)
+print(json.dumps({
+ "inherited_status": inherited.returncode,
+ "inherited_records": [json.loads(line) for line in inherited.stdout.splitlines()],
+ "foreign_status": foreign.returncode,
+ "foreign_records": [json.loads(line) for line in foreign.stdout.splitlines()],
+}))
+`;
+
+const NOT_APPLICABLE_HARNESS = String.raw`
+import importlib.util
+import json
+import sys
+
+spec = importlib.util.spec_from_file_location("guard", sys.argv[1])
+guard = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = guard
+spec.loader.exec_module(guard)
+guard.os.geteuid = lambda: 0
+guard._production_identity = lambda: object()
+guard._validate_action_readiness = lambda *_args, **_kwargs: False
+status = guard.main([
+ "unlock-failed-startup",
+ "--config-dir", guard.PRODUCTION_CONFIG_DIR,
+ "--plan-json", "{}",
+])
+print(json.dumps({"status": status}))
+`;
+
+describe("OpenClaw failed-startup unlock transaction (#8304)", () => {
+ it("relocks both state layers after a config handoff failure", () => {
+ const result = spawnSync(PYTHON, ["-c", TRANSACTION_HARNESS, GUARD_PATH], {
+ encoding: "utf-8",
+ timeout: 10000,
+ });
+
+ expect(result.status, result.stderr).toBe(0);
+ expect(JSON.parse(result.stdout)).toEqual({
+ timeout_code: "state-dir-transition-timeout",
+ lock_fd_flag: ["--transition-lock-fd", "91"],
+ pass_fds: [91],
+ freeze_flags: [true, true],
+ events: [
+ "state-unlock",
+ "config-unlock-quarantine-True",
+ "config-lock-quarantine-True",
+ "state-lock",
+ ],
+ rollback_reserve: 720,
+ transaction_error: {
+ code: "mutable-handoff-incomplete",
+ detail: "handoff failed; rollback issues: state-dir lock: lock failed",
+ },
+ timeout_events: ["state-unlock", "config-lock-quarantine-True", "state-lock"],
+ timeout_rollback_reserve: 720,
+ timeout_rollback_remaining: 121,
+ timeout_transaction_code: "state-dir-transition-timeout",
+ });
+ });
+});
+
+describe("OpenClaw failed-startup host classification (#8304)", () => {
+ it("emits a distinct machine code when recovery is not applicable", () => {
+ const result = spawnSync(PYTHON, ["-c", NOT_APPLICABLE_HARNESS, GUARD_PATH], {
+ encoding: "utf-8",
+ timeout: 10000,
+ });
+
+ expect(result.status, result.stderr).toBe(0);
+ const records = result.stdout
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ expect(records).toContainEqual(
+ expect.objectContaining({
+ type: "issue",
+ code: "failed-startup-not-proven",
+ }),
+ );
+ expect(records).toContainEqual({ status: 1 });
+ });
+});
+
+describe.skipIf(process.platform === "win32")(
+ "OpenClaw config guard startup-failure gate (#8304)",
+ () => {
+ it("shares the held mutation lock with the recursive state guard", () => {
+ const result = spawnSync(PYTHON, ["-c", LOCK_HANDOFF_HARNESS, STATE_GUARD_PATH], {
+ encoding: "utf-8",
+ timeout: 10000,
+ });
+
+ expect(result.status, result.stderr).toBe(0);
+ const outcome = JSON.parse(result.stdout);
+ expect(outcome.inherited_status, JSON.stringify(outcome)).toBe(0);
+ expect(outcome.inherited_records).toContainEqual(
+ expect.objectContaining({ type: "result", action: "unlock", status: "ok" }),
+ );
+ expect(outcome.foreign_status).toBe(1);
+ expect(outcome.foreign_records).toContainEqual(
+ expect.objectContaining({ type: "issue", code: "transition-lock-not-inherited" }),
+ );
+ });
+
+ it("combines the real process census with the action gate", () => {
+ const result = spawnSync(PYTHON, ["-c", HARNESS, GUARD_PATH], {
+ encoding: "utf-8",
+ timeout: 10000,
+ });
+
+ expect(result.status, result.stderr).toBe(0);
+ expect(JSON.parse(result.stdout)).toEqual({
+ cli_exposes_recovery: true,
+ // Only the dedicated atomic action is reachable through the escape.
+ failed_recovery: "allowed",
+ // The multi-step host sequence stays refused, so its first step fails
+ // closed and no recursive state is mutated on stale evidence.
+ failed_preflight: "startup-not-ready",
+ failed_unlock: "startup-not-ready",
+ failed_lock: "startup-not-ready",
+ failed_write: "startup-not-ready",
+ failed_seal: "startup-not-ready",
+ failed_recover: "startup-not-ready",
+ live_lock: "allowed",
+ duplicate_unlock: "startup-not-ready",
+ foreign_unlock: "startup-not-ready",
+ stale_marker_unlock: "startup-not-ready",
+ // A bounded-scan overflow makes the census undeterminable. Both
+ // predicates must map that onto False so neither path authenticates.
+ bounded_scan_unlock: "startup-not-ready",
+ // Only the failed-startup path reports a provisional authorization, so
+ // the mutex-held reconfirm runs for that path and nothing else.
+ provisional_failed_recovery: true,
+ provisional_live_lock: false,
+ // A healthy sandbox must never reach the recovery action.
+ live_recovery: "startup-not-ready",
+ // The reconfirm is what binds the census to the effect: a start child
+ // or a marker appearing after the pre-mutex scan revokes the escape.
+ reconfirm_still_childless: "ok",
+ reconfirm_child_appeared: "startup-not-ready",
+ reconfirm_marker_appeared: "startup-not-ready",
+ });
+ });
+ },
+);