From 801c0838fdb19823b6d215f6ad15522ee7c3c007 Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Sun, 23 Aug 2026 06:25:56 -0400 Subject: [PATCH 01/12] fix(bin): require project clone roots during fleet sync (#2849) * fix(bin): require a clone root before fleet-sync touches a project Git repository discovery walks upward, so `git -C projects/` on a plain directory nested under projects/ resolves to the enclosing repository - in a firstmate home, the firstmate checkout itself. fm-fleet-sync.sh guarded its candidates with `rev-parse --is-inside-work-tree`, which such a directory passes, so every later git call read, pruned and fast-forwarded firstmate's own default branch and reported it under the project directory's label. A running session's AGENTS.md changed underneath it, and the report named a project that had nothing to do with the change. Require each candidate to be the root of its own work tree before any other git command: compare `rev-parse --show-toplevel` against the directory's own physical path. Both sides are physical, so a symlinked clone still compares equal. Anything else is skipped by name, naming the repository that would have been touched, and bootstrap relays that as a FLEET_SYNC line. Regression coverage reproduces the wrong-repo fast-forward against a home nested inside another repository, in both the whole-fleet and single-project forms, and pins that a symlinked clone dir still syncs. * no-mistakes(review): Keep enclosing fixture clean during clone-root regression --- bin/fm-fleet-sync.sh | 22 ++++++++- tests/fm-fleet-sync.test.sh | 94 +++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/bin/fm-fleet-sync.sh b/bin/fm-fleet-sync.sh index d5c951e1a74..dd00be86baa 100755 --- a/bin/fm-fleet-sync.sh +++ b/bin/fm-fleet-sync.sh @@ -13,6 +13,11 @@ # stashed, or discarded. # Still skips (benignly) local-only/no-origin projects, missing remotes/branches, # and fetch failures. +# A candidate under projects/ must be the root of its own work tree: git discovery +# walks up, so a plain nested directory would otherwise resolve to the enclosing +# repository (the firstmate checkout) and be synced under that directory's label. +# Anything else is reported as "skipped: not a clone root" naming the repository +# that would have been touched. # Pruning never deletes the checked-out branch or a branch that still has a # worktree, so it cannot discard unlanded work; set FM_FLEET_PRUNE=0 to disable it. # When the fetch fails on an orphaned .git/packed-refs.lock (left by a ref rewrite @@ -300,10 +305,25 @@ sync_project() { echo "$label: skipped: not a directory" return 0 fi - if ! git -C "$PROJ" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + # Git repository discovery walks UP from $PROJ, so a plain directory merely + # nested inside a repository - a worktree container left under projects/, say - + # resolves to the ENCLOSING repository, which in a firstmate home is the + # firstmate checkout itself. Every later `git -C "$PROJ"` would then read, prune + # and fast-forward that repository under this project's label, turning a routine + # refresh into an unrequested self-update reported as a project sync. Require + # $PROJ to be the root of its own work tree before any other git command runs. + proj_top=$(git -C "$PROJ" rev-parse --show-toplevel 2>/dev/null) || proj_top="" + if [ -z "$proj_top" ]; then echo "$label: skipped: not a git repo" return 0 fi + # Both sides are physical paths (git resolves --show-toplevel through symlinks), + # so a symlinked clone dir still compares equal to its own root. + proj_abs=$(cd "$PROJ" && pwd -P) || proj_abs="" + if [ "$proj_top" != "$proj_abs" ]; then + echo "$label: skipped: not a clone root (git would act on $proj_top)" + return 0 + fi mode_line=$("$FM_ROOT/bin/fm-project-mode.sh" "$label" 2>/dev/null || echo "no-mistakes off") mode=${mode_line%% *} if [ "$mode" = "local-only" ]; then diff --git a/tests/fm-fleet-sync.test.sh b/tests/fm-fleet-sync.test.sh index b1fcd0a38e2..c2ea85ae361 100755 --- a/tests/fm-fleet-sync.test.sh +++ b/tests/fm-fleet-sync.test.sh @@ -12,6 +12,12 @@ # The pre-existing fast-forward / already-current / local-only / no-origin paths # must be unchanged, and bootstrap must relay the new outcomes as FLEET_SYNC lines. # +# It also pins the clone-root guard: a plain directory under projects/ resolves, +# through git's upward repository discovery, to the ENCLOSING repository - in a +# firstmate home, the firstmate checkout itself - so it must be skipped by name +# with the enclosing repo left untouched, in both the whole-fleet and +# single-project forms, while a symlinked clone dir still syncs. +# # It also pins the orphaned .git/packed-refs.lock recovery in the fetch step # (fetch_with_packed_refs_lock_guard, backed by bin/fm-lock-lib.sh's shared # staleness proof): a provably-stale lock is retried then removed and the clone @@ -90,6 +96,40 @@ run_sync() { FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" "$ROOT/bin/fm-fleet-sync.sh" "$@" 2>/dev/null } +# build_enclosing_home : an FM_HOME that is itself nested inside another git +# repository - firstmate's own layout, where projects/ sits inside the firstmate +# checkout. The enclosing repo is a clean clone of a bare origin that is one commit +# ahead, so a sync that walked git discovery UP out of projects/ would find a +# fast-forward available and visibly take it. Echoes the enclosing repo, which is +# also the home. Its work tree is left pristine so the only thing under projects/ +# is what the test puts there. +build_enclosing_home() { + local name=$1 root work remote enclosing remote_abs + root="$TMP_ROOT/enclosing-$name" + work="$root/work" + remote="$root/remote.git" + enclosing="$root/enclosing" + mkdir -p "$root" + + git init -q "$work" + git -C "$work" symbolic-ref HEAD refs/heads/main + printf '/projects/\n' > "$work/.gitignore" + git -C "$work" add .gitignore + commit_file "$work" AGENTS.md v0 C0 + + git clone --quiet --bare "$work" "$remote" + remote_abs=$(cd "$remote" && pwd) + git -C "$work" remote add origin "file://$remote_abs" + git -C "$work" push -q -u origin main + + git clone --quiet "file://$remote_abs" "$enclosing" + commit_file "$work" AGENTS.md v1 C1 + git -C "$work" push -q origin main + + mkdir -p "$enclosing/projects" + printf '%s\n' "$enclosing" +} + # --- packed-refs.lock fixtures ---------------------------------------------- # build_packed_prunable : like build_pair, but the clone has PACKED @@ -582,6 +622,57 @@ test_transient_packed_refs_lock_self_clears() { pass "a transient packed-refs.lock that self-clears is retried without a force-remove" } +test_non_clone_dir_never_syncs_the_enclosing_repo() { + local home before out after + home=$(build_enclosing_home nonclone) + # A worktree container, not a clone: the repo is one level BELOW it. + mkdir -p "$home/projects/not-a-clone/wt" + before=$(head_sha "$home") + + out=$(run_sync "$home") + after=$(head_sha "$home") + + assert_contains "$out" "not-a-clone: skipped: not a clone root" \ + "a non-repo directory under projects/ must be skipped by name" + assert_not_contains "$out" "not-a-clone: synced" \ + "a non-repo directory must never be reported as a synced project" + [ "$before" = "$after" ] || \ + fail "fleet-sync fast-forwarded the enclosing repo ($before -> $after) under a project's label" + pass "a non-repo directory under projects/ never fast-forwards the enclosing repo" +} + +test_non_clone_dir_named_directly_never_syncs_the_enclosing_repo() { + local home before out after + home=$(build_enclosing_home nonclonedirect) + mkdir -p "$home/projects/not-a-clone" + before=$(head_sha "$home") + + out=$(run_sync "$home" not-a-clone) + after=$(head_sha "$home") + + assert_contains "$out" "not-a-clone: skipped: not a clone root" \ + "the single-project form must apply the same clone-root guard" + [ "$before" = "$after" ] || \ + fail "the single-project form fast-forwarded the enclosing repo ($before -> $after)" + pass "the single-project form also refuses a directory that is not its own clone root" +} + +test_symlinked_clone_still_syncs() { + local home clone out + home=$(new_home) + clone=$(build_pair "$home" sigma) + advance_origin "$home" sigma C1 + # A symlinked clone dir is a real clone root; the guard compares resolved paths, + # so it must not be mistaken for a directory nested in someone else's repo. + mv "$clone" "$home/real-sigma" + ln -s "$home/real-sigma" "$clone" + + out=$(run_sync "$home") + + assert_contains "$out" "sigma: synced" "a symlinked clone must still fast-forward" + pass "the clone-root guard accepts a symlinked clone directory" +} + test_non_signature_fetch_failure_is_not_retried() { local home fakebin clone out err home=$(new_home) @@ -625,3 +716,6 @@ test_live_packed_refs_lock_is_never_removed test_live_git_cwd_in_clone_dir_blocks_removal test_transient_packed_refs_lock_self_clears test_non_signature_fetch_failure_is_not_retried +test_non_clone_dir_never_syncs_the_enclosing_repo +test_non_clone_dir_named_directly_never_syncs_the_enclosing_repo +test_symlinked_clone_still_syncs From 505c8195122b6d3e3a04fa48c13cd184df0321ba Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Sun, 23 Aug 2026 06:26:32 -0400 Subject: [PATCH 02/12] fix(bin): retry transient Lavish poll interruptions (#2846) * fix(procevent): retry a transient Lavish poll interruption quietly A live Lavish listener can be cut short by the server with exactly error: Lavish Editor poll response was interrupted code: SERVER_ERROR while the session's marks remain available. Firstmate registered raw `lavish-axi poll` output, so the generic process-event runner captured that transient response as a result and woke the whole fleet over what is really an internal retry. The Lavish adapter now registers its own listener command, which reruns the published blocking poll up to 12 times at 5 second intervals for that one exact two-line response. The match is deliberately narrow: real feedback, ended and missing sessions, any other SERVER_ERROR, and the same interruption still standing once the bound is spent all pass straight through and are captured and announced as before. The retry is a Lavish fact, so the generic runner stays adapter-agnostic. `FM_LAVISH_POLL_RETRY_DELAY` is a bounded 0 to 60 second override for the interval only, refused rather than rounded when malformed, so a test can exercise the real bound without waiting it out. * no-mistakes(review): Harden Lavish retry matching, validation, and cleanup * no-mistakes(review): Bound Lavish retry staging and stabilize regression * no-mistakes(document): docs: explain Lavish retry adoption * no-mistakes(lint): Restore Lavish trap ShellCheck suppression --- bin/fm-procevent-lavish.sh | 152 ++++++++++++++++++++++++++++- docs/configuration.md | 3 + tests/fm-procevent.test.sh | 190 +++++++++++++++++++++++++++++++++++++ 3 files changed, 342 insertions(+), 3 deletions(-) diff --git a/bin/fm-procevent-lavish.sh b/bin/fm-procevent-lavish.sh index 03bba8c31b1..2a73281ee6c 100755 --- a/bin/fm-procevent-lavish.sh +++ b/bin/fm-procevent-lavish.sh @@ -8,9 +8,14 @@ # fm-procevent-lavish.sh answers # fm-procevent-lavish.sh source-id # fm-procevent-lavish.sh retire +# fm-procevent-lavish.sh poll # # classify Print the lifecycle state a handler should act on: feedback, ended, # waiting, missing, or unknown. +# poll The registered listener command `arm` publishes, not a command to +# run in a conversational turn. It runs the published blocking poll +# and prints its response verbatim, absorbing only the one exact +# transient interruption described below. # terminal Exit 0 when the captured result means this Lavish source will never # produce another result, so the runner may retire it; any other exit # keeps it armed. This is the generic adapter contract bin/fm-procevent.sh @@ -39,6 +44,23 @@ # server-side events. It adds no periodic discovery, no timer fallback, and no # dependency on any unreleased capability. # +# BOUNDED QUIET RETRY, owned here and nowhere else. A live listener can be cut +# short by the server with exactly this two-line response while the session's +# marks remain available: +# +# error: Lavish Editor poll response was interrupted +# code: SERVER_ERROR +# +# That is an internal retry, not news, so registering the raw poll made the +# generic runner capture it and wake the whole fleet. `poll` therefore re-runs +# the published poll up to POLL_RETRY_LIMIT times for that exact response, with +# POLL_RETRY_DELAY_DEFAULT seconds between attempts. The match is exact and +# deliberately narrow: real feedback, ended and missing sessions, any other +# SERVER_ERROR, and the same interruption still standing after the bound is +# spent are all printed straight through and captured normally. The retry is a +# Lavish fact, so the generic runner in bin/fm-procevent.sh stays +# adapter-agnostic and learns nothing about it. +# # LOSS LIMITATION, stated plainly. The published poll destructively clears # feedback before returning it. A result lost after that clearing and before the # runner reads the process output is unrecoverable, and no Firstmate wrapper can @@ -59,7 +81,7 @@ FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" . "$SCRIPT_DIR/fm-procevent-lib.sh" die() { printf 'error: %s\n' "$1" >&2; exit 1; } -usage() { sed -n '2,47p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 2; } +usage() { sed -n '2,69p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 2; } # Canonical identity is physical, not the path string: Lavish itself keys a # session on the realpath of the artifact, so two names for one file are one @@ -83,11 +105,16 @@ cmd_arm() { [ -n "$artifact" ] || usage [ "$#" -eq 1 ] || usage command -v lavish-axi >/dev/null 2>&1 || die "lavish-axi is not installed" + poll_retry_delay >/dev/null id=$(cmd_source_id "$artifact") || exit 1 real=$(perl -MCwd=realpath -e '$p = realpath($ARGV[0]); defined($p) or exit 1; print "$p\n"' "$artifact" 2>/dev/null) \ || die "cannot resolve the artifact path: $artifact" - # The plain blocking form: no --timeout-ms, so completion is a server event. - "$SCRIPT_DIR/fm-procevent.sh" register lavish "$id" -- lavish-axi poll "$real" || exit 1 + # This adapter's own listener command, which runs the plain blocking form with + # no --timeout-ms so completion is a server event, and absorbs only the exact + # transient interruption. Registering raw poll output is what let that + # interruption reach the runner as a captured result. + "$SCRIPT_DIR/fm-procevent.sh" register lavish "$id" \ + -- "$SCRIPT_DIR/fm-procevent-lavish.sh" poll "$real" || exit 1 printf 'armed: %s\n' "$id" printf 'artifact: %s\n' "$real" } @@ -99,6 +126,124 @@ cmd_retire() { "$SCRIPT_DIR/fm-procevent.sh" retire "$id" } +# The bounded quiet retry described in the header. The bound is a constant +# because it is a property of the transient response, not an operator choice; +# only the delay takes an override, so a test can exercise the real bound +# without waiting it out. +POLL_RETRY_LIMIT=12 +POLL_RETRY_DELAY_DEFAULT=5 +POLL_RETRY_DELAY_MAX=60 + +# Exit 0 only for the exact two-line interruption, and nothing else. The whole +# response must be those two lines with those exact bytes: whitespace variants, +# a longer response that merely opens with them, and any other SERVER_ERROR are +# genuine errors this adapter must never swallow. +poll_response_filter() { # + perl -e ' + use strict; + use warnings; + my ($stage) = @ARGV; + my $expected = "error: Lavish Editor poll response was interrupted\ncode: SERVER_ERROR\n"; + open my $staged, ">", $stage or exit 2; + binmode STDIN; + binmode STDOUT; + binmode $staged; + my ($candidate, $streaming) = ("", 0); + sub write_all { + my ($handle, $bytes) = @_; + my $offset = 0; + while ($offset < length $bytes) { + my $written = syswrite $handle, $bytes, length($bytes) - $offset, $offset; + exit 2 unless defined $written; + $offset += $written; + } + } + while (1) { + my $count = sysread STDIN, my $chunk, 65536; + exit 2 unless defined $count; + last if $count == 0; + if ($streaming) { + write_all(*STDOUT, $chunk); + next; + } + my $room = length($expected) + 1 - length($candidate); + my $take = length($chunk) < $room ? length($chunk) : $room; + my $prefix = substr($chunk, 0, $take); + $candidate .= $prefix; + write_all($staged, $prefix); + my $matches_prefix = length($candidate) <= length($expected) + && substr($expected, 0, length($candidate)) eq $candidate; + if (!$matches_prefix) { + write_all(*STDOUT, $candidate); + write_all(*STDOUT, substr($chunk, $take)); + $streaming = 1; + } + } + exit 10 if !$streaming && $candidate eq $expected; + write_all(*STDOUT, $candidate) unless $streaming; + ' "$1" +} + +# Seconds between retries. FM_LAVISH_POLL_RETRY_DELAY is a bounded test +# override; a malformed or out-of-range value is refused rather than quietly +# rounded, because silently changing a retry cadence is how a bound stops +# meaning anything. +poll_retry_delay() { + local delay=${FM_LAVISH_POLL_RETRY_DELAY-} + if [ -z "$delay" ]; then + printf '%s\n' "$POLL_RETRY_DELAY_DEFAULT" + return 0 + fi + case "$delay" in + *[!0-9]*) die "FM_LAVISH_POLL_RETRY_DELAY must be whole seconds from 0 to $POLL_RETRY_DELAY_MAX: $delay" ;; + esac + [ "$delay" -le "$POLL_RETRY_DELAY_MAX" ] \ + || die "FM_LAVISH_POLL_RETRY_DELAY must be whole seconds from 0 to $POLL_RETRY_DELAY_MAX: $delay" + printf '%s\n' "$delay" +} + +cmd_poll() { + local artifact=${1-} delay attempt=0 response cleanup_command rc filter_rc + local pipeline_status + [ -n "$artifact" ] || usage + [ "$#" -eq 1 ] || usage + command -v lavish-axi >/dev/null 2>&1 || die "lavish-axi is not installed" + delay=$(poll_retry_delay) || exit 1 + response=$(mktemp "${TMPDIR:-/tmp}/fm-lavish-poll.XXXXXX") || die "cannot stage the poll response" + printf -v cleanup_command 'rm -f -- %q' "$response" + # shellcheck disable=SC2064 # $cleanup_command must expand now, while the staged path is still set. + trap "$cleanup_command" EXIT + # Retirement stops this listener by signalling its process group, and bash runs + # no EXIT trap for an uncaught signal, so each one cleans up the staged + # response and then re-raises itself with the default disposition, leaving the + # process dying exactly as the runner expects. + local signal + for signal in INT TERM HUP; do + # shellcheck disable=SC2064 # Same reason: expand now, while both are set. + trap "$cleanup_command; trap - $signal; kill -$signal $$" "$signal" + done + while :; do + lavish-axi poll "$artifact" | poll_response_filter "$response" + pipeline_status=("${PIPESTATUS[@]}") + rc=${pipeline_status[0]} + filter_rc=${pipeline_status[1]} + case "$filter_rc" in + 0) break ;; + 10) + if [ "$attempt" -lt "$POLL_RETRY_LIMIT" ]; then + attempt=$((attempt + 1)) + sleep "$delay" + else + cat -- "$response" + break + fi + ;; + *) die "cannot classify the poll response" ;; + esac + done + return "$rc" +} + # Read one field of the response's leading `session:` block. Those fields are # INDENTED, so each is read as the first indented match inside that block rather # than an anchored whole-line match; anchoring on "^status:" silently never @@ -242,6 +387,7 @@ cmd_answers() { case "${1-}" in arm) shift; cmd_arm "$@" ;; retire) shift; cmd_retire "$@" ;; + poll) shift; cmd_poll "$@" ;; source-id) shift; cmd_source_id "$@" ;; classify) shift; cmd_classify "$@" ;; terminal) shift; cmd_terminal "$@" ;; diff --git a/docs/configuration.md b/docs/configuration.md index c9d0d293a52..d520c7e6072 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -509,6 +509,9 @@ See [verification/public-followup.md](verification/public-followup.md) for the c A long-polling external process is registered as a *source* through its adapter, whose header and `--help` own the commands and flags. `bin/fm-procevent.sh` owns the generic contract; `bin/fm-procevent-lavish.sh` is the first adapter and wraps only the currently published `lavish-axi poll` interface. +That adapter, and only that adapter, retries the one exact transient response a cut-short listener returns while its marks remain available (`error: Lavish Editor poll response was interrupted` with `code: SERVER_ERROR`), up to 12 times at 5 second intervals, so an internal retry never reaches the runner as a captured result. +Real feedback, ended and missing sessions, any other `SERVER_ERROR`, and that same interruption still standing once the bound is spent are all captured and announced normally; `FM_LAVISH_POLL_RETRY_DELAY` is a bounded 0 to 60 second test override for the interval only, and the runner itself stays adapter-agnostic. +An already-armed Lavish source keeps its registered listener command until it is retired and armed again, so re-arm a live board once to adopt this retry policy. The `when` adapter (`bin/fm-procevent-when.sh`) turns this channel into a condition->action primitive: it registers a deterministic condition and a deterministic action once, its blocking child polls the condition without waking firstmate, and a stable true fires the action at most once before one terminal outcome is durably captured and published as a wake that remains eligible for re-announcement until handled. The (condition, action) spec is stored privately under `state/when/` and hash-bound by a trust record the same way `bin/fm-check-register.sh` binds a custom check, while the spec separately binds the resolved action executable's bytes; a mutated or unregistered spec or a changed action executable is refused before the action runs. diff --git a/tests/fm-procevent.test.sh b/tests/fm-procevent.test.sh index 878f71ac81b..3b93b1c1c37 100755 --- a/tests/fm-procevent.test.sh +++ b/tests/fm-procevent.test.sh @@ -595,6 +595,196 @@ out=$(PATH="$LAVISH_BIN:$PATH" FM_HOME="$HLT" "$ROOT/bin/fm-procevent-lavish.sh" assert_contains "$out" "retired: $lavish_id" "explicit adapter retirement stays supported after automatic retirement" pass "one Send & End yields exactly one captured result, automatic retirement, and no recurring poll" +# --- end-user-aligned regression: a transient poll interruption is not news --- +# The dogfood defect: a live board listener can answer with exactly +# error: Lavish Editor poll response was interrupted +# code: SERVER_ERROR +# while the board's marks remain available. Firstmate registered raw poll output, +# so the generic runner captured that transient response and woke the whole fleet +# over what is really an internal retry. Every scenario below runs through the +# adapter's own arm command and the real runner, so registration, capture, and +# publication are exercised for real. +LAVISH_SCRIPTED_BIN=$(fm_fakebin "$TMP_ROOT/lavish-scripted-stub") +cat > "$LAVISH_SCRIPTED_BIN/lavish-axi" <<'SH' +#!/usr/bin/env bash +# Stand-in for `lavish-axi poll `, scripted per scenario: LAVISH_SCRIPT +# names the response for each successive poll, one word per poll, and its last +# word repeats forever. `interrupt` is the exact transient response the server +# returns while the board's marks stay available. +n=$(cat "$LAVISH_COUNT" 2>/dev/null || echo 0) +n=$((n + 1)) +printf '%s\n' "$n" > "$LAVISH_COUNT" +read -r -a plan <<< "$LAVISH_SCRIPT" +i=$((n - 1)) +[ "$i" -ge "${#plan[@]}" ] && i=$((${#plan[@]} - 1)) +case "${plan[$i]}" in + interrupt) + printf 'error: Lavish Editor poll response was interrupted\ncode: SERVER_ERROR\n'; exit 1 ;; + near-interrupt) + printf 'error: Lavish Editor poll response was interrupted \ncode: SERVER_ERROR\n'; exit 1 ;; + other-server-error) + printf 'error: Lavish Editor session store is unavailable\ncode: SERVER_ERROR\n'; exit 1 ;; + feedback) + printf 'session:\n file: /board.html\n status: feedback\n session_ended: true\n ended_by: user\nfeedback[1]{text}:\n ship it\n' ;; + stream) + printf 'x%.0s' {1..4096} + printf 'ready\n' > "$LAVISH_STREAM_READY" + while [ ! -e "$LAVISH_STREAM_RELEASE" ]; do sleep 0.05; done + printf '\n' ;; +esac +SH +chmod +x "$LAVISH_SCRIPTED_BIN/lavish-axi" +export LAVISH_COUNT LAVISH_SCRIPT +# A bounded test override keeps the retry policy's real bound under test without +# making the suite wait out the production delay. +export FM_LAVISH_POLL_RETRY_DELAY=0 + +# Two interruptions, then the captain's real feedback: the retries are silent and +# only the feedback becomes a captured result and a check wake. +HRETRY="$TMP_ROOT/hretry"; new_home "$HRETRY" +RETRY_ART="$TMP_ROOT/retry-board.html" +printf '

retry

\n' > "$RETRY_ART" +retry_id=$("$ROOT/bin/fm-procevent-lavish.sh" source-id "$RETRY_ART") +PE_TRACKED+=("$HRETRY|$retry_id") +LAVISH_COUNT="$TMP_ROOT/retry-count"; LAVISH_SCRIPT="interrupt interrupt feedback" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HRETRY" \ + "$ROOT/bin/fm-procevent-lavish.sh" arm "$RETRY_ART" >/dev/null +PATH="$LAVISH_SCRIPTED_BIN:$PATH" pe "$HRETRY" reconcile >/dev/null +wait_for "$HRETRY/state/.wake-queue" || fail "feedback after interrupted polls produced no wake" +[ "$(cat "$LAVISH_COUNT")" = 3 ] \ + || fail "the interrupted listener was polled $(cat "$LAVISH_COUNT") times, not the two quiet retries plus the delivering poll" +[ "$(count_results "$HRETRY" "$retry_id")" = 1 ] \ + || fail "a retried interruption produced $(count_results "$HRETRY" "$retry_id") captured results instead of one" +[ "$(wake_payloads "$HRETRY" | sort -u | grep -c .)" = 1 ] \ + || fail "a retried interruption woke the fleet: $(wake_payloads "$HRETRY" | sort -u)" +assert_contains "$(wake_payloads "$HRETRY")" "procevent lavish $retry_id 1" \ + "feedback arriving after quiet retries is captured and announced" +assert_grep 'ship it' "$(first_result "$HRETRY" "$retry_id")" \ + "the announced result is the captain's feedback, not the interruption" +pass "a transient Lavish poll interruption is retried quietly and never announced" + +# Exhaustion is news: after the bounded retries the same exact response is +# captured and announced normally rather than being swallowed forever. +HEXH="$TMP_ROOT/hexh"; new_home "$HEXH" +EXH_ART="$TMP_ROOT/exhaust-board.html" +printf '

exhaust

\n' > "$EXH_ART" +exh_id=$("$ROOT/bin/fm-procevent-lavish.sh" source-id "$EXH_ART") +PE_TRACKED+=("$HEXH|$exh_id") +LAVISH_COUNT="$TMP_ROOT/exhaust-count"; LAVISH_SCRIPT="interrupt" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HEXH" \ + "$ROOT/bin/fm-procevent-lavish.sh" arm "$EXH_ART" >/dev/null +PATH="$LAVISH_SCRIPTED_BIN:$PATH" pe "$HEXH" start "$exh_id" >/dev/null +[ "$(cat "$LAVISH_COUNT")" = 13 ] \ + || fail "the retry bound polled $(cat "$LAVISH_COUNT") times, not the first poll plus 12 bounded retries" +[ "$(count_results "$HEXH" "$exh_id")" = 1 ] \ + || fail "exhaustion produced $(count_results "$HEXH" "$exh_id") captured results instead of one" +assert_contains "$(wake_payloads "$HEXH")" "procevent lavish $exh_id 1" \ + "the interruption that survives the bound is announced normally" +assert_grep 'poll response was interrupted' "$(first_result "$HEXH" "$exh_id")" \ + "the announced result is the exact interruption the server returned" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HEXH" \ + "$ROOT/bin/fm-procevent-lavish.sh" retire "$EXH_ART" >/dev/null +pass "an interruption that outlives the bounded retries is captured and announced" + +# A different SERVER_ERROR is a genuine error, never a retry: no fail-open drift +# from the one exact transient response this adapter owns. +HOTHER="$TMP_ROOT/hother"; new_home "$HOTHER" +OTHER_ART="$TMP_ROOT/other-board.html" +printf '

other

\n' > "$OTHER_ART" +other_id=$("$ROOT/bin/fm-procevent-lavish.sh" source-id "$OTHER_ART") +PE_TRACKED+=("$HOTHER|$other_id") +LAVISH_COUNT="$TMP_ROOT/other-count"; LAVISH_SCRIPT="other-server-error" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HOTHER" \ + "$ROOT/bin/fm-procevent-lavish.sh" arm "$OTHER_ART" >/dev/null +PATH="$LAVISH_SCRIPTED_BIN:$PATH" pe "$HOTHER" start "$other_id" >/dev/null +[ "$(cat "$LAVISH_COUNT")" = 1 ] \ + || fail "an unrelated SERVER_ERROR was retried $(cat "$LAVISH_COUNT") times instead of surfacing at once" +assert_contains "$(wake_payloads "$HOTHER")" "procevent lavish $other_id 1" \ + "an unrelated SERVER_ERROR is captured and announced immediately" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HOTHER" \ + "$ROOT/bin/fm-procevent-lavish.sh" retire "$OTHER_ART" >/dev/null +pass "only the exact interruption is retried; an unrelated SERVER_ERROR still surfaces" +unset FM_LAVISH_POLL_RETRY_DELAY + +# A whitespace variant is not the exact transient response and must surface on +# the first poll instead of drifting into the quiet retry policy. +HNEAR="$TMP_ROOT/hnear"; new_home "$HNEAR" +NEAR_ART="$TMP_ROOT/near-board.html" +printf '

near

\n' > "$NEAR_ART" +near_id=$("$ROOT/bin/fm-procevent-lavish.sh" source-id "$NEAR_ART") +PE_TRACKED+=("$HNEAR|$near_id") +LAVISH_COUNT="$TMP_ROOT/near-count"; LAVISH_SCRIPT="near-interrupt feedback" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HNEAR" FM_LAVISH_POLL_RETRY_DELAY=0 \ + "$ROOT/bin/fm-procevent-lavish.sh" arm "$NEAR_ART" >/dev/null +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HNEAR" pe "$HNEAR" start "$near_id" >/dev/null +[ "$(cat "$LAVISH_COUNT")" = 1 ] \ + || fail "a near-match interruption was retried instead of surfacing on its first poll" +assert_contains "$(wake_payloads "$HNEAR")" "procevent lavish $near_id 1" \ + "a whitespace variant of the interruption is captured and announced immediately" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HNEAR" \ + "$ROOT/bin/fm-procevent-lavish.sh" retire "$NEAR_ART" >/dev/null +pass "only the literal two-line interruption enters the quiet retry policy" + +# The public arm boundary refuses invalid retry intervals before it publishes a +# source registration, rather than arming a listener that can only fail later. +HINVALID="$TMP_ROOT/hinvalid"; new_home "$HINVALID" +INVALID_ART="$TMP_ROOT/invalid-delay-board.html" +printf '

invalid delay

\n' > "$INVALID_ART" +invalid_id=$("$ROOT/bin/fm-procevent-lavish.sh" source-id "$INVALID_ART") +for invalid_delay in 61 invalid; do + invalid_status=0 + invalid_out=$(PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HINVALID" \ + FM_LAVISH_POLL_RETRY_DELAY="$invalid_delay" \ + "$ROOT/bin/fm-procevent-lavish.sh" arm "$INVALID_ART" 2>&1) || invalid_status=$? + [ "$invalid_status" -ne 0 ] \ + || fail "arm accepted invalid retry delay: $invalid_delay" + assert_contains "$invalid_out" "must be whole seconds from 0 to 60" \ + "arm explains the rejected retry delay" + assert_absent "$HINVALID/state/procevent/$invalid_id.source" \ + "arm publishes no source registration for an invalid retry delay" +done +pass "arm rejects malformed and out-of-range retry delays before registration" + +# Shell-safe cleanup must preserve a valid TMPDIR containing an apostrophe. +QUOTED_TMPDIR="$TMP_ROOT/poll's-stage" +mkdir -p "$QUOTED_TMPDIR" +LAVISH_COUNT="$TMP_ROOT/quoted-count"; LAVISH_SCRIPT="feedback" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" TMPDIR="$QUOTED_TMPDIR" \ + "$ROOT/bin/fm-procevent-lavish.sh" poll "$NEAR_ART" >/dev/null +quoted_staged=("$QUOTED_TMPDIR"/fm-lavish-poll.*) +[ ! -e "${quoted_staged[0]}" ] \ + || fail "poll left its staged response behind in an apostrophe-containing TMPDIR" +pass "poll cleanup safely handles an apostrophe-containing TMPDIR" + +HSTREAM="$TMP_ROOT/hstream"; new_home "$HSTREAM" +STREAM_ART="$TMP_ROOT/stream-board.html" +STREAM_TMPDIR="$TMP_ROOT/stream-stage" +LAVISH_STREAM_READY="$TMP_ROOT/stream-ready" +LAVISH_STREAM_RELEASE="$TMP_ROOT/stream-release" +mkdir -p "$STREAM_TMPDIR" +printf '

stream

\n' > "$STREAM_ART" +stream_id=$("$ROOT/bin/fm-procevent-lavish.sh" source-id "$STREAM_ART") +PE_TRACKED+=("$HSTREAM|$stream_id") +LAVISH_COUNT="$TMP_ROOT/stream-count"; LAVISH_SCRIPT="stream" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HSTREAM" \ + "$ROOT/bin/fm-procevent-lavish.sh" arm "$STREAM_ART" >/dev/null +PATH="$LAVISH_SCRIPTED_BIN:$PATH" TMPDIR="$STREAM_TMPDIR" \ + LAVISH_STREAM_READY="$LAVISH_STREAM_READY" LAVISH_STREAM_RELEASE="$LAVISH_STREAM_RELEASE" \ + FM_PROCEVENT_MAX_OUTPUT_BYTES=100 pe "$HSTREAM" reconcile >/dev/null +wait_for "$LAVISH_STREAM_READY" || fail "streaming poll did not start" +stream_staged=("$STREAM_TMPDIR"/fm-lavish-poll.*) +[ -e "${stream_staged[0]}" ] || fail "streaming poll created no classifier staging file" +[ "$(wc -c < "${stream_staged[0]}" | tr -d ' ')" -le 100 ] \ + || fail "streaming poll exceeded its bounded classifier staging" +: > "$LAVISH_STREAM_RELEASE" +wait_for "$HSTREAM/state/.wake-queue" || fail "streaming poll produced no wake" +stream_result=$(first_result "$HSTREAM" "$stream_id" || true) +[ "$(wc -c < "$stream_result" | tr -d ' ')" -le 100 ] \ + || fail "streaming poll bypassed the runner output bound" +PATH="$LAVISH_SCRIPTED_BIN:$PATH" FM_HOME="$HSTREAM" \ + "$ROOT/bin/fm-procevent-lavish.sh" retire "$STREAM_ART" >/dev/null +pass "Lavish classification staging stays bounded while nonmatches stream" + # --- end-user-aligned regression: the exact drain-before-handling restart cut # Reproduces the confirmed defect through the public interface end to end: a # real blocking source completes, its result is captured and published, the From 86dd2f6cbaef3c7075ce467a0f8e565f20112bba Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:22:56 -0400 Subject: [PATCH 03/12] fix(brief): stop the documented {TASK} fill from corrupting the Herdr gate (#2838) The unguarded Herdr declaration quoted `{TASK}` in its own prose while the scaffold instructs firstmate to replace every `{TASK}` placeholder. The documented global replace therefore spliced the whole task body into the middle of the safety gate's sentence, silently destroying the one contract that exists precisely because the scaffold cannot inspect the task text. Reword the gate to refer to the task text filled in above, leaving the placeholder only at its genuine fill site. Rewording rather than renaming the token keeps the unfilled-charter guards in fm-home-seed.sh and fm-remote-home-seed.sh working unchanged. Add a regression test that performs the documented global fill on ship and scout scaffolds and asserts the body lands once and the gate survives. --- bin/fm-brief.sh | 2 +- tests/fm-brief.test.sh | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index 63ca1f054e1..3528fd3866b 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -291,7 +291,7 @@ HERDR_SECTION=$(printf '%s\n' \ else IFS= read -r -d '' HERDR_SECTION <<'EOF' || true # Herdr lifecycle declaration - NOT ENABLED -**HARD SAFETY GATE:** this scaffold cannot inspect the task text that replaces `{TASK}` later. +**HARD SAFETY GATE:** this scaffold cannot inspect the task text filled in above. If the task will start, stop, delete, restart, profile, or otherwise drive Herdr lifecycle behavior, stop and regenerate the brief with `--herdr-lab` before dispatch. Do not add Herdr lifecycle commands to this unguarded brief by hand. EOF diff --git a/tests/fm-brief.test.sh b/tests/fm-brief.test.sh index 05d732cba21..a4342d758f0 100755 --- a/tests/fm-brief.test.sh +++ b/tests/fm-brief.test.sh @@ -439,6 +439,41 @@ test_herdr_lab_omission_is_loud_for_ship_and_scout() { pass "fm-brief.sh: ship and scout scaffolds make omitted Herdr intent fail-visible" } +# Regression (issue #2575): AGENTS.md section 11 and this script's own help tell +# firstmate to replace EVERY `{TASK}` placeholder. The unguarded Herdr gate used +# to quote `{TASK}` in its own prose, so that documented global replace spliced +# the whole task body into the middle of the gate's sentence - silently +# destroying the one contract that exists precisely because the scaffold cannot +# see the task text. The placeholder must exist only at the genuine fill site, +# so the documented fill leaves the gate intact and the body appears once. +test_documented_global_replace_leaves_the_herdr_gate_intact() { + local home id brief kind count content filled body + home="$TMP_ROOT/task-fill-site-home" + mkdir -p "$home/data" + body='Restart the herdr session, then profile it' + for kind in ship scout; do + id="brief-fill-site-$kind" + if [ "$kind" = scout ]; then + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" firstmate --scout >/dev/null 2>&1 + else + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" firstmate --mode no-mistakes >/dev/null 2>&1 + fi + brief="$home/data/$id/brief.md" + assert_present "$brief" "$kind brief was not scaffolded" + count=$(grep -c -F '{TASK}' "$brief") + [ "$count" = 1 ] \ + || fail "$kind brief must carry exactly one {TASK} fill site, found $count" + content=$(cat "$brief") + filled=${content//'{TASK}'/$body} + count=$(printf '%s\n' "$filled" | grep -c -F "$body") + [ "$count" = 1 ] \ + || fail "$kind brief: the documented global {TASK} replace duplicated the task body $count times" + printf '%s\n' "$filled" | grep -qF 'this scaffold cannot inspect the task text' \ + || fail "$kind brief: the Herdr safety gate did not survive the documented global replace" + done + pass "fm-brief.sh: the documented {TASK} fill cannot corrupt the Herdr safety gate" +} + test_secondmate_no_projects_charter() { local home brief status home="$TMP_ROOT/no-projects-home" @@ -725,6 +760,7 @@ test_ship_project_memory_wording test_herdr_lab_contract_is_explicit_and_complete test_herdr_lab_contract_quotes_foreign_firstmate_path test_herdr_lab_omission_is_loud_for_ship_and_scout +test_documented_global_replace_leaves_the_herdr_gate_intact test_herdr_lab_contract_applies_to_scouts_but_not_secondmates test_secondmate_no_projects_charter test_secondmate_marked_request_reporting_contract From 266fdb9654d8e19f5f17e21794e03dd48ad31ae6 Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:23:21 -0400 Subject: [PATCH 04/12] fix(bin): resolve the busy-state lock mtime with the platform's own stat form (#2837) The writer lock's stale-lock branch read the lock's mtime with `stat -f %m ... || stat -c %Y ...`. On GNU coreutils `-f` is filesystem stat, so it consumed the format string as a path, complained on stderr, printed a partial filesystem dump (" File: ...") on stdout, and still exited 0. The GNU form in the fallback therefore never ran, and the following arithmetic evaluated the word `File`, aborting the writer under `set -u` with "File: unbound variable". fm-teardown.sh died there after returning the worktree, leaving state/.meta, .status, .busy-gen, .busy-state, .busy-state.lock/ and .turn-ended behind. The surviving metadata kept the watcher monitoring an endpoint whose agent was gone, so a finished task produced stale wakes forever, and every re-run died identically because the abandoned lock was never broken. Detect the platform once and pick the right stat form, the pattern bin/fm-watch.sh already documents, and treat any non-numeric result as "just created" so a future portability surprise degrades to a lock-timeout refusal rather than killing teardown mid-way. --- bin/fm-busy-event.sh | 19 +++++++++++- tests/fm-busy-state.test.sh | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/bin/fm-busy-event.sh b/bin/fm-busy-event.sh index 51896dc1c15..0abcab8ee39 100755 --- a/bin/fm-busy-event.sh +++ b/bin/fm-busy-event.sh @@ -95,6 +95,19 @@ REC=$(fm_busy_record_path "$STATE" "$ID") GEN_FILE=$(fm_busy_gen_path "$STATE" "$ID") LOCK="$REC.lock" +# Portable mtime in epoch seconds. macOS (BSD) stat uses `-f `; Linux (GNU) +# stat uses `-c `. Do NOT collapse this into `stat -f ... || stat -c +# ...`: on GNU `-f` is *filesystem* stat, so it reads the format string as +# a path, reports that on stderr, prints a partial filesystem dump (" File: +# ...") on stdout, and still exits 0 - the fallback never runs and the caller +# gets a non-numeric token. Detect the platform once and pick the right form, +# exactly as bin/fm-watch.sh does. +if [ "$(uname)" = Darwin ]; then + lock_mtime() { stat -f %m "$1" 2>/dev/null; } +else + lock_mtime() { stat -c %Y "$1" 2>/dev/null; } +fi + # Serialize writers. The lock protects seq advancement and the sidecar/record # pair; a holder that died mid-write is broken after FM_BUSY_LOCK_STALE_SECS. lock_acquire() { @@ -103,7 +116,11 @@ lock_acquire() { tries=$((tries + 1)) if [ "$tries" -ge 40 ]; then now=$(date +%s) - mtime=$(stat -f %m "$LOCK" 2>/dev/null || stat -c %Y "$LOCK" 2>/dev/null || echo "$now") + mtime=$(lock_mtime "$LOCK" || true) + # Anything unreadable or non-numeric reads as "just created", so an + # unforeseen stat surprise degrades to a lock-timeout refusal instead of + # aborting the writer - and its caller, fm-teardown.sh - under `set -u`. + case "$mtime" in ''|*[!0-9]*) mtime=$now ;; esac age=$((now - mtime)) if [ "$age" -ge "${FM_BUSY_LOCK_STALE_SECS:-5}" ]; then rmdir "$LOCK" 2>/dev/null || rm -rf "$LOCK" 2>/dev/null || true diff --git a/tests/fm-busy-state.test.sh b/tests/fm-busy-state.test.sh index b86c0108bed..e295871fa20 100755 --- a/tests/fm-busy-state.test.sh +++ b/tests/fm-busy-state.test.sh @@ -107,6 +107,64 @@ test_retire_serializes_and_rejects_stale_gen() { pass "retire waits for the writer lock and cannot remove a new incarnation" } +# Regression for issue #2625: the writer lock's stale-lock branch resolved the +# lock's mtime with `stat -f %m ... || stat -c %Y ...`. On GNU coreutils `-f` is +# *filesystem* stat, so it consumes the format string as a path, complains on +# stderr, prints " File: ..." on stdout, and still exits 0 - the GNU form in the +# fallback never ran. The following `$((now - mtime))` then evaluated the word +# `File`, which under `set -u` aborted the writer with "File: unbound variable". +# fm-teardown.sh died there after returning the worktree, leaving state/.meta +# and friends behind to generate stale wakes forever, and every re-run died +# identically because the abandoned lock directory was never broken. +# +# The stat and uname stubs make this deterministic on any host: the writer must +# take the Linux path and still break a provably stale lock. +test_stale_lock_broken_under_gnu_stat() { + local state gen fakebin real_uname out status + state=$(new_state_dir gnu-stat-lock) + gen=$("$EV" arm "$state" t1) + fakebin=$(fm_fakebin "$TMP_ROOT/gnu-stat-lock") + real_uname=$(command -v uname) + + # GNU coreutils semantics, self-contained so no real stat is consulted. + cat > "$fakebin/stat" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = -c ] && [ "${2:-}" = %Y ]; then + printf '%s\n' 1000000000 # long-abandoned lock + exit 0 +fi +if [ "${1:-}" = -f ]; then + echo "stat: cannot read file system information for '$2': No such file or directory" >&2 + shift 2 + printf ' File: "%s"\n' "${1:-}" + exit 0 +fi +exit 1 +SH + chmod +x "$fakebin/stat" + cat > "$fakebin/uname" <&1) && status=0 || status=$? + case "$out" in + *'unbound variable'*) fail "the writer still dies on GNU stat output: $out" ;; + esac + [ "$status" = 0 ] || fail "retire did not break a provably stale writer lock: $out" + [ ! -e "$state/t1.busy-state" ] || fail "retire left the record behind" + [ ! -e "$state/t1.busy-gen" ] || fail "retire left the gen sidecar behind" + [ ! -e "$state/t1.busy-state.lock" ] || fail "retire left the stale lock behind" + + # Teardown must be able to run again over the same task without failing. + PATH="$fakebin:$PATH" "$EV" retire "$state" t1 --current-gen \ + || fail "a repeated retire over already-cleaned state was not idempotent" + pass "the writer breaks a stale lock instead of dying on GNU stat output" +} + test_retire_missing_sidecar_is_idempotent() { local state gen state=$(new_state_dir retire-missing) @@ -387,6 +445,7 @@ test_apply_current_gen_reset test_apply_unarmed_refused test_retire_serializes_and_rejects_stale_gen test_retire_missing_sidecar_is_idempotent +test_stale_lock_broken_under_gnu_stat test_stale_gen_event_rejected test_stale_gen_record_unknown test_missing_record_unknown_not_idle From f170cedeb735759e9547a5b9de1a26eca7ea6d71 Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:36:44 -0400 Subject: [PATCH 05/12] fix(stow): add opt-in pass horizon for memory decay (#2850) * fix(stow): give memory decay a per-pass horizon so the clock fires The tiered decay clocks were wall-clock only, while admission is per-pass: each /stow admits the findings that pass produced. In a home that stows daily those two rates diverge by the stow cadence, an entry the fleet keeps exercising never reaches 30 days unreinforced, and memory only grows while the pass reports decay evaluated. Give each dated marker an optional unreinforced-pass counter and make both tiers stale at whichever horizon comes first: 10 passes or 30 days for aging, 3 passes or 7 days for perishable. Reinforcement clears the counter and nothing else does, so the existing evidence-based restamp rule stays the only way an entry renews its lease. An absent /N means zero, so entries that stay exercised carry no extra marker bytes, and a rarely stowed home keeps its current behaviour through the unchanged date horizon. * no-mistakes(document): Align stow workflow with dual decay clocks * fix(stow): make the per-pass decay horizon opt-in The unreinforced-pass horizon shipped as a new default archival cadence, which is a product default rather than a restoration of the existing wall-clock contract. Keep the 30-day and 7-day horizons as the only default clock, and put the 10-pass and 3-pass horizons behind an explicit opt-in: config/stow-pass-horizon for the firstmate home, and the file's own header pointer for the public skill. With the opt-in absent no counter is written and no counter is read, so a home that does not ask for it decays exactly as it does today. * no-mistakes(review): Preserve frozen counters and correct archive provenance --- .agents/skills/stow/SKILL.md | 30 ++++++++++++++++++++++++++++-- AGENTS.md | 1 + docs/configuration.md | 10 ++++++++++ skills/stow/SKILL.md | 14 ++++++++++++-- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/.agents/skills/stow/SKILL.md b/.agents/skills/stow/SKILL.md index c7d96ce30db..348a9975471 100644 --- a/.agents/skills/stow/SKILL.md +++ b/.agents/skills/stow/SKILL.md @@ -20,6 +20,8 @@ Markers are compact trailing HTML comments, deliberately cheap because marker by - `` - an `aging` entry; the embedded date is its last-reinforced date. - `` - a `perishable` entry; the embedded date is its last-reinforced date. +- `` - only in a home that has opted in to the pass horizon below: either dated marker may carry `/N`, the number of passes that evaluated the entry without reinforcing it. + An absent `/N` means zero, so an entry the fleet keeps exercising costs no counter bytes at all, and a home that has not opted in never writes one. - `` - an explicitly `pinned` entry in a file whose default tier is not `pinned`. - `` - migration-only: an unconfirmed legacy entry that has consumed its one grace cycle, carrying no date because grace is not reinforcement. @@ -27,6 +29,7 @@ Markers are compact trailing HTML comments, deliberately cheap because marker by - Treehouse pool slots share one repo, so workers must create their task branch before editing. - While state/.afk exists, the away-daemon owns triage (until the afk-wake fix lands; tracked: afk-pi-wake-bypass-r1). - Never restart the shared no-mistakes daemon while runs are active. +- Codex writes its trust prompt to stderr, not stdout. ``` The tier names say what the pass does with an entry: @@ -43,13 +46,33 @@ Marking rules: - An entry matching its file's `pinned` default carries no marker at all; every `aging` and `perishable` entry always carries its dated marker, whose letter names the tier, so a clock-carrying entry is never ambiguous with unmarked legacy material. - Marker and header-pointer bytes count toward the startup-memory budget: the pass's own bookkeeping is costed content, never free, which is why the spellings above are as short as they are. - Each memory file's header carries at most a one-line pointer naming this skill as the scheme owner, such as ``. - This skill text is the single owner of tier semantics, marker spellings, and clocks - deliberately policy, not configuration - and no memory file header may restate them. + This skill text is the single owner of tier semantics, marker spellings, and clocks, and no memory file header may restate them. + The one exception is the `config/stow-pass-horizon` presence flag below, which turns a single extra horizon on for this home and changes nothing else on this page. - Inspect each editable file's header pointer on every pass and add or correct it; for a read-only `data/captain-shared.md`, leave the file byte-identical and route a missing or outdated pointer to the primary owner. The required receipt action for that file is `routed`, not `unchanged`; name the ownership exception and do not declare the session reset-safe. - A pre-existing missing or hand-dropped marker is never grounds for destructive treatment: it means the file's default tier; an unmarked entry in a default-pinned file is simply pinned, while an unmarked entry in a file whose default tier carries a clock follows the migration rule below. Decay advances only when a pass runs, so a home stowed less often than a clock experiences that clock at its stow interval. +### Optional pass horizon (config/stow-pass-horizon) + +The wall-clock horizons above are this skill's default contract, and a home gets exactly them unless it asks for more. +A home may opt in to a second, per-pass horizon by creating the local, gitignored `config/stow-pass-horizon` presence flag. +While that file is absent nothing else in this section applies: no counter is written, no counter already in a file is read, and every entry decays on its date alone. + +Opt in where admission and decay are not commensurable. +A pass admits the findings that pass produced, so growth is a per-pass quantity, while a wall-clock horizon alone is a per-day one. +In a home that stows daily those two rates diverge by the stow cadence, an entry the fleet keeps exercising never sits unreinforced for 30 wall-clock days, and the date horizon is evaluated vacuously every pass while the file only grows. +A home stowed monthly already exceeds its date horizon on a single pass and gains nothing from the flag. + +While the flag is present: + +- An `aging` entry is stale at whichever horizon it reaches first: 10 passes that evaluated it without reinforcing it, or 30 days since its last-reinforced date. +- A `perishable` entry is stale at whichever it reaches first: 3 unreinforced passes, or 7 days. +- Reinforcement refreshes the date and clears the counter, and nothing else clears it, so the evidence hard rule in step 4 stays the only way an entry renews its lease. +- An existing dated marker with no `/N` reads as counter zero, so a home that opts in migrates nothing. +- Removing the flag returns the home to the default contract on its next pass: any `/N` already written is then neither read nor advanced, and is left in place rather than rewritten. + ## Required startup-memory pass Every `/stow` invocation performs this complete pass, even when the session contains no new finding: @@ -72,10 +95,12 @@ Every `/stow` invocation performs this complete pass, even when the session cont Retain lower-utility material only while budget remains. 4. Reinforce and stamp. Refresh an entry's last-reinforced date to today only when this session actually exercised, confirmed, or re-derived it. + Where the optional pass horizon is enabled, refreshing that date also clears the entry's unreinforced-pass counter, and nothing else clears it. **Hard rule: reinforcement requires independent evidence from this session that you can name in the receipt; plausibility, importance, prior knowledge, and the entry's own text are not evidence, and any explicit statement that no confirming session evidence exists requires the no-evidence path.** For an unmarked `data/learnings.md` entry with no such evidence, the no-evidence path is always to append `` and retain it for this entire pass; never stamp or archive it during that same invocation. Stamp each newly written entry with today's date and its tier per the marking rules, and admit a new `perishable` entry only with its named checkable expiry condition in the prose. 5. Evaluate every dated entry in each editable memory file against its tier clock. + Where the optional pass horizon is enabled, first increment the unreinforced-pass counter of every dated entry step 4 did not reinforce - that increment is the pass tick - then judge each dated entry against both of its horizons and treat it as stale at whichever it reaches first. Re-validate a stale `aging` entry from current evidence and refresh its date, or archive it. Re-confirm a stale `perishable` entry against its named condition: still open means refresh the date, while resolved, expired, or no longer checkable means archive it in this pass. Promote `perishable` to `aging` when its condition keeps proving durable past its expected life, and retier in place when a supersession changes an entry's lifetime. @@ -108,6 +133,7 @@ Never describe the session as reset-safe while the memory total is over budget o Stale never means deleted: pruning an entry from an editable memory file always means moving it to `data/memory-archive.md`, this home's append-only, never-injected cold tier, gitignored with the rest of `data/` and never counted by the budget report. Each archived entry keeps its provenance under a dated pass heading: source file, tier, last-reinforced date, and the reason it left. +Include the unreinforced-pass counter only when the optional pass horizon itself made the entry stale, using the exact reason `unreinforced p`; omit the counter when the wall-clock horizon or any other reason caused archival, even if the active marker carried one. Archive provenance stays verbose rather than compact because the cold tier is never budget-counted. ```markdown @@ -115,7 +141,7 @@ Archive provenance stays verbose rather than compact because the cold tier is ne - (from learnings.md, tier: perishable, reinforced: 2026-06-30) While state/.afk exists, the away-daemon owns triage... [archived: unreinforced 39d] ``` -Reasons include `unreinforced d`, `budget oldest-first`, and `legacy-unvalidated`. +Reasons include `unreinforced d`, `unreinforced p`, `budget oldest-first`, and `legacy-unvalidated`. Archiving is a move, not a removal, and recovery is `grep` plus copy back with no tooling. Each home keeps its own archive, the archive never cascades, and truncating a grown archive is a captain decision, not a mechanism. diff --git a/AGENTS.md b/AGENTS.md index 10fc4132eca..06685cee0dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,6 +71,7 @@ config/backlog-backend backlog backend override; LOCAL, gitignored; absent or " config/backend runtime session-provider backend override for new tasks; LOCAL, gitignored; absent = falls through to runtime auto-detection (the runtime firstmate itself is executing inside), then tmux; tmux is the verified reference backend (docs/tmux-backend.md), while herdr, zellij, orca, and cmux are experimental spawn backends (docs/herdr-backend.md, docs/zellij-backend.md, docs/orca-backend.md, docs/cmux-backend.md) - herdr and cmux can also be selected by runtime auto-detection, zellij and orca never are (always explicit), and codex-app is not accepted; see docs/codex-app-backend.md; inherited by secondmate homes under the primary-authoritative contract in secondmate-provisioning config/calm Pi Calm presentation preference; LOCAL, gitignored, and not inherited; see docs/configuration.md "Pi Calm preference" config/startup-memory-budget primary-authoritative per-home startup-memory budget; LOCAL, gitignored, materialized as 7,500 estimated tokens by locked primary bootstrap and inherited into secondmate homes; see docs/configuration.md "Startup memory budget" +config/stow-pass-horizon optional presence flag opting this home in to /stow's default-off pass-count decay horizon; LOCAL, gitignored, and not inherited; see docs/configuration.md "Stow pass horizon" config/herdr-presentation-spaces optional "off" opt-out from, or "on" opt-in to, Herdr's default-on disposable single-task visual projection, which is unconfigured-default-on only at or above a Herdr version floor; LOCAL, gitignored; inherited by secondmate homes; see docs/herdr-backend.md "Presentation spaces" config/trace-context optional presence flag enabling default-off native W3C trace-context propagation to spawned agents; LOCAL, gitignored; inherited by secondmate homes; see docs/configuration.md "Trace context propagation" and docs/trace-context.md config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") diff --git a/docs/configuration.md b/docs/configuration.md index d520c7e6072..df86ffb2798 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -163,6 +163,16 @@ An inherited `data/captain-shared.md` counts in a secondmate's total but remains The internal [`/stow` skill](../.agents/skills/stow/SKILL.md) owns curation and its automatic secondmate cascade, which accounts every home against this same per-home allowance separately rather than against a fleet total. The helper's header owns exact parsing, publication, and report output mechanics. +## Stow pass horizon (config/stow-pass-horizon) + +`config/stow-pass-horizon` is an optional local, gitignored presence flag that opts this home in to the pass-count decay horizon in the internal [`/stow` skill](../.agents/skills/stow/SKILL.md). +Without it a `/stow` pass decays memory entries on their wall-clock horizons alone - 30 days for `aging`, 7 days for `perishable` - which is the default and unchanged behavior. +With it, an entry is also stale after 10 passes (`aging`) or 3 passes (`perishable`) that evaluated it without reinforcing it, whichever horizon it reaches first. +Opt in for a home that stows often enough that entries never sit unreinforced for a wall-clock horizon, so memory only grows against the startup-memory budget above; a home that stows rarely already exceeds its date horizon on a single pass and gains nothing. +The flag is per home and is not inherited by secondmate homes, because stow cadence is a property of the home doing the stowing. +Only the file's presence is read, so its contents are ignored; remove it to return to the default contract on the next pass. +The skill text owns the marker spelling, the tick order, and the reinforcement rule. + ## Secondmate routes (data/secondmates.md) Persistent secondmate routes live locally in `data/secondmates.md`. diff --git a/skills/stow/SKILL.md b/skills/stow/SKILL.md index 95522b37ed5..b45ebe8fd9b 100644 --- a/skills/stow/SKILL.md +++ b/skills/stow/SKILL.md @@ -93,6 +93,8 @@ Markers are compact trailing HTML comments, deliberately cheap because marker by - `` - an `aging` entry; the embedded date is its last-reinforced date. - `` - a `perishable` entry; the embedded date is its last-reinforced date. +- `` - only in a file whose header pointer opts in to the pass horizon below: either dated marker may carry `/N`, the number of passes that evaluated the entry without reinforcing it. + An absent `/N` means zero, so an entry you keep exercising costs no counter bytes at all, and a file that has not opted in never writes one. - `` - an explicitly `pinned` entry in a file whose default tier is not `pinned`. - `` - migration-only: an unconfirmed legacy entry that has consumed its one grace cycle, carrying no date because grace is not reinforcement. @@ -100,6 +102,7 @@ Markers are compact trailing HTML comments, deliberately cheap because marker by - The staging deploy needs the VPN profile active or the smoke test hangs. - CI is red on the flaky auth test until the pinned runner image updates (tracked in TODO). - Always run the schema linter before touching migrations. +- The staging seed script must run before the fixture import. ``` The tier names say what this skill does with an entry: @@ -114,14 +117,21 @@ Rules: - Unless a file's own header pointer names a different default, a user-level memory file defaults to `pinned`, while a project memory file and `.stow-notes.md` default to `aging`. - An entry matching its file's `pinned` default carries no marker at all; every `aging` and `perishable` entry always carries its dated marker, whose letter names the tier, so a clock-carrying entry is never ambiguous with unmarked legacy material. - Marker and pointer bytes are part of the file's cost, so bookkeeping stays minimal by design. -- Every governed memory file this skill curates carries at most a one-line header pointer naming this skill as the scheme owner, such as ``, optionally naming that file's default tier when it deviates. - The tier semantics, marker spellings, and clocks live only in this skill and are never restated in a file header. +- Every governed memory file this skill curates carries at most a one-line header pointer naming this skill as the scheme owner, such as ``, optionally naming that file's default tier when it deviates and the pass horizon when that file opts in, as in ``. + The tier semantics, marker spellings, and clocks live only in this skill and are never restated in a file header, which names an option but never its numbers. During one-time migration, add the pointer even to a default-pinned file that contains only unmarked entries, so every governed file names its scheme owner. - Refresh an entry's last-reinforced date only on real evidence from the current session: the fact was used, confirmed, or re-derived. Mere presence in the file is not evidence, and re-reading memory is never reinforcement. +- The dates above are the default and only clock, and a file gets exactly them unless its header pointer opts in to the pass horizon. + Opt a file in where you stow often enough that the date clock never fires: admitting findings is a per-pass event, so an entry you keep exercising never sits unreinforced for 30 wall-clock days and the file only grows, while a project you stow rarely already passes its date horizon in a single pass and gains nothing. + Never add that opt-in on your own initiative; the user chooses it, one file at a time. +- While a file is opted in, an `aging` entry there is stale at whichever comes first - 10 passes that evaluated it without reinforcing it, or 30 days - and a `perishable` entry at whichever comes first - 3 unreinforced passes, or 7 days. + Increment the counter of every dated entry that pass did not reinforce before judging staleness, read a dated marker with no `/N` as counter zero so nothing needs migrating, and clear the counter only by refreshing the date on real evidence. + In a file that is not opted in, never write a counter and never read one that is already there; preserve any existing `/N` byte-for-byte instead of normalizing or removing it. - Re-confirm a stale `perishable` entry against its named condition: still open means refresh the date, while resolved, expired, or no longer checkable means archive it now. - Decay is evaluated only when this skill runs; nothing happens between passes, so an infrequently stowed project experiences the clocks at its stow interval. - Stale never means deleted: a stale entry moves to a `.stow-archive.md` in the source file's own directory, never loaded by any session, and its archive record includes the source filename, tier, reinforcement date when present, and a one-line reason. + Include the unreinforced-pass counter only when the pass horizon itself made the entry stale, using the exact reason `unreinforced p`; omit the counter when the wall-clock horizon or any other reason caused archival, even if the active marker carried one. In a git worktree, verify that this archive path is not already tracked in the index before writing any archived fact there. If it is tracked, do not write to it and report that archival is blocked until the user chooses a safe destination. Otherwise add a `.stow-archive.md` line to a `.gitignore` file in the archive's directory, and never write archived facts into a git-tracked file. From 2f250c7ab37d68a42aa313b7459997504a009f86 Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:25:06 -0400 Subject: [PATCH 06/12] test(watcher): stop fixture confirmation budgets racing real child startup (#2876) tests/fm-watcher-lock.test.sh passed in isolation but failed intermittently under full-suite and ambient concurrent load. bin/fm-watch-arm.sh computes its confirmation deadline immediately after forking the real child watcher, so the child's entire fork, exec, lock acquisition and beacon publication has to land inside that wall clock. Two cases shrank that budget to one second, leaving a two-second window for work measured at 3.1-4.9s under CPU oversubscription, so the arm honestly reported "FAILED - no live watcher with a fresh beacon" and their premises collapsed. A third case ran on the production budget, but its child must also execute a registered check before exiting: measured at 1.9-2.3s idle and 9.1-13.1s under load, against an 11s budget. The two cases that must confirm a real child now hold the arm to production's own budget instead of a shrunken fixture one, the immediate-wake case gets an explicit budget with headroom over its measured loaded cost, and the two waits for the arm's typed failure are sized off the largest production default rather than a fixed eight seconds. No bin/ change and no default behavior change: the lock's fail-closed semantics, SIGSTOP handling, stale-heartbeat detection and the arm's typed failures are untouched. Verified 4/4 green at 3x CPU oversubscription (loadavg 75-80) after 3/3 red before the change, and CONTRIBUTING.md records the convention. --- CONTRIBUTING.md | 2 ++ tests/fm-watcher-lock.test.sh | 39 ++++++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98cc88a5f68..aa53fa2efb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,6 +103,8 @@ Family selection is the ordinary local path; `--all` is deliberate full regressi CI owns broad regression across required portable parallel shards, the portable serial lane's separate-runner shards, the Herdr lane, lint, invariants, the coverage guard, and stock macOS Bash compatibility in [`.github/workflows/ci.yml`](.github/workflows/ci.yml). Use `bin/fm-test-run.sh --list-lanes` for exact lane names and `--help` for `--jobs` rules and required gate-skip flags when reproducing a lane locally. Discover tests by listing `tests/*.test.sh`: each is a self-contained bash script named `.test.sh`, and its header comment describes what it covers, so pass one to `bin/fm-test-run.sh` to focus on a subject with canonical timing output. +A fixture may shorten a production timeout to keep a failure path prompt, but never below what the real work inside that window costs on a loaded machine: a fork, an exec, a lock acquisition, a beacon publication, or a first-poll check. +Where a case's assertion is not about the timeout itself, give that window headroom over the measured loaded cost, and bound the test's own waiting with iteration-counted poll loops, which stretch under load where a wall-clock budget does not. Tests that need a real optional backend or an explicit opt-in (real herdr/zellij/cmux smoke tests, the live Pi regression) skip themselves and print the tool or environment gate needed to enable them, so the portable suite remains safe on machines without those tools. The [Herdr backend guide](docs/herdr-backend.md#destructive-lab-safety) owns the lane's isolation boundary, while [runtime backend verification](docs/verification/runtime-backends.md#herdr) owns active empirical evidence; live harness credential tests remain opt-in. diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index a3628b1694f..482e425a9f5 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -13,6 +13,13 @@ WATCH_ARM="$ROOT/bin/fm-watch-arm.sh" DRAIN="$ROOT/bin/fm-wake-drain.sh" LIB="$ROOT/bin/fm-wake-lib.sh" +# An arm only reports its typed failure after wait_for_healthy_successor has +# spent the whole confirmation budget, so cases that wait for that failure must +# outlast the largest production default (30s on MSYS, 10s elsewhere - see +# ARM_CONFIRM_DEFAULT in bin/fm-watch-arm.sh). This is a ceiling spent only when +# an arm genuinely fails to exit; a passing case returns as soon as it does. +ARM_FAIL_EXIT_POLLS=400 + TMP_ROOT=$(fm_test_tmproot fm-watcher-lock-tests) mark_pr_check_migration_complete() { @@ -536,7 +543,14 @@ test_arm_self_eviction_is_loud_without_successor() { fakebin="$dir/fakebin" armout="$dir/arm.out" mark_pr_check_migration_complete "$state" - PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=0.2 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_ARM_CONFIRM_TIMEOUT=1 "$WATCH_ARM" > "$armout" & + # The arm's confirmation budget bounds a REAL child startup (fork, exec, lock + # acquisition, beacon publication), so this case holds the arm to production's + # own budget rather than a shrunken fixture one: a one-second budget turned + # ordinary CPU contention into an honest "FAILED - no live watcher with a fresh + # beacon" and broke this case's premise under full-suite load (issue #2844). + # It stays at the production default rather than something roomier because the + # same budget bounds the successor wait this case deliberately spends below. + PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=0.2 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$armout" & armpid=$! i=0 while [ "$i" -lt 80 ]; do @@ -551,7 +565,7 @@ test_arm_self_eviction_is_loud_without_successor() { # self-evict normally. With no verified successor, the arm must turn that # otherwise clean empty close into the typed nonzero failure. printf '%s\n' "$$" > "$state/.watch.lock/pid" - wait_for_exit "$armpid" 80 + wait_for_exit "$armpid" "$ARM_FAIL_EXIT_POLLS" status=$? [ "$status" -ne 0 ] && [ "$status" -ne 124 ] || fail "self-evicted arm did not fail nonzero (status $status)" grep -qF 'watcher: FAILED - cycle ended without an actionable reason' "$armout" || fail "self-evicted arm omitted the typed cycle-end failure" @@ -742,7 +756,13 @@ SH FM_STATE_OVERRIDE="$state" "$ROOT/bin/fm-check-register.sh" task >/dev/null \ || fail "could not register immediate-wake custom check" rc=0 - PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_GUARD_GRACE=0 FM_POLL=5 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=0 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$armout" || rc=$? + # This case asserts wake propagation, not the confirmation deadline, and its + # child must also run the registered check before exiting: measured at 1.9-2.3s + # idle but 9.1-13.1s at 3x CPU oversubscription, against an 11s production + # budget. An explicit budget takes the deadline out of the assertion and costs + # nothing on a passing run, because the arm returns as soon as the child + # settles (issue #2844). + PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_GUARD_GRACE=0 FM_POLL=5 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=0 FM_HEARTBEAT=999999 FM_ARM_CONFIRM_TIMEOUT=60 "$WATCH_ARM" > "$armout" || rc=$? [ "$rc" -eq 0 ] || fail "arm returned non-zero for an immediate wake (status $rc): $(cat "$armout")" grep -F "check: $check_file: merged: https://example.test/pr/7" "$armout" >/dev/null || fail "arm did not propagate the immediate check wake" ! grep -qF 'watcher: FAILED' "$armout" || fail "arm printed FAILED after a valid immediate wake" @@ -766,12 +786,15 @@ test_arm_waits_for_peer_beacon_after_child_stands_down() { printf '%s\n' "$dir" > "$state/.watch.lock/fm-home" printf '%s\n' "$WATCH" > "$state/.watch.lock/watcher-path" printf '%s\n' "$identity" > "$state/.watch.lock/pid-identity" - PATH="$fakebin:$PATH" FM_HOME="$dir" FM_POLL=5 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_ARM_CONFIRM_TIMEOUT=1 FM_ARM_ATTACH_POLL=0.1 "$WATCH_ARM" > "$armout" & + # Same budget contract as the self-eviction case: the owned child's real + # startup and stand-down happen inside the arm's confirmation window, so the + # window stays production-sized (issue #2844). + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_POLL=5 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_ARM_ATTACH_POLL=0.1 "$WATCH_ARM" > "$armout" & armpid=$! # Synchronize on the owned child declining the live peer lock before making - # the peer healthy. Sleeping for the same one-second budget as the arm made - # this regression fixture race the confirmation deadline under full-suite - # load, rather than testing the intended successor-handshake boundary. + # the peer healthy. Sleeping for the same budget the arm spends made this + # regression fixture race the confirmation deadline under full-suite load, + # rather than testing the intended successor-handshake boundary. i=0 while [ "$i" -lt 80 ]; do grep -qF "watcher: already running pid $peer" "$state"/.watch-arm-output.* 2>/dev/null && break @@ -793,7 +816,7 @@ test_arm_waits_for_peer_beacon_after_child_stands_down() { # After the peer dies without a successor, the attached arm must fail loudly. kill "$peer" 2>/dev/null || true wait "$peer" 2>/dev/null || true - wait_for_exit "$armpid" 80 + wait_for_exit "$armpid" "$ARM_FAIL_EXIT_POLLS" status=$? [ "$status" -ne 0 ] && [ "$status" -ne 124 ] || fail "attached arm did not fail after peer died (status $status): $(cat "$armout")" grep -qF 'watcher: FAILED - cycle ended without an actionable reason' "$armout" || fail "peer-attached arm did not emit the typed cycle-end failure" From 197afbb79f8bcc0d1da0239caa859fcd5e445d04 Mon Sep 17 00:00:00 2001 From: Christopher McKay <101884182+karotkriss@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:26:04 -0400 Subject: [PATCH 07/12] fix(bin): deterministically order remote tool paths (#2870) * fix(bin): order discovered tool installs by the shell's own expansion fm_remote_job_compose_operator_path built the asdf and mise install directories with `compgen -G`, which does not sort. Bash sorts glob matches in pathexp.c, on the shell's own pathname-expansion path only; `compgen -G` reaches the same glob_filename through pcomplete.c, which sorts nothing. On bash 3.2 (macOS /bin/bash) and every bash before 5.3 that handed the composition raw readdir order, so which install of a multi-version tool a remote job resolved was decided by directory order on disk rather than by this composition. Expand the globs at the call sites and let the function take the matches, so the composition and the documented portable-PATH contract are the same operation. Quoting the account home at the call site also stops a home whose name contains glob metacharacters from being reinterpreted. The colocated regression pins both the order and the mechanism: bash 5.3 moved sorting into the glob library, so an order-only assertion cannot see the defect there. * no-mistakes(review): Remove source-reading PATH regression guard --- bin/fm-remote-job-lib.sh | 26 ++++++++++++++++++-------- tests/fm-remote-job.test.sh | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/bin/fm-remote-job-lib.sh b/bin/fm-remote-job-lib.sh index 73bffa54c70..25d7bb73b40 100755 --- a/bin/fm-remote-job-lib.sh +++ b/bin/fm-remote-job-lib.sh @@ -30,7 +30,10 @@ # PATH, HOME, FM_HOME, FM_ROOT_OVERRIDE, and FM_REMOTE_JOB_ACTIVE=1. The PATH # is intentionally filesystem-discovered rather than login-shell-derived: # ~/.local/bin; nvm, asdf, and mise shims/install bins; Nix; Homebrew; and the -# system tail. No shell startup files are evaluated. +# system tail. No shell startup files are evaluated. Each discovered set is +# appended in the shell's own sorted pathname-expansion order, so which install +# of a multi-version tool wins is fixed by this composition rather than by the +# order the filesystem happens to return. # # On macOS the worker is Firstmate's Aqua LaunchAgent # dev.firstmate.remote-job at ~/Library/LaunchAgents/dev.firstmate.remote-job.plist @@ -130,11 +133,18 @@ fm_remote_job_path_append_resolved_dir() { # fm_remote_job_path_append "$physical" } -fm_remote_job_append_glob_dirs() { # - local pattern=$1 directory - while IFS= read -r directory; do +# Callers pass an already-expanded glob rather than the pattern, because only +# the shell's own pathname expansion sorts its matches: bash sorts +# glob_filename's result in pathexp.c, while `compgen -G` reaches the same +# glob_filename through pcomplete.c, which does not sort. On bash 3.2 (macOS +# /bin/bash) that handed back raw readdir order, so which install of a +# multi-version tool a remote job resolved depended on the filesystem instead +# of on this composition. +fm_remote_job_append_dirs() { # + local directory + for directory in "$@"; do fm_remote_job_path_append_if_dir "$directory" - done < <(compgen -G "$pattern" || true) + done } fm_remote_job_nvm_default_selector() { # @@ -217,11 +227,11 @@ fm_remote_job_compose_operator_path() { # nvm_bin=$(fm_remote_job_nvm_selected_bin "$account_home" 2>/dev/null || true) [ -z "$nvm_bin" ] || fm_remote_job_path_append "$nvm_bin" fm_remote_job_path_append_if_dir "$account_home/.asdf/shims" - fm_remote_job_append_glob_dirs "$account_home/.asdf/installs/*/*/bin" + fm_remote_job_append_dirs "$account_home"/.asdf/installs/*/*/bin fm_remote_job_path_append_if_dir "$account_home/.local/share/mise/shims" fm_remote_job_path_append_if_dir "$account_home/.mise/shims" - fm_remote_job_append_glob_dirs "$account_home/.local/share/mise/installs/*/*/bin" - fm_remote_job_append_glob_dirs "$account_home/.mise/installs/*/*/bin" + fm_remote_job_append_dirs "$account_home"/.local/share/mise/installs/*/*/bin + fm_remote_job_append_dirs "$account_home"/.mise/installs/*/*/bin fm_remote_job_path_append_resolved_dir "$account_home/.nix-profile/bin" account_user=$(id -un 2>/dev/null || true) if [ -n "$account_user" ]; then diff --git a/tests/fm-remote-job.test.sh b/tests/fm-remote-job.test.sh index 0b6dede4d65..436dfd44123 100755 --- a/tests/fm-remote-job.test.sh +++ b/tests/fm-remote-job.test.sh @@ -165,6 +165,25 @@ case ":$FM_REMOTE_JOB_OPERATOR_PATH:" in esac pass "operator PATH resolves the authorized Nix profile bin link" +# Which install of a multi-version tool a remote job resolves is decided by the +# order these directories land on PATH, so the composition has to be sorted +# rather than whatever order the filesystem returns. The fixture is created in +# a deliberately unsorted order, and the expectation is the shell's own +# pathname expansion - the mechanism the portable-PATH contract in +# tests/fm-on.test.sh reconstructs. +MISE_INSTALLS="$ACCOUNT_HOME/.local/share/mise/installs" +for TOOL_VERSION in node/26.7.0 node/8.1 node/26 bun/1.4 bun/1.3.14 python/3.12.7; do + mkdir -p "$MISE_INSTALLS/$TOOL_VERSION/bin" +done +fm_remote_job_compose_operator_path "$ACCOUNT_HOME" >/dev/null +MISE_COMPOSED=$(printf '%s\n' "$FM_REMOTE_JOB_OPERATOR_PATH" | tr ':' '\n' | grep -F "$MISE_INSTALLS/" || true) +MISE_EXPECTED=$(printf '%s\n' "$MISE_INSTALLS"/*/*/bin) +[ "$MISE_COMPOSED" = "$MISE_EXPECTED" ] \ + || fail "the composed operator PATH did not order tool installs like the shell's own expansion"$'\n'"expected: $MISE_EXPECTED"$'\n'"actual: $MISE_COMPOSED" +# This assertion detects the defect on bash 3.2 and 5.2, where compgen -G returns unsorted glob matches, but reads green on bash 5.3+ because glob sorting moved into the glob library so both mechanisms agree there. +rm -rf -- "$ACCOUNT_HOME/.local/share/mise" +pass "operator PATH orders discovered tool installs deterministically" + HOME="$ACCOUNT_HOME" PATH="$RUNTIME_BIN:/usr/bin:/bin:/usr/sbin:/sbin" FM_FAKE_PERL_LOG="$FAKE_PERL_LOG" \ FM_ROOT_OVERRIDE="$REMOTE_ROOT" FM_REMOTE_JOB_STATE_ROOT="$STATE_ROOT" \ FM_REMOTE_JOB_PLATFORM_OVERRIDE=Linux FM_REMOTE_JOB_TIMEOUT=5 \ From 52f62ab155e8696d62490eb8e735f8dea359c224 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:54:14 -0700 Subject: [PATCH 08/12] fix(bin): prevent routed secondmate work from stranding (#2848) * fix: surface stalled secondmate queues and wake handoffs * no-mistakes(review): Make handoff wakes retryable and stall alerts crash-safe * no-mistakes(review): Prevent duplicate handoff wakes and cover remote delivery * no-mistakes(review): Serialize local handoffs and preserve pre-move wake intent * no-mistakes(review): Serialize teardown with handoffs and retain remote wake confirmation * no-mistakes(review): Reconcile correlated handoff wake delivery after crashes * no-mistakes(review): Keep failed wakes retryable and isolate stall receipts * no-mistakes(review): Reset known-undelivered wake attempts for durable retries * no-mistakes(review): Refuse duplicate sends for unresolved delivery attempts * no-mistakes(review): Atomically restore retryability after reconciled send failures * no-mistakes(review): Serialize delivery confirmation with reconciliation * no-mistakes(document): Document routed wake and stall supervision * no-mistakes(lint): Fix ShellCheck expansion and subshell warnings * no-mistakes: apply CI fixes * no-mistakes: apply CI fixes * no-mistakes(review): Retire stale wake state and defer pre-move wakes * no-mistakes(review): Secure markers, bind batches, and preserve teardown routes * no-mistakes(review): Preserve unresolved prepared wakes across unrelated handoffs * no-mistakes(review): Preserve prepared wakes before unrelated moving handoffs * no-mistakes(document): Document prepared wake batch ownership * no-mistakes: apply CI fixes * no-mistakes: apply CI fixes * no-mistakes(review): Make local wake retirement recoverable * no-mistakes(document): Clarify handoff recovery and teardown documentation --- .agents/skills/bootstrap-diagnostics/SKILL.md | 4 +- .../skills/secondmate-provisioning/SKILL.md | 4 +- bin/fm-backlog-handoff.sh | 303 +++++++- bin/fm-pending-reply-lib.sh | 78 +- bin/fm-send.sh | 29 +- bin/fm-teardown.sh | 255 ++++++- bin/fm-wake-drain.sh | 4 + bin/fm-wake-lib.sh | 63 ++ bin/fm-watch.sh | 95 +++ docs/architecture.md | 13 +- docs/configuration.md | 6 +- docs/herdr-backend.md | 4 +- docs/remote-secondmates.md | 4 +- docs/scripts.md | 4 +- tests/fm-backlog-handoff.test.sh | 671 ++++++++++++++++++ tests/fm-gotmp.test.sh | 10 + tests/fm-pending-reply.test.sh | 55 ++ tests/fm-remote-backlog-handoff.test.sh | 86 +++ ...fm-remote-secondmate-lifecycle-e2e.test.sh | 16 + tests/fm-secondmate-lifecycle-e2e.test.sh | 26 +- tests/fm-wake-queue.test.sh | 206 ++++++ 21 files changed, 1897 insertions(+), 39 deletions(-) diff --git a/.agents/skills/bootstrap-diagnostics/SKILL.md b/.agents/skills/bootstrap-diagnostics/SKILL.md index 95932444f83..0aad8846387 100644 --- a/.agents/skills/bootstrap-diagnostics/SKILL.md +++ b/.agents/skills/bootstrap-diagnostics/SKILL.md @@ -53,8 +53,8 @@ When any diagnostic needs captain attention, report the plain consequence and re - `SECONDMATE_SYNC: secondmate : skipped: ` - secondmate convergence left a live home on its existing checkout because the home was dirty, diverged, unsafe, on the wrong branch, missing its placement-specific target commit, unreachable, or otherwise not fast-forwardable, or because inherited local-material propagation failed; bootstrap continued, but inspect the reason because the secondmate's tracked instructions, inherited settings, or shared captain preferences may be stale after a primary update. - `SECONDMATE_LIVENESS: secondmate : skipped: |respawn failed after : ` - the session-start liveness sweep could not guarantee that the registered secondmate is running a real agent process. Investigate the reason because that secondmate is not guaranteed live. -- `SECONDMATE_HANDOFF: secondmate : pending delivery: item(s)` - queued work has already left the main dispatchable backlog and remains safe in the named remote route's backlog-format outbox. - Preserve that outbox and rerun `bin/fm-backlog-handoff.sh --resume-pending` after same-host connectivity returns; never re-add or dispatch the items from the main backlog. +- `SECONDMATE_HANDOFF: secondmate : pending delivery: item(s)` - queued work has already left the main dispatchable backlog and remains safe in the named remote route's backlog-format outbox, pending backlog receipt or receiver-wake confirmation. + Preserve that outbox and rerun `bin/fm-backlog-handoff.sh --resume-pending` after the route or endpoint problem is resolved; never re-add or dispatch the items from the main backlog. An unsafe-outbox variant requires path and file-type inspection before any retry. - `NUDGE_SECONDMATES: secondmate : send failed: ` - secondmate convergence changed a running home's loaded instructions or inherited config, but the deterministic `fm-send.sh fm-` re-read nudge failed. Inspect the reason, keep the pending marker under `state/.secondmate-nudge-pending/` intact, and rerun session start after the endpoint or metadata issue is fixed so bootstrap can retry the exact same marked send on the same local or remote route. diff --git a/.agents/skills/secondmate-provisioning/SKILL.md b/.agents/skills/secondmate-provisioning/SKILL.md index b878c6f7658..07428f7b8fd 100644 --- a/.agents/skills/secondmate-provisioning/SKILL.md +++ b/.agents/skills/secondmate-provisioning/SKILL.md @@ -189,7 +189,9 @@ After seeding, run this handoff for the new secondmate's in-scope queued items. For an existing or inherited domain, complete record intake first so no already-shipped plan row is handed off as open work. For a local route, the helper resolves and validates the secondmate home from `data/secondmates.md`, then delegates the item move to `tasks-axi mv` (the single owner of the backlog format), which moves each named item - and a whole connected set, blocker plus dependents, atomically - from the main `data/backlog.md` into the secondmate home's `data/backlog.md`. For a remote route, the same helper first moves the dependency-closed set atomically from the main backlog into `data/handoff/.outbox.md`, then transfers that backlog-format outbox through `fm-on.sh` and lets the remote home's `fm-backlog-receive.sh` move every not-already-present key under the destination lock. -The outbox is the whole recovery record: its presence means delivery is unfinished, `--resume-pending` safely re-delivers it, and confirmed receipt removes it. +After a new local placement or a remote outbox receipt becomes durable, the helper sends one marked routed-work instruction through the receiving secondmate's recorded endpoint; missing or failed delivery makes the command fail loudly with the moved work intact, and the same handoff command retries known-undelivered wake intent without moving an already-present item again. +An unresolved delivery attempt is never blindly resent. +For a remote route, the outbox remains until both backlog receipt and receiver wake are confirmed; `--resume-pending` retries unfinished outboxes, while the script header owns its stable wake-correlation recovery state. There is no two-phase handoff journal and no tasks-axi release beyond the already-required atomic `mv` capability. Bootstrap retries pending outboxes when mutation is authorized and emits `SECONDMATE_HANDOFF:` for any that remain. This delegated route remains required when `config/backlog-backend=manual`, which controls only routine firstmate backlog edits. diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index 97bda75c331..fa729c9d1b6 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -50,7 +50,16 @@ # Remote routes use an outbox handoff: one atomic local tasks-axi mv removes the # selected set from the dispatchable backlog into data/handoff/.outbox.md, # then an idempotent confined transfer and fm-backlog-receive.sh deliver it. -# A present outbox is the whole recovery record. No two-phase journal exists. +# A present outbox remains the remote retry trigger until backlog receipt and +# receiver wake are both confirmed; a companion pending-reply correlation makes +# crash recovery reconcile an attempted or confirmed wake instead of blindly +# resending it. A prepared local wake is bound to the exact sorted +# requested-key batch; an unrelated handoff to that mate refuses until the +# original batch is retried, so it cannot discard wake intent for work that +# already moved. No two-phase journal exists. +# Every newly durable backlog delivery also sends one marked wake to the +# receiving endpoint. A missing endpoint or a live endpoint that rejects the +# wake makes the handoff fail with the delivered backlog intact. # Usage: fm-backlog-handoff.sh ... # fm-backlog-handoff.sh --resume-pending set -eu @@ -70,6 +79,10 @@ MAIN_BACKLOG="$DATA/backlog.md" . "$SCRIPT_DIR/fm-wake-lib.sh" # shellcheck source=bin/fm-public-followup-lib.sh . "$SCRIPT_DIR/fm-public-followup-lib.sh" +# shellcheck source=bin/fm-pending-reply-lib.sh +. "$SCRIPT_DIR/fm-pending-reply-lib.sh" + +RECEIVER_WAKE_MESSAGE='New routed work is in your backlog. Run bin/fm-session-start.sh now, then act on the routed task.' ACTIVE_HANDOFF_LOCK= ACTIVE_REGISTRY_LOCK= @@ -99,6 +112,7 @@ if [ "${1:-}" = --resume-pending ]; then else [ "$#" -ge 2 ] || { echo "usage: fm-backlog-handoff.sh ..." >&2; exit 1; } ID=$1 + case "$ID" in ''|*[!A-Za-z0-9._-]*) echo "error: unsafe secondmate id: $ID" >&2; exit 1 ;; esac shift fi @@ -300,12 +314,212 @@ warn_stale_public_commitments() { # ... return 0 } +# Wake a live receiver after its backlog has become durable. The marked message +# uses the normal endpoint route, so local and remote secondmates share the same +# verified submit and failure semantics. A seeded but not-yet-spawned home is a +# valid handoff destination, but its missing endpoint is reported rather than +# pretending the task was started. +receiver_wake_batch_id() { # ... + local digest + if command -v shasum >/dev/null 2>&1; then + digest=$(printf '%s\n' "$@" | LC_ALL=C sort | shasum -a 256 2>/dev/null | awk '{print $1}') + else + digest=$(printf '%s\n' "$@" | LC_ALL=C sort | sha256sum 2>/dev/null | awk '{print $1}') + fi + printf '%s' "$digest" | grep -Eq '^[a-f0-9]{64}$' || return 1 + printf '%s' "${digest:0:16}" +} + +receiver_wake_state_write() { # + local id=$1 value=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" tmp + case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$value" in + pending|confirmed) ;; + prepared:*) printf '%s' "$value" | grep -Eq '^prepared:[a-f0-9]{16}:[a-f0-9]{16}$' || return 1 ;; + pending:*) printf '%s' "$value" | grep -Eq '^pending:[a-f0-9]{16}$' || return 1 ;; + confirmed:*) printf '%s' "$value" | grep -Eq '^confirmed:[a-f0-9]{16}$' || return 1 ;; + *) return 1 ;; + esac + tmp=$(umask 077; mktemp "$STATE/.backlog-handoff-wake.XXXXXX") || return 1 + if ! printf '%s\n' "$value" > "$tmp" || ! chmod 600 "$tmp" || ! mv -f -- "$tmp" "$marker"; then + rm -f -- "$tmp" + return 1 + fi +} + +receiver_wake_mark() { # [batch-id] + local id=$1 wake_phase=$2 batch=${3:-} marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec + local wake_state + case "$wake_phase" in prepared|pending) ;; *) return 1 ;; esac + if [ -e "$marker" ] || [ -L "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + prepared:*|pending:*) + corr=${value#*:} + corr=${corr%%:*} + rec=$(fm_pending_reply_path "$STATE" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$id" ] + return $? + ;; + pending) ;; + *) return 1 ;; + esac + fi + corr=$(fm_pending_reply_create "$FM_HOME" "$STATE" "$id" "$RECEIVER_WAKE_MESSAGE") || return 1 + wake_state="$wake_phase:$corr" + if [ "$wake_phase" = prepared ]; then + printf '%s' "$batch" | grep -Eq '^[a-f0-9]{16}$' || return 1 + wake_state="$wake_state:$batch" + fi + if ! receiver_wake_state_write "$id" "$wake_state"; then + fm_pending_reply_discard_undelivered "$STATE" "$corr" || true + return 1 + fi +} + +receiver_wake_mark_pending() { # + receiver_wake_mark "$1" pending +} + +receiver_wake_mark_prepared() { # + receiver_wake_mark "$1" prepared "$2" +} + +receiver_wake_discard_prepared() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + prepared:*) + corr=${value#prepared:} + corr=${corr%%:*} + ;; + *) return 1 ;; + esac + fm_pending_reply_discard_undelivered "$STATE" "$corr" || return 1 + rm -f -- "$marker" +} + +receiver_wake_promote_prepared() { # + local id=$1 batch=$2 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + prepared:*:"$batch") + corr=${value#prepared:} + corr=${corr%%:*} + ;; + pending:*) return 0 ;; + *) return 1 ;; + esac + receiver_wake_state_write "$id" "pending:$corr" +} + +receiver_wake_discard_pending() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value corr + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + pending:*) + corr=${value#pending:} + fm_pending_reply_discard_undelivered "$STATE" "$corr" || return 1 + ;; + pending) ;; + *) return 1 ;; + esac + rm -f -- "$marker" +} + +receiver_wake_clear_confirmed() { # + local id=$1 marker="$STATE/.backlog-handoff-$1.wake-pending" value + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + pending|pending:*) return 0 ;; + confirmed|confirmed:*) rm -f -- "$marker" ;; + *) return 1 ;; + esac +} + +wake_secondmate_receiver() { # + local id=$1 corr=$2 meta="$STATE/$1.meta" out rc=0 + if [ ! -f "$meta" ] || [ -L "$meta" ]; then + printf 'error: handed off work to secondmate %s, but no live receiver endpoint is recorded; the destination backlog is durable and the receiver was not woken\n' "$id" >&2 + return 1 + fi + [ "$(grep '^kind=' "$meta" | cut -d= -f2-)" = secondmate ] || { + printf 'error: secondmate %s has non-secondmate endpoint metadata; backlog is durable but the receiver was not woken\n' "$id" >&2 + return 1 + } + out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_ROOT_OVERRIDE="$FM_ROOT" \ + FM_PENDING_REPLY_EXISTING_CORR="$corr" \ + "$SCRIPT_DIR/fm-send.sh" "$id" "$RECEIVER_WAKE_MESSAGE" 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + [ -z "$out" ] || printf '%s\n' "$out" >&2 + printf 'error: backlog delivery to secondmate %s succeeded, but its receiver wake failed; rerun this handoff to retry the wake\n' "$id" >&2 + return 1 + fi + [ -z "$out" ] || printf '%s\n' "$out" +} + +wake_pending_secondmate_receiver() { # [retain-confirmed] + local id=$1 retain=${2:-0} marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec delivered + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + if [ ! -f "$marker" ] || [ -L "$marker" ]; then + printf 'error: receiver wake state for secondmate %s is unsafe or invalid\n' "$id" >&2 + return 1 + fi + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + confirmed|confirmed:*) return 0 ;; + prepared|prepared:*) + printf 'error: receiver wake for secondmate %s was prepared before its backlog became durable\n' "$id" >&2 + return 1 + ;; + pending) + receiver_wake_mark_pending "$id" || return 1 + value=$(cat "$marker" 2>/dev/null || true) + ;; + esac + case "$value" in pending:*) corr=${value#pending:} ;; *) + printf 'error: receiver wake state for secondmate %s is unsafe or invalid\n' "$id" >&2 + return 1 + ;; + esac + rec=$(fm_pending_reply_path "$STATE" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$id" ] || return 1 + fm_pending_reply_reconcile_delivery "$STATE" "$corr" >/dev/null 2>&1 || true + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + if [ -z "$delivered" ]; then + fm_pending_reply_corr_reusable "$STATE" "$corr" "$id" || { + printf 'error: receiver wake delivery for secondmate %s is unresolved; refusing to resend correlation %s\n' "$id" "$corr" >&2 + return 1 + } + wake_secondmate_receiver "$id" "$corr" || return 1 + fi + if [ "$retain" = 1 ]; then + receiver_wake_state_write "$id" "confirmed:$corr" || { + printf 'error: receiver wake for secondmate %s was confirmed, but confirmed state could not be recorded\n' "$id" >&2 + return 1 + } + else + rm -f -- "$marker" || { + printf 'error: receiver wake for secondmate %s was confirmed, but pending state could not be cleared\n' "$id" >&2 + return 1 + } + fi +} + outbox_item_count() { # awk '/^- \[[ x]\] / { count++ } END { print count + 0 }' "$1" } remote_deliver_outbox() { # - local id=$1 outbox=$2 remote_rel receive_out snapshot bytes hash generation counter counter_tmp current + local id=$1 outbox=$2 remote_rel receive_out snapshot bytes hash generation counter counter_tmp current marker [ -f "$outbox" ] && [ ! -L "$outbox" ] || { echo "error: pending outbox is unavailable or unsafe: $outbox" >&2 return 1 @@ -348,8 +562,24 @@ remote_deliver_outbox() { # echo "error: handoff receipt by $id was unavailable or completion is unknown; outbox preserved at $outbox" >&2 return 1 fi + marker="$STATE/.backlog-handoff-$id.wake-pending" + case "$(cat "$marker" 2>/dev/null || true)" in + pending:*|confirmed|confirmed:*) ;; + *) receiver_wake_mark_pending "$id" || { + echo "error: remote backlog is durable at $id, but receiver wake state could not be recorded; outbox preserved at $outbox" >&2 + return 1 + } ;; + esac + if ! wake_pending_secondmate_receiver "$id" 1; then + echo "error: remote backlog is durable at $id; outbox preserved at $outbox for wake retry" >&2 + return 1 + fi rm -f -- "$outbox" || { - echo "error: remote receipt was confirmed but local outbox cleanup failed: $outbox" >&2 + echo "error: receiver wake was confirmed but local outbox cleanup failed: $outbox" >&2 + return 1 + } + rm -f -- "$marker" || { + echo "error: remote outbox cleanup succeeded but confirmed receiver wake state could not be cleared: $marker" >&2 return 1 } printf '%s\n' "$receive_out" @@ -388,6 +618,12 @@ remote_handoff() { # outbox="$DATA/handoff/$id.outbox.md" validate_backlog_file "main backlog" "$MAIN_BACKLOG" || return 1 validate_backlog_file "remote handoff outbox" "$outbox" || return 1 + if [ ! -e "$outbox" ] && [ ! -L "$outbox" ]; then + receiver_wake_clear_confirmed "$id" || { + echo "error: stale receiver wake state for secondmate $id could not be cleared" >&2 + return 1 + } + fi fm_tasks_axi_compatible || { echo "error: a compatible tasks-axi with atomic multi-ID mv support is required to stage remote handoffs; run bin/fm-bootstrap.sh for the required version" >&2 return 1 @@ -429,6 +665,18 @@ remote_handoff() { # return 1 done < <(backlog_key_noncanonical_body_lines "$MAIN_BACKLOG" "$key") done + # Do not append a fresh handoff to an older recovery batch. In particular, a + # confirmed wake can survive when outbox cleanup fails; if new work were + # staged into that outbox, the old confirmation would suppress the wake for + # the new work. Finish receipt, wake reconciliation, and cleanup for the old + # batch first. A failure leaves the fresh items dispatchable in main. + if [ "${#to_move[@]}" -gt 0 ] && [ -f "$outbox" ] \ + && [ "$(outbox_item_count "$outbox")" -gt 0 ]; then + remote_deliver_outbox "$id" "$outbox" || { + echo "error: previous remote handoff for secondmate $id could not be completed; nothing new was staged" >&2 + return 1 + } + fi seed_backlog_scaffold "$outbox" if [ "${#to_move[@]}" -gt 0 ]; then if ! mv_out=$(tasks-axi mv "${to_move[@]}" --file "$MAIN_BACKLOG" --to "$outbox" 2>&1); then @@ -502,7 +750,10 @@ if [ "$REMOTE" = 1 ]; then release_remote_locks exit "$rc" fi -release_remote_locks +ACTIVE_HANDOFF_LOCK="$STATE/.backlog-handoff-$ID.lock" +fm_lock_acquire_wait "$ACTIVE_HANDOFF_LOCK" +fm_lock_release "$ACTIVE_REGISTRY_LOCK" +ACTIVE_REGISTRY_LOCK= RAW_HOME=$(secondmate_home "$ID") || exit 1 [ -n "$RAW_HOME" ] || { echo "error: secondmate $ID has no home in $REG" >&2; exit 1; } @@ -556,8 +807,22 @@ if [ "$FAILED" -ne 0 ]; then exit 1 fi +REQUESTED_BATCH=$(receiver_wake_batch_id "$@") || { + echo "error: receiver wake batch identity could not be recorded; nothing was moved" >&2 + exit 1 +} + if [ "${#TO_MOVE[@]}" -eq 0 ]; then + WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" + case "$(cat "$WAKE_PENDING_MARKER" 2>/dev/null || true)" in + prepared:*:"$REQUESTED_BATCH") receiver_wake_promote_prepared "$ID" "$REQUESTED_BATCH" || exit 1 ;; + prepared:*) + echo "error: a prepared receiver wake for secondmate $ID belongs to a different routed batch; retry that original handoff before handling ${ALREADY[*]}" >&2 + exit 1 + ;; + esac echo "nothing to move: ${ALREADY[*]:-no keys} already present in $SUB_BACKLOG" + wake_pending_secondmate_receiver "$ID" || exit 1 exit 0 fi @@ -579,6 +844,27 @@ if ! fm_tasks_axi_compatible; then exit 1 fi +WAKE_PENDING_MARKER="$STATE/.backlog-handoff-$ID.wake-pending" +if [ -e "$WAKE_PENDING_MARKER" ] || [ -L "$WAKE_PENDING_MARKER" ]; then + case "$(cat "$WAKE_PENDING_MARKER" 2>/dev/null || true)" in + prepared:*:"$REQUESTED_BATCH") receiver_wake_discard_prepared "$ID" || exit 1 ;; + prepared:*) + echo "error: a prepared receiver wake for secondmate $ID belongs to a different routed batch; retry that original handoff before moving ${TO_MOVE[*]}" >&2 + exit 1 + ;; + *) + wake_pending_secondmate_receiver "$ID" || { + echo "error: previous receiver wake for secondmate $ID is unresolved; nothing new was moved" >&2 + exit 1 + } + ;; + esac +fi +receiver_wake_mark_prepared "$ID" "$REQUESTED_BATCH" || { + echo "error: receiver wake state for secondmate $ID could not be recorded; nothing was moved" >&2 + exit 1 +} + # Seed the destination with firstmate's standard three-section scaffold when it # does not exist yet, so the moved item lands under the right section. (Left to # create the file itself, tasks-axi mv writes its own `# Backlog` title format, @@ -599,6 +885,10 @@ if ! MV_OUT=$(tasks-axi mv "${TO_MOVE[@]}" --file "$MAIN_BACKLOG" --to "$SUB_BAC if [ "$SUB_CREATED" -eq 1 ]; then rm -f "$SUB_BACKLOG" fi + receiver_wake_discard_prepared "$ID" || { + echo "error: tasks-axi mv failed and receiver wake state could not be cleared" >&2 + exit 1 + } if [ -n "$MV_OUT" ]; then printf '%s\n' "$MV_OUT" >&2 fi @@ -608,6 +898,11 @@ fi echo "handed off ${#TO_MOVE[@]} item(s) to $ID: ${TO_MOVE[*]}" echo " into $SUB_BACKLOG" +receiver_wake_promote_prepared "$ID" "$REQUESTED_BATCH" || { + echo "error: handed off work to secondmate $ID, but durable receiver wake state could not be recorded" >&2 + exit 1 +} +wake_pending_secondmate_receiver "$ID" || exit 1 if [ "${#ALREADY[@]}" -gt 0 ]; then echo " already present (skipped): ${ALREADY[*]}" fi diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 5453585d0e2..b32fff8672c 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -344,6 +344,18 @@ fm_pending_reply_prepare_delivery() { # } fm_pending_reply_confirm_delivery() { # + local state=$1 corr=$2 lock rc=0 + local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK + STATE=$state + lock="$state/.pending-reply-$corr.lock" + . "$_FM_PENDING_REPLY_LIB_DIR/fm-wake-lib.sh" + fm_lock_acquire_wait "$lock" || return 1 + _fm_pending_reply_confirm_delivery_locked "$@" || rc=$? + fm_lock_release "$lock" + return "$rc" +} + +_fm_pending_reply_confirm_delivery_locked() { # local state=$1 corr=$2 now marker marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") if ! fm_pending_reply_prepare_delivery "$state" "$corr"; then @@ -372,7 +384,7 @@ fm_pending_reply_mark_delivery_unknown() { # fm_pending_reply_set "$rec" phase delivery_unknown } -fm_pending_reply_reconcile_delivery() { # +_fm_pending_reply_reconcile_delivery_locked() { # local state=$1 corr=$2 rec delivered marker entry delivery_state value epoch local grace now age phase rec=$(fm_pending_reply_path "$state" "$corr") @@ -412,6 +424,68 @@ fm_pending_reply_reconcile_delivery() { # return 1 } +fm_pending_reply_reconcile_delivery() { # + local state=$1 corr=$2 lock rc=0 + local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK + STATE=$state + lock="$state/.pending-reply-$corr.lock" + . "$_FM_PENDING_REPLY_LIB_DIR/fm-wake-lib.sh" + fm_lock_acquire_wait "$lock" || return 1 + _fm_pending_reply_reconcile_delivery_locked "$@" || rc=$? + fm_lock_release "$lock" + return "$rc" +} + +fm_pending_reply_delivery_attempt_unresolved() { # + local state=$1 corr=$2 rec delivered marker entry + rec=$(fm_pending_reply_path "$state" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -z "$delivered" ] || return 1 + marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + entry=$(cat "$marker" 2>/dev/null || true) + case "$entry" in attempted=*) return 0 ;; esac + return 1 +} + +# A definitive backend rejection makes the existing correlation retryable again. +# Reconciliation may have aged the same attempted sidecar to delivery_unknown +# while the backend call was in flight, so both undelivered phases converge here +# under the per-correlation lock; a confirmed delivery can never be reset. +fm_pending_reply_reset_known_undelivered() { # + local state=$1 corr=$2 lock rc=0 + local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK + STATE=$state + lock="$state/.pending-reply-$corr.lock" + . "$_FM_PENDING_REPLY_LIB_DIR/fm-wake-lib.sh" + fm_lock_acquire_wait "$lock" || return 1 + _fm_pending_reply_reset_known_undelivered_locked "$@" || rc=$? + fm_lock_release "$lock" + return "$rc" +} + +_fm_pending_reply_reset_known_undelivered_locked() { # + local state=$1 corr=$2 rec delivered phase marker entry + rec=$(fm_pending_reply_path "$state" "$corr") + [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -z "$delivered" ] || return 1 + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in awaiting_report|delivery_unknown) ;; *) return 1 ;; esac + marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") + [ -e "$marker" ] || [ -L "$marker" ] || { + [ "$phase" = awaiting_report ] + return $? + } + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + entry=$(cat "$marker" 2>/dev/null || true) + case "$entry" in attempted=*) ;; *) return 1 ;; esac + [ "$phase" = awaiting_report ] \ + || fm_pending_reply_set "$rec" phase awaiting_report || return 1 + rm -f -- "$marker" +} + # Drop an undelivered expectation after a failed send so transport failure does # not masquerade as a missed report later. fm_pending_reply_discard_undelivered() { # @@ -1049,7 +1123,7 @@ _fm_pending_reply_maybe_escalate_locked() { # [ -f "$rec" ] || return 1 phase=$(fm_pending_reply_get "$rec" phase) if [ "$phase" = delivery_unknown ]; then - fm_pending_reply_reconcile_delivery "$state" "$corr" || true + _fm_pending_reply_reconcile_delivery_locked "$state" "$corr" || true phase=$(fm_pending_reply_get "$rec" phase) [ "$phase" = delivery_unknown ] || return 0 fi diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 512df6245c7..99594b03506 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -376,6 +376,14 @@ MARK_FROM_FIRSTMATE=0 PENDING_REPLY_CORR= PENDING_REPLY_CREATED=0 TARGET_TASK_ID= +fm_send_known_undelivered_cleanup() { + [ -n "$PENDING_REPLY_CORR" ] || return 0 + if [ "$PENDING_REPLY_CREATED" = 1 ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" + else + fm_pending_reply_reset_known_undelivered "$STATE" "$PENDING_REPLY_CORR" + fi +} if [ -n "$TARGET_SELECTOR" ] && [ -n "$TARGET_META" ] && [ "$(fm_meta_get "$TARGET_META" kind)" = secondmate ]; then MARK_FROM_FIRSTMATE=1 TARGET_TASK_ID=$(fm_send_id_from_meta "$TARGET_META") @@ -548,9 +556,14 @@ else PENDING_REPLY_CREATED=1 fi fm_pending_reply_embed_corr "$MESSAGE" "$PENDING_REPLY_CORR" MESSAGE - if [ "$PENDING_REPLY_CREATED" = 1 ] \ - && ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + if [ "$PENDING_REPLY_CREATED" != 1 ] \ + && fm_pending_reply_delivery_attempt_unresolved "$STATE" "$PENDING_REPLY_CORR"; then + echo "error: pending-reply delivery for $TARGET_TASK_ID is unresolved; refusing to resend correlation $PENDING_REPLY_CORR" >&2 + exit 1 + fi + if ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then + [ "$PENDING_REPLY_CREATED" != 1 ] \ + || fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true echo "error: failed to durably prepare pending-reply delivery for $TARGET_TASK_ID" >&2 exit 1 fi @@ -608,9 +621,8 @@ else echo "error: text delivery to remote secondmate $TARGET_REMOTE_ID is unknown; do not resend - same-host reconciliation is required" >&2 exit 1 fi - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + fm_send_known_undelivered_cleanup || \ + echo "error: known-undelivered pending-reply state could not be reset for $TARGET_TASK_ID" >&2 echo "error: text not sent to $T ($TARGET_BACKEND send failed; tried $RESOLUTION_TRIED)" >&2 exit 1 fi @@ -618,9 +630,8 @@ else empty) ;; send-failed) - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + fm_send_known_undelivered_cleanup || \ + echo "error: known-undelivered pending-reply state could not be reset for $TARGET_TASK_ID" >&2 echo "error: text not sent to $T ($TARGET_BACKEND send failed; tried $RESOLUTION_TRIED)" >&2 exit 1 ;; diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index c84669d3632..d83ce4d0567 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -50,8 +50,12 @@ # is the approved discard path that prevalidates child removal targets, locks each # descendant home's task set before enumeration, and holds those locks through # child cleanup. Contention refuses the complete forced teardown before child -# mutation. It then discards child work, kills child runtime endpoints, and removes -# the retired home. Removing a leased home releases its durable treehouse lease so the pool slot is freed, +# mutation. Local and remote retirement serialize their destructive phase with +# that mate's backlog-handoff lock under the registry lock. Pending handoff wake +# state is retired with the home, and local removal failure restores that state +# before preserving the route for retry. Teardown then discards child work, kills +# child runtime endpoints, and removes the retired home. Removing a leased home +# releases its durable treehouse lease so the pool slot is freed, # never left leased forever. If the treehouse return fails, teardown leaves the # leased home and state in place instead of hiding a still-held lease. # Usage: fm-teardown.sh [--force] @@ -166,6 +170,8 @@ SUB_HOME_PARENT_MARKER=".fm-secondmate-parent" . "$SCRIPT_DIR/fm-secondmate-parent-lib.sh" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-pending-reply-lib.sh +. "$SCRIPT_DIR/fm-pending-reply-lib.sh" # shellcheck source=bin/fm-nm-run-lib.sh . "$SCRIPT_DIR/fm-nm-run-lib.sh" if [ "$#" -lt 1 ] || ! fm_task_id_path_safe "$1"; then @@ -194,6 +200,18 @@ teardown_release_locks() { fm_lock_release "${DESCENDANT_LOCK_PATHS[$i]}" || true done DESCENDANT_LOCK_PATHS=() + if [ -n "${HANDOFF_WAKE_RETIRE_LOCK:-}" ]; then + fm_lock_release "$HANDOFF_WAKE_RETIRE_LOCK" || true + HANDOFF_WAKE_RETIRE_LOCK= + fi + if [ -n "${LOCAL_HANDOFF_LOCK:-}" ]; then + fm_lock_release "$LOCAL_HANDOFF_LOCK" || true + LOCAL_HANDOFF_LOCK= + fi + if [ -n "${LOCAL_REGISTRY_LOCK:-}" ]; then + fm_lock_release "$LOCAL_REGISTRY_LOCK" || true + LOCAL_REGISTRY_LOCK= + fi if [ "$META_LOCK_HELD" = 1 ]; then fm_lock_release "$META_LOCK" || true META_LOCK_HELD=0 @@ -230,6 +248,208 @@ REMOTE_PENDING_DIR_REAL= REMOTE_HANDOFF_LOCK= REMOTE_REGISTRY_LOCK= REMOTE_REPLY_LIFECYCLE_LOCK= +LOCAL_HANDOFF_LOCK= +LOCAL_REGISTRY_LOCK= +HANDOFF_WAKE_RETIRE_MARKER= +HANDOFF_WAKE_RETIRE_VALUE= +HANDOFF_WAKE_RETIRE_CORR= +HANDOFF_WAKE_RETIRE_LOCK= +HANDOFF_WAKE_RETIRE_STAGE= + +handoff_wake_retire_validate() { + local marker="$STATE/.backlog-handoff-$ID.wake-pending" value corr rec confirmation + HANDOFF_WAKE_RETIRE_MARKER= + HANDOFF_WAKE_RETIRE_VALUE= + HANDOFF_WAKE_RETIRE_CORR= + [ -e "$marker" ] || [ -L "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] || { + echo "REFUSED: receiver wake state for secondmate $ID is unsafe" >&2 + return 1 + } + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + pending|confirmed) ;; + prepared:*) + corr=${value#prepared:} + corr=${corr%%:*} + printf '%s' "$value" | grep -Eq '^prepared:[a-f0-9]{16}:[a-f0-9]{16}$' || { + echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 + return 1 + } + ;; + pending:*|confirmed:*) + corr=${value#*:} + printf '%s' "$corr" | grep -Eq '^[a-f0-9]{16}$' || { + echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 + return 1 + } + ;; + *) + echo "REFUSED: receiver wake state for secondmate $ID is invalid" >&2 + return 1 + ;; + esac + if [ -n "$corr" ]; then + rec=$(fm_pending_reply_path "$STATE" "$corr") + if [ -e "$rec" ] || [ -L "$rec" ]; then + [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$ID" ] || { + echo "REFUSED: receiver wake correlation for secondmate $ID is unsafe or belongs to another task" >&2 + return 1 + } + fi + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") + if [ -e "$confirmation" ] || [ -L "$confirmation" ]; then + [ -f "$confirmation" ] && [ ! -L "$confirmation" ] || { + echo "REFUSED: receiver wake delivery state for secondmate $ID is unsafe" >&2 + return 1 + } + fi + HANDOFF_WAKE_RETIRE_CORR=$corr + fi + HANDOFF_WAKE_RETIRE_MARKER=$marker + HANDOFF_WAKE_RETIRE_VALUE=$value +} + +handoff_wake_retire() { + local marker=$HANDOFF_WAKE_RETIRE_MARKER corr=$HANDOFF_WAKE_RETIRE_CORR lock rec confirmation rc=0 + [ -n "$marker" ] || return 0 + [ -f "$marker" ] && [ ! -L "$marker" ] \ + && [ "$(cat "$marker" 2>/dev/null || true)" = "$HANDOFF_WAKE_RETIRE_VALUE" ] || return 1 + if [ -n "$corr" ]; then + lock="$STATE/.pending-reply-$corr.lock" + fm_lock_acquire_wait "$lock" || return 1 + rec=$(fm_pending_reply_path "$STATE" "$corr") + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") + if { [ ! -e "$rec" ] && [ ! -L "$rec" ]; } \ + || { [ -f "$rec" ] && [ ! -L "$rec" ] \ + && [ "$(fm_pending_reply_get "$rec" task_id)" = "$ID" ]; }; then + rm -f -- "$confirmation" "$rec" "$marker" || rc=$? + else + rc=1 + fi + fm_lock_release "$lock" + return "$rc" + fi + rm -f -- "$marker" +} + +handoff_wake_retire_stage_restore() { + local stage=$HANDOFF_WAKE_RETIRE_STAGE marker rec confirmation name destination + [ -n "$stage" ] || return 0 + marker="$STATE/.backlog-handoff-$ID.wake-pending" + rec= + confirmation= + if [ -n "$HANDOFF_WAKE_RETIRE_CORR" ]; then + rec=$(fm_pending_reply_path "$STATE" "$HANDOFF_WAKE_RETIRE_CORR") + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$HANDOFF_WAKE_RETIRE_CORR") + fi + for name in record confirmation marker; do + [ -e "$stage/$name" ] || continue + case "$name" in + record) destination=$rec ;; + confirmation) destination=$confirmation ;; + marker) destination=$marker ;; + esac + [ -n "$destination" ] && [ ! -e "$destination" ] && [ ! -L "$destination" ] \ + && mv -- "$stage/$name" "$destination" || return 1 + done + rm -f -- "$stage/corr" || return 1 + rmdir -- "$stage" || return 1 + if [ -n "$HANDOFF_WAKE_RETIRE_LOCK" ]; then + fm_lock_release "$HANDOFF_WAKE_RETIRE_LOCK" || return 1 + HANDOFF_WAKE_RETIRE_LOCK= + fi + HANDOFF_WAKE_RETIRE_STAGE= +} + +handoff_wake_retire_stage_commit() { + local stage=$HANDOFF_WAKE_RETIRE_STAGE retired + [ -n "$stage" ] || return 0 + retired="$stage.retired.$$" + [ ! -e "$retired" ] && [ ! -L "$retired" ] || return 1 + mv -- "$stage" "$retired" || return 1 + HANDOFF_WAKE_RETIRE_STAGE= + if [ -n "$HANDOFF_WAKE_RETIRE_LOCK" ]; then + fm_lock_release "$HANDOFF_WAKE_RETIRE_LOCK" || return 1 + HANDOFF_WAKE_RETIRE_LOCK= + fi + rm -rf -- "$retired" || echo "warning: retired receiver wake state remains at $retired" >&2 +} + +handoff_wake_retire_stage_recover() { + local home=$1 stage="$STATE/.backlog-handoff-$ID.wake-retiring" corr + [ -e "$stage" ] || [ -L "$stage" ] || return 0 + [ -d "$stage" ] && [ ! -L "$stage" ] || { + echo "REFUSED: receiver wake retirement state for secondmate $ID is unsafe" >&2 + return 1 + } + if [ ! -e "$stage/corr" ] && [ ! -L "$stage/corr" ]; then + rmdir -- "$stage" 2>/dev/null && return 0 + echo "REFUSED: receiver wake retirement state for secondmate $ID is incomplete" >&2 + return 1 + fi + [ -f "$stage/corr" ] && [ ! -L "$stage/corr" ] || { + echo "REFUSED: receiver wake retirement state for secondmate $ID is unsafe" >&2 + return 1 + } + corr=$(cat "$stage/corr" 2>/dev/null || true) + [ -z "$corr" ] || printf '%s' "$corr" | grep -Eq '^[a-f0-9]{16}$' || { + echo "REFUSED: receiver wake retirement correlation for secondmate $ID is invalid" >&2 + return 1 + } + local staged + for staged in "$stage/marker" "$stage/record" "$stage/confirmation"; do + [ ! -e "$staged" ] && [ ! -L "$staged" ] && continue + [ -f "$staged" ] && [ ! -L "$staged" ] || { + echo "REFUSED: receiver wake retirement state for secondmate $ID is unsafe" >&2 + return 1 + } + done + HANDOFF_WAKE_RETIRE_CORR=$corr + HANDOFF_WAKE_RETIRE_STAGE=$stage + if [ -n "$corr" ]; then + HANDOFF_WAKE_RETIRE_LOCK="$STATE/.pending-reply-$corr.lock" + fm_lock_acquire_wait "$HANDOFF_WAKE_RETIRE_LOCK" || return 1 + fi + if [ -e "$home" ] || [ -L "$home" ]; then + handoff_wake_retire_stage_restore + else + handoff_wake_retire_stage_commit + fi +} + +handoff_wake_retire_stage() { + local stage="$STATE/.backlog-handoff-$ID.wake-retiring" marker=$HANDOFF_WAKE_RETIRE_MARKER + local corr=$HANDOFF_WAKE_RETIRE_CORR rec confirmation + [ -n "$marker" ] || return 0 + [ ! -e "$stage" ] && [ ! -L "$stage" ] || return 1 + (umask 077; mkdir -- "$stage") || return 1 + HANDOFF_WAKE_RETIRE_STAGE=$stage + printf '%s\n' "$corr" > "$stage/corr" || { handoff_wake_retire_stage_restore || true; return 1; } + if [ -n "$corr" ]; then + HANDOFF_WAKE_RETIRE_LOCK="$STATE/.pending-reply-$corr.lock" + fm_lock_acquire_wait "$HANDOFF_WAKE_RETIRE_LOCK" || { + HANDOFF_WAKE_RETIRE_LOCK= + handoff_wake_retire_stage_restore || true + return 1 + } + rec=$(fm_pending_reply_path "$STATE" "$corr") + confirmation=$(fm_pending_reply_delivery_confirmation_path "$STATE" "$corr") + if [ -e "$rec" ] && ! mv -- "$rec" "$stage/record"; then + handoff_wake_retire_stage_restore || true + return 1 + fi + if [ -e "$confirmation" ] && ! mv -- "$confirmation" "$stage/confirmation"; then + handoff_wake_retire_stage_restore || true + return 1 + fi + fi + if ! mv -- "$marker" "$stage/marker"; then + handoff_wake_retire_stage_restore || true + return 1 + fi +} remote_teardown_locks_release() { if [ -n "$REMOTE_REPLY_LIFECYCLE_LOCK" ]; then @@ -342,6 +562,7 @@ remote_secondmate_teardown() { [ "$route_host" = "$remote_host" ] && [ "$route_root" = "$remote_root" ] && [ "$route_home" = "$remote_home" ] \ || { echo "REFUSED: remote secondmate metadata does not match its registry route" >&2; return 1; } [ -z "$FORCE" ] || [ "$FORCE" = --force ] || { echo "error: invalid teardown option: $FORCE" >&2; return 2; } + handoff_wake_retire_validate || return 1 remote_recovery_paths_validate initial || return 1 if [ "$FORCE" != --force ] && [ "$REMOTE_OUTBOX_PRESENT" -eq 1 ]; then echo "REFUSED: remote secondmate $ID still has a pending backlog outbox; deliver it or explicitly discard with --force" >&2 @@ -391,6 +612,8 @@ remote_secondmate_teardown() { fi remote_pending_replies_cleanup \ || { echo "error: remote pending-reply cleanup failed; preserving the local route for retry" >&2; return 1; } + handoff_wake_retire \ + || { echo "error: remote receiver wake cleanup failed; preserving the local route for retry" >&2; return 1; } tmp="$SECONDMATE_REG.tmp.$$" grep -vE "^- $ID( |$)" "$SECONDMATE_REG" > "$tmp" || true mv -f -- "$tmp" "$SECONDMATE_REG" @@ -2267,21 +2490,30 @@ cleanup_firstmate_home_children() { } remove_secondmate_registry_entry() { - local id=$1 tmp lock rc=0 + local id=$1 tmp lock rc=0 acquired=0 [ -f "$SECONDMATE_REG" ] || return 0 lock=$(secondmate_registry_lock_path "$STATE") - fm_lock_acquire_wait "$lock" || return 1 + if [ "$LOCAL_REGISTRY_LOCK" != "$lock" ]; then + fm_lock_acquire_wait "$lock" || return 1 + acquired=1 + fi tmp="$SECONDMATE_REG.tmp.$$" grep -vE "^- $id( |$)" "$SECONDMATE_REG" > "$tmp" || true mv "$tmp" "$SECONDMATE_REG" || rc=$? - fm_lock_release "$lock" + [ "$acquired" -eq 0 ] || fm_lock_release "$lock" return "$rc" } validate_pr_poll_cleanup "$STATE" "$ID" || exit 1 if [ "$KIND" = secondmate ]; then + LOCAL_REGISTRY_LOCK=$(secondmate_registry_lock_path "$STATE") + fm_lock_acquire_wait "$LOCAL_REGISTRY_LOCK" || exit 1 + LOCAL_HANDOFF_LOCK="$STATE/.backlog-handoff-$ID.lock" + fm_lock_acquire_wait "$LOCAL_HANDOFF_LOCK" || exit 1 [ -n "$HOME_PATH" ] || HOME_PATH=$WT + handoff_wake_retire_stage_recover "$HOME_PATH" || exit 1 + handoff_wake_retire_validate || exit 1 validate_firstmate_home_for_removal "$HOME_PATH" "secondmate home" "$ID" >/dev/null || exit 1 if [ "$FORCE" = "--force" ]; then validate_firstmate_home_children_removal "$HOME_PATH" || exit 1 @@ -2542,7 +2774,18 @@ if [ "$BACKEND" = herdr ]; then fi if [ "$KIND" = secondmate ]; then [ -n "$HOME_PATH" ] || HOME_PATH=$WT - remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID" || exit $? + handoff_wake_retire_stage \ + || { echo "error: receiver wake cleanup could not be staged; preserving the secondmate home and route" >&2; exit 1; } + if remove_firstmate_home "$HOME_PATH" "secondmate home" "$ID"; then + : + else + rc=$? + handoff_wake_retire_stage_restore \ + || echo "error: receiver wake restoration failed; recovery state remains at $HANDOFF_WAKE_RETIRE_STAGE" >&2 + exit "$rc" + fi + handoff_wake_retire_stage_commit \ + || { echo "error: receiver wake cleanup failed; preserving the secondmate route for retry" >&2; exit 1; } remove_secondmate_registry_entry "$ID" fi remove_grok_turnend_auth "$STATE" "$ID" || exit 1 diff --git a/bin/fm-wake-drain.sh b/bin/fm-wake-drain.sh index 203765be80f..14599aaf8da 100755 --- a/bin/fm-wake-drain.sh +++ b/bin/fm-wake-drain.sh @@ -298,6 +298,10 @@ if [ -n "$ACK_THROUGH" ]; then awk -F '\t' -v cutoff="$ACK_THROUGH" ' NF < 5 || $2 !~ /^[0-9]+$/ || $2 > cutoff { print } ' "$FM_WAKE_QUEUE" > "$DRAIN_TMP" || exit 1 + fm_wake_commit_secondmate_stall_receipts_through "$ACK_THROUGH" || { + echo "wake drain: secondmate stall receipt could not be recorded safely" >&2 + exit 1 + } if [ ! -s "$DRAIN_TMP" ]; then fm_recovery_marker_ack "$RECOVERY_MARKER" "$ACK_GENERATION" RECOVERY_ACK_STATUS=$? diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 28249b661f3..8ce2195ac8d 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -1176,6 +1176,69 @@ fm_wake_queued_keys_locked() { "$FM_WAKE_QUEUE" 2>/dev/null || true } +fm_wake_secondmate_stall_marker_write() { # + local task=$1 row_key=$2 marker tmp + case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$row_key" in ''|*[!0-9-]*) return 1 ;; esac + marker="$STATE/.secondmate-wake-stall-$task" + if [ -e "$marker" ] || [ -L "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + fi + tmp=$(mktemp "$STATE/.secondmate-wake-stall.XXXXXX") || return 1 + if ! printf '%s\n' "$row_key" > "$tmp" || ! chmod 0600 "$tmp" \ + || ! _fm_atomic_replace "$tmp" "$marker"; then + rm -f -- "$tmp" + return 1 + fi +} + +fm_wake_secondmate_stall_receipt_write() { # + local task=$1 row_key=$2 root task_dir receipt tmp + case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$row_key" in ''|*[!0-9-]*) return 1 ;; esac + root="$STATE/.secondmate-wake-stall-receipts" + task_dir="$root/$task" + if [ -e "$root" ] || [ -L "$root" ]; then + [ -d "$root" ] && [ ! -L "$root" ] || return 1 + else + mkdir "$root" || return 1 + chmod 0700 "$root" || return 1 + fi + if [ -e "$task_dir" ] || [ -L "$task_dir" ]; then + [ -d "$task_dir" ] && [ ! -L "$task_dir" ] || return 1 + else + mkdir "$task_dir" || return 1 + chmod 0700 "$task_dir" || return 1 + fi + receipt="$task_dir/$row_key" + [ "$(cat "$receipt" 2>/dev/null || true)" != "$row_key" ] || return 0 + tmp=$(mktemp "$task_dir/.receipt.XXXXXX") || return 1 + if ! printf '%s\n' "$row_key" > "$tmp" || ! chmod 0600 "$tmp" \ + || ! _fm_atomic_replace "$tmp" "$receipt"; then + rm -f -- "$tmp" + return 1 + fi +} + +fm_wake_commit_secondmate_stall_receipts_through() { # + local cutoff=$1 key seq rest epoch task row_key + while IFS= read -r key; do + seq=${key##*-} + rest=${key%-*} + epoch=${rest##*-} + task=${rest#secondmate-wake-loop-} + task=${task%-"$epoch"} + case "$seq" in ''|*[!0-9]*) return 1 ;; esac + case "$epoch" in ''|*[!0-9]*) return 1 ;; esac + case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + row_key="$epoch-$seq" + fm_wake_secondmate_stall_receipt_write "$task" "$row_key" || return 1 + done < <(awk -F '\t' -v cutoff="$cutoff" ' + NF >= 5 && $2 ~ /^[0-9]+$/ && $2 <= cutoff && $3 == "check" \ + && $4 ~ /^secondmate-wake-loop-[A-Za-z0-9._-]+-[0-9]+-[0-9]+$/ { print $4 } + ' "$FM_WAKE_QUEUE" 2>/dev/null) +} + fm_wake_restore_queue() { local drained=$1 restore restore="$STATE/.wake-queue.restore.$(fm_current_pid)" diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index d1d59d3ceb5..1d883caf8bb 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -68,6 +68,11 @@ # check: inactive-outcome bounded poll-loop reconciliation found a suspicious # inactive terminal outcome that still lacks its durable # upstream receipt +# check: secondmate wake-loop stalled: mate= row= age=s +# the oldest valid row in an endpoint-recorded local +# secondmate home's durable wake queue exceeded +# FM_SECONDMATE_WAKE_STALL_SECS; observation is read-only +# and one parent receipt suppresses repeats for that row # For normal supervision, resume the session-start primary-harness protocol # after each printed reason. Direct duplicate invocations of this script still # no-op through the watcher singleton lock. @@ -174,6 +179,9 @@ STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provabl # turn-ended and resets the age. Set generously above any legitimate interval # between completed turns, including long tool calls, builds, or test runs. BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600} +# A local secondmate's foreign queue is checked on every poll, but only after this +# bounded age can it produce a parent notification. +SECONDMATE_WAKE_STALL_SECS=${FM_SECONDMATE_WAKE_STALL_SECS:-60} # A crew that declared a pause is idling on a known external wait, so its stale # pane is absorbed rather than wedge-escalated. # A captain-held or paused crew whose agent has confidently exited uses the same @@ -294,6 +302,85 @@ recorded_windows() { done } +# Print the oldest structurally valid row in a local secondmate's foreign queue. +# This is a read-only observation: the receiving home owns acknowledgement and +# this parent never changes the row or the foreign queue. +secondmate_oldest_queue_row() { # + local queue=$1 + [ -f "$queue" ] && [ ! -L "$queue" ] || return 0 + awk -F '\t' ' + NF >= 5 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ { + if (!found || $2 < seq) { + found = 1 + seq = $2 + row = $0 + } + } + END { if (found) print row } + ' "$queue" 2>/dev/null || true +} + +# Surface one durable parent check for one unchanged foreign row after its +# bounded age. The primary marker and queued-key check make repeated watcher +# cycles converge without a notification storm, while an empty queue removes +# only this home's marker so a later row can be observed. +secondmate_wake_stall_tick() { + local now=$(( $(date +%s) )) threshold=$SECONDMATE_WAKE_STALL_SECS + local meta task kind remote_host home queue row epoch seq row_key marker receipt receipt_dir notify_key queued age reason + case "$threshold" in ''|*[!0-9]*|0) threshold=60 ;; esac + # Endpoint metadata admits this queue-loop check; secondmate-liveness owns registered mates whose endpoint is missing or dead. + for meta in "$STATE"/*.meta; do + [ -e "$meta" ] || continue + kind=$(fm_meta_get "$meta" kind) + [ "$kind" = secondmate ] || continue + remote_host=$(fm_meta_get "$meta" remote_host) + [ -z "$remote_host" ] || continue + task=${meta##*/} + task=${task%.meta} + case "$task" in ''|*[!A-Za-z0-9._-]*) continue ;; esac + home=$(fm_meta_get "$meta" home) + [ -n "$home" ] || continue + [ -f "$home/.fm-secondmate-home" ] && [ ! -L "$home/.fm-secondmate-home" ] || continue + [ "$(cat "$home/.fm-secondmate-home" 2>/dev/null || true)" = "$task" ] || continue + queue="$home/state/.wake-queue" + row=$(secondmate_oldest_queue_row "$queue") + marker="$STATE/.secondmate-wake-stall-$task" + receipt_dir="$STATE/.secondmate-wake-stall-receipts/$task" + if [ -z "$row" ]; then + rm -f "$marker" + if [ -e "$receipt_dir" ] || [ -L "$receipt_dir" ]; then + [ -d "$receipt_dir" ] && [ ! -L "$receipt_dir" ] || return 1 + rm -rf -- "$receipt_dir" || return 1 + fi + continue + fi + IFS=$(printf '\t') read -r epoch seq _row_kind _row_key _row_payload </dev/null || true)" = "$row_key" ] && continue + [ "$(cat "$receipt" 2>/dev/null || true)" = "$row_key" ] && continue + notify_key="secondmate-wake-loop-$task-$row_key" + reason="check: secondmate wake-loop stalled: mate=$task row=$seq age=${age}s" + queued=$(fm_wake_queued_keys check) + if ! printf '%s\n' "$queued" | grep -Fx "$notify_key" >/dev/null 2>&1; then + fm_wake_append check "$notify_key" "$reason" || return 1 + fi + fm_wake_secondmate_stall_receipt_write "$task" "$row_key" || return 1 + fm_wake_secondmate_stall_marker_write "$task" "$row_key" || return 1 + wake "$reason" + done + return 0 +} + # Consecutive wedge-escalation count for a window past FM_WEDGE_DEMAND_INSPECT_COUNT # (default 3): a pane that keeps re-wedging on the SAME stale hash - each # escalation gets absorbed again as "still validating" one poll later, since the @@ -971,6 +1058,14 @@ while :; do # No conversation scraping; unresolved records are never silently expired. fm_pending_reply_tick "$STATE" || true + # A live secondmate endpoint does not prove that its own wake loop is alive. + # Observe the foreign queue before the rest of this cycle so an aged row wakes + # the parent without consuming or rewriting the receiving home's record. + secondmate_wake_stall_tick || { + echo "watcher: secondmate wake-loop observation failed" >&2 + exit 1 + } + # Process-to-event liveness repair. This never discovers a result by polling: # each registered source has its own child blocking on that source, and this # only republishes results already captured durably and restarts a source diff --git a/docs/architecture.md b/docs/architecture.md index 0f1cf8dd9ac..e0b5affa08e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,11 +14,15 @@ Repeated provably-working stale escalations on the same unchanged pane add an es A pane holding a file newer than the start of its own quiet window, anywhere in the worktree recorded for that task, is deferred instead of escalated, because a crew writing source, then tests, then documentation behind a static pane is liveness that neither pane quietness nor the run step can show. That deferral re-surfaces on the same `FM_PAUSE_RESURFACE_SECS` cadence as a declared wait, with a reason naming the write evidence rather than a wedge, and it is bounded to one pruned, depth-bounded, wall-clock-bounded walk (`FM_WORKTREE_WRITE_PRUNE`, `FM_WORKTREE_WRITE_MAXDEPTH`, `FM_WORKTREE_WRITE_TIMEOUT`) taken only in the branch that was about to escalate, never on every poll. Every absence of write evidence, including a missing worktree record, a torn-down worktree, a walk that outlives its wall-clock bound on a hung mount, and a failed walk, leaves the existing escalation schedule untouched, so a crew that writes nothing still escalates exactly as before. -A secondmate is never probed at all, because the worktree recorded for it is a provisioned firstmate home whose own supervision keeps writing inside it whether or not the mate produces anything, so its panes keep escalating on the unchanged schedule. +A secondmate's recorded worktree is never probed for write activity, because it is a provisioned firstmate home whose own supervision keeps writing inside it whether or not the mate produces anything, so its panes keep escalating on the unchanged schedule. A busy pane is otherwise exempt from staleness, but only until its latest `state/.turn-ended` marker reaches `FM_BUSY_TURN_MAX_SECS`, or its `state/.meta` spawn record reaches that age before any turn completes; past that bound it is routed through the same wedge escalation, with the identical reason, escalation count, worktree-write deferral, and `demand-deep-inspection` marker, for inspection only - never an automatic interrupt, signal, or restart. A crew that declared an external wait (`paused:`) or a verified captain-held transfer is the one exception to that bound: its busy verdict supplies liveness while identifying the long-running foreground call as the declared wait, so it takes the bounded `FM_PAUSE_RESURFACE_SECS` recheck instead of a wedge escalation. Lifting the declaration restores the unchanged busy-pane wedge path, while a pane that is no longer busy returns to the existing idle declared-wait classification. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) only after generation-bound recovery evidence is published, so an interrupted watcher or handling turn can be recovered without losing the queue record. +Agent endpoint liveness and queue-consumption liveness are separate: on each poll, the primary watcher reads the oldest valid row from every endpoint-recorded local secondmate home's durable wake queue without locking, consuming, or rewriting that foreign queue. +Once that row reaches `FM_SECONDMATE_WAKE_STALL_SECS`, the primary appends one keyed `check` wake naming the mate, row sequence, and observed age; parent receipts and queued-key deduplication suppress repeats for the same row across watcher and handling crashes, while empty and younger queues remain silent. +Endpointless registered mates remain outside this scan because startup secondmate-liveness owns dead or missing endpoint recovery, and remote homes retain their host-local supervision boundary. +`tests/fm-wake-queue.test.sh` pins the notification, idempotence, quiet-queue, and byte-for-byte foreign-row preservation guarantees. When a canonical validated PR poll returns exactly `merged`, the watcher appends that durable notification before publishing a private receipt bound to the poll's registration, bytes, file identities, metadata, provider, URL, and task ID. The receipt makes retirement safely retryable across restarts: fixed-path recovery revalidates the same evidence, removes the runnable check first, removes its registration and data sidecars, removes the receipt last, and preserves task metadata including `pr=` and `pr_head=`. A concurrent replacement remains armed, every non-merged or invalid observation remains unchanged, and retirement never performs task or persistent-secondmate cleanup. @@ -220,9 +224,10 @@ Secondmates are idle by default: after startup recovery reconciles only work alr When called with `FM_HOME=` or when `FM_HOME` is already set to the active firstmate home, metadata-routed `fm-send.sh` requests to a live `kind=secondmate` use the live-charter-compatible `from-firstmate` carrier owned by `bin/fm-operational-input.sh`, so the secondmate returns terse answers through status lines and detailed answers through docs plus status pointers instead of replying only in its own chat. The parent guards every marked request against a missing correlated report without reading the secondmate conversation; `bin/fm-pending-reply-lib.sh` owns the correlation, recovery, escalation, and retention contract. Explicit backend-target sends and direct human typing stay unmarked, so captain intervention in a secondmate pane remains conversational. -After seeding a secondmate, `fm-backlog-handoff.sh` validates the fleet-specific handoff, then atomically delegates already-judged in-scope queued item moves to `tasks-axi mv` so the domain queue starts in the right place. -Remote routes move that dependency-closed set into a non-dispatchable backlog-format outbox before transfer, then use an idempotent remote receive under the destination backlog's own lock. -The outbox is the complete retry record, so no two-phase journal or transport-level retry is needed. +After seeding a secondmate, `fm-backlog-handoff.sh` validates the fleet-specific handoff, atomically delegates already-judged in-scope queued item moves to `tasks-axi mv`, and then sends a marked routed-work wake through the receiver's recorded endpoint. +A durable move with a missing, failed, or unresolved wake is reported as failure rather than success; rerunning the same handoff recovers known-undelivered wake intent without moving the item again, while an unresolved delivery is never blindly resent. +Remote routes move that dependency-closed set into a non-dispatchable backlog-format outbox before transfer, then use an idempotent remote receive under the destination backlog's own lock and retain the outbox until the receiver wake is confirmed. +The script header owns the wake correlation and recovery mechanics; `tests/fm-backlog-handoff.test.sh` and `tests/fm-remote-backlog-handoff.test.sh` pin the local and remote delivery boundaries. An unreachable remote host is unknown rather than dead, preserves its route and durable work, and is never failed over or relaunched locally. Idle secondmate panes are healthy; teardown is explicit and refuses while the secondmate home has in-flight work unless the captain has approved discard with `--force`. diff --git a/docs/configuration.md b/docs/configuration.md index df86ffb2798..ecf8c64b2f5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -37,7 +37,7 @@ This preference is local to each Firstmate home and is not part of secondmate in The tracked `.tasks.toml` pins the default `tasks-axi` markdown backend to `data/backlog.md`, with `done_keep = 10` and an archive at `data/done-archive.md`. When the default backend is selected and compatible `tasks-axi` is on `PATH`, firstmate uses its verbs for routine backlog mutations. -Secondmate handoffs are separate and unconditional: `fm-backlog-handoff.sh` keeps only its own fleet-level validation and always delegates the item move to `tasks-axi mv`, the single owner of the backlog format. +Secondmate handoffs bypass that routine-backend choice: `fm-backlog-handoff.sh` keeps only its own fleet-level validation, delegates the item move to `tasks-axi mv`, and requires a verified receiver wake after a new move becomes durable. It moves in-scope `## Queued` items only and refuses `## In flight` and historical `## Done` records, which stay with their home for pruning or archiving. Handoff item bodies must use at least two leading spaces, and the helper refuses a selected item with a single-space or tab-indented continuation rather than risk orphaning it. Because bootstrap requires `tasks-axi` on `PATH` on every profile, that delegation works fleet-wide, and the `config/backlog-backend=manual` knob governs firstmate's own hand-editing of its backlog, not this validated helper. @@ -191,7 +191,8 @@ The lease is held under the secondmate id until explicit retirement or seed roll Teardown of a leased home fails closed if `treehouse return` cannot release the lease; plain-clone homes with no treehouse pool slot are removed directly. Secondmate routes cover `no-mistakes` and `direct-PR` projects; `local-only` projects remain main-firstmate work. For `no-mistakes` projects, seeding initializes only projects newly cloned into a secondmate home and refuses to mutate a preexisting clone that is not already initialized. -After creating a secondmate, move existing main-backlog queued items that you have judged in-scope with `fm-backlog-handoff.sh ...`; it is idempotent and refuses In flight, Done, or non-secondmate homes. +After creating a secondmate, move existing main-backlog queued items that you have judged in-scope with `fm-backlog-handoff.sh ...`; it refuses In flight, Done, or non-secondmate homes, and a new move succeeds only after waking the recorded receiver. +If the wake is known to have failed, the moved item remains durable and rerunning the same handoff retries it idempotently; an unresolved delivery is reported and never blindly resent. Set `FM_SECONDMATE_CHARTER` to seed from inline charter text when no filled charter brief exists; set `FM_SECONDMATE_SCOPE` when the routing scope should differ from the charter text. The seeded home's `data/charter.md` owns the standard secondmate lifecycle and escalation contract; the route file points to it through the existing `home:` field instead of adding another pointer. Each seed writes an `.fm-secondmate-home` identity marker at the home root, alongside a durable `.fm-secondmate-parent` record of the home's route to its parent (see "Provision a route" in [`docs/remote-secondmates.md`](remote-secondmates.md)). @@ -692,6 +693,7 @@ FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart; a declared external wait or verified captain-held transfer takes the FM_PAUSE_RESURFACE_SECS recheck below instead FM_PAUSE_RESURFACE_SECS=3600 # seconds before the watcher re-surfaces a declared external wait or verified captain-held transfer for a recheck, including a live busy pane past FM_BUSY_TURN_MAX_SECS; the away-mode daemon uses the same setting for a declared external wait or verified captain-held transfer +FM_SECONDMATE_WAKE_STALL_SECS=60 # minimum age of the oldest valid foreign wake-queue row before an endpoint-recorded local secondmate produces one durable parent wake-loop-stall notification; zero or invalid values use 60 FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WORKTREE_WRITE_PRUNE='.git node_modules .venv venv __pycache__ .mypy_cache .pytest_cache .ruff_cache .tox target dist build .next .cache vendor' # directory names the wedge detector's task-worktree write probe skips; the default keeps .git out so a supervisor's own read-only git command can never look like crew progress; set it to the empty string to prune nothing, which widens the probe to the whole depth-bounded tree rather than disabling it FM_WORKTREE_WRITE_MAXDEPTH=6 # depth that same probe walks below the recorded worktree; it runs only at the moment a wedge escalation would otherwise fire, never on every poll; no probe knob applies to a secondmate, whose recorded worktree is a provisioned home the probe skips entirely diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index a8ee9556774..f8a9bbd701d 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -269,7 +269,7 @@ A structurally gone pane becomes `missing`, a restored agent-less shell becomes Unlike tmux process-name inspection, native registration can classify Pi without guessing from a generic interpreter name. The session-start sweep uses this probe. -Mid-session secondmate liveness is not implemented because idle secondmates are deliberately exempt from stale-pane escalation and need a separate periodic identity signal. +Mid-session secondmate agent-process liveness is not implemented because idle secondmates are deliberately exempt from stale-pane escalation and need a separate periodic identity signal. ## Push events and polling fallback @@ -320,7 +320,7 @@ Tests use thin compatibility wrappers in `tests/herdr-test-safety.sh` and never - Mutable labels can collide; they are never placement or destructive authority. - A Firstmate outside Herdr cannot resolve a launcher workspace, so a colliding home label refuses new spawns until the collision is cleared. - Ghost and placeholder recognition uses ANSI de-emphasis when available; an unstyled glyph row carrying trailing non-idle text fails safely to `unknown`. -- Mid-session secondmate liveness is not implemented. +- Mid-session secondmate agent-process liveness is not implemented. - Only tmux and Herdr can host the away-mode supervisor terminal. ## Regression entry points diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index c5f471875d5..b5ac0f9db3b 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -204,8 +204,8 @@ bin/fm-backlog-handoff.sh ... For a remote route, `tasks-axi mv` first moves the dependency-closed set atomically from the primary backlog into `data/handoff/.outbox.md`. The outbox is then copied to the remote handoff scratch directory and `fm-backlog-receive.sh` atomically ingests every destination-absent key under the remote backlog's own lock. -Confirmed receipt removes the outbox. -An existing outbox is the complete retry record, and `--resume-pending` safely re-delivers it. +After receipt, the helper sends a marked routed-work instruction through the recorded remote endpoint and removes the outbox only after that wake is confirmed. +A failed wake leaves the remote backlog intact and the outbox available for `--resume-pending`; an unresolved send is reported without a blind resend. Bootstrap retries pending outboxes and emits `SECONDMATE_HANDOFF:` only when one remains. There is no two-phase journal and no additional tasks-axi release requirement. diff --git a/docs/scripts.md b/docs/scripts.md index 3359c32e6c8..c724eabb9c0 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -24,7 +24,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-remote-job-worker.sh` | Long-lived remote queue worker for tracked `fm-*.sh` commands in the account runtime | | `fm-remote-job-reap-orphans.sh` | Stop remote job workers left running by a pruned code root, never one whose checkout still exists | | `fm-remote-doctor.sh` | Check, and with `--fix` repair, one remote account's second-mate readiness (remote job worker, Herdr, Aqua launch agents, PATH, and required tools) | -| `fm-backlog-handoff.sh` | Validate and delegate queued backlog-item moves into a secondmate home | +| `fm-backlog-handoff.sh` | Move queued backlog items into a secondmate home and durably wake its recorded receiver | | `fm-backlog-receive.sh` | Idempotently ingest one confined remote handoff outbox through tasks-axi | | `fm-captain-hold.sh` | Hold tasks for the captain, record the captain's answers, gate investigation completion, and report record divergence between the status log and the backlog | | `fm-decision-hold.sh` | One-release compatibility shim mapping the retired decision commands onto fm-captain-hold.sh | @@ -72,7 +72,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-gate-refuse-lib.sh` | Shared no-mistakes gate-context refusal for fleet lifecycle entrypoints | | `fm-watch-arm.sh` | Verified home-scoped watcher arm wrapper with loud cycle endings and bounded lifecycle ledger | | `fm-watch-checkpoint.sh` | Run one bounded foreground watcher checkpoint for Codex-style supervision | -| `fm-watch.sh` | Singleton-safe always-on watcher: absorb benign wakes, queue and exit on actionable ones | +| `fm-watch.sh` | Singleton-safe watcher: absorb benign wakes, detect stalled local-secondmate wake queues, and exit on actionable ones | | `fm-inactive-reconcile.sh` | Reconcile long-inactive direct crewmate terminal outcomes without forge access | | `fm-afk-start.sh` | Run the common sourceable away-mode daemon entry in the foreground | | `fm-afk-launch.sh` | Own away-mode entry, exit, rollback, and any backend terminal lifecycle | diff --git a/tests/fm-backlog-handoff.test.sh b/tests/fm-backlog-handoff.test.sh index 94b50f8a647..e0194495c18 100755 --- a/tests/fm-backlog-handoff.test.sh +++ b/tests/fm-backlog-handoff.test.sh @@ -14,6 +14,12 @@ set -u command -v tasks-axi >/dev/null 2>&1 || { echo "skip: tasks-axi not found (required by the delegated handoff path)"; exit 0; } TMP_ROOT=$(fm_test_tmproot fm-backlog-handoff) +HANDOFF_FAKEBIN=$(make_fake_tmux "$TMP_ROOT/default-fake") +export PATH="$HANDOFF_FAKEBIN:$PATH" +export FM_FAKE_TMUX_WINDOW='firstmate:fm-design' +export FM_FAKE_TMUX_LOG="$TMP_ROOT/default-tmux.log" +export FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/default-fake/pane.txt" +export FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 setup_homes() { local home=$1 subhome=$2 id=${3:-design} @@ -23,6 +29,660 @@ setup_homes() { sub_abs=$(cd "$subhome" && pwd -P) printf -- '- %s - feature work (home: %s; scope: feature work; projects: alpha; added 2026-07-09)\n' \ "$id" "$sub_abs" > "$home/data/secondmates.md" + cat > "$home/state/$id.meta" < "$home/state/design.meta" < "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] wake-item - routed to a live receiver (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + fakebin=$(make_fake_tmux "$TMP_ROOT/live-wake-fake") + out="$TMP_ROOT/live-wake.out" + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" PATH="$fakebin:$PATH" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/live-wake-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/live-wake-fake/pane.txt" \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 \ + "$ROOT/bin/fm-backlog-handoff.sh" design wake-item > "$out" 2>&1 \ + || fail "handoff to a live receiver failed: $(cat "$out")" + grep -F 'wake-item' "$sub/data/backlog.md" >/dev/null \ + || fail "live receiver did not receive the routed backlog item" + grep -F 'send-keys' "$TMP_ROOT/live-wake-tmux.log" >/dev/null \ + || fail "handoff did not wake the live receiver endpoint" + grep -F 'New routed work is in your backlog.' "$TMP_ROOT/live-wake-tmux.log" >/dev/null \ + || fail "receiver wake did not carry the routed-work instruction" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/live-wake-tmux.log") + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" PATH="$fakebin:$PATH" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/live-wake-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/live-wake-fake/pane.txt" \ + FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 \ + "$ROOT/bin/fm-backlog-handoff.sh" design wake-item > "$TMP_ROOT/live-wake-rerun.out" 2>&1 \ + || fail "idempotent successful handoff rerun failed: $(cat "$TMP_ROOT/live-wake-rerun.out")" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/live-wake-tmux.log")" -eq "$wake_count" ] \ + || fail "idempotent successful handoff rerun duplicated the receiver wake" + pass "a routed handoff wakes once and a successful rerun stays idempotent" +} + +test_failed_wake_retries_when_the_item_is_already_present() { + local home="$TMP_ROOT/retry-wake-main" sub="$TMP_ROOT/retry-wake-sub" out corr rc=0 + setup_homes "$home" "$sub" + rm -f "$home/state/design.meta" + mkdir -p "$sub/data" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] retry-item - wake must be retried (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + + out=$(FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design retry-item 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "handoff without a receiver endpoint reported success" + assert_contains "$out" "receiver was not woken" "missing receiver failure was not observable" + assert_grep 'retry-item' "$sub/data/backlog.md" "failed wake lost the durably handed-off item" + corr=$(cut -d: -f2- "$home/state/.backlog-handoff-design.wake-pending") + assert_absent "$home/state/pending-replies/.delivery-confirmed-$corr" \ + "missing endpoint was recorded as an attempted delivery" + + cat > "$home/state/design.meta" < "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design retry-item > "$TMP_ROOT/retry-wake.out" 2>&1 \ + || fail "an already-present handoff did not retry its receiver wake: $(cat "$TMP_ROOT/retry-wake.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "the recovery handoff did not retry delivery through the receiver endpoint" + pass "a failed receiver wake is loud and retries from an already-present handoff" +} + +test_known_receiver_failure_remains_retryable_after_grace() { + local home="$TMP_ROOT/known-fail-main" sub="$TMP_ROOT/known-fail-sub" + local basebin rejectbin="$TMP_ROOT/known-fail-reject" out corr phase rc=0 + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$rejectbin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] known-fail - retry after known receiver rejection (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + basebin=$(make_fake_tmux "$TMP_ROOT/known-fail-fake") + cat > "$rejectbin/tmux" <<'SH' +#!/usr/bin/env bash +[ "${1:-}" != send-keys ] || exit 1 +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$rejectbin/tmux" + + out=$(PATH="$rejectbin:$basebin:$PATH" FM_BASE_TMUX="$basebin/tmux" \ + FM_HOME="$home" FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/known-fail-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/known-fail-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design known-fail 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "known receiver rejection reported handoff success" + assert_grep 'known-fail' "$sub/data/backlog.md" "known receiver rejection lost the durable item" + corr=$(cut -d: -f2- "$home/state/.backlog-handoff-design.wake-pending") + assert_absent "$home/state/pending-replies/.delivery-confirmed-$corr" \ + "known receiver rejection retained an attempted-delivery marker" + FM_PENDING_REPLY_NOW=9999999999 bash -c ' + . "$1" + fm_pending_reply_reconcile_delivery "$2" "$3" >/dev/null 2>&1 || true + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$home/state" "$corr" + phase=$(sed -n 's/^phase=//p' "$home/state/pending-replies/$corr") + [ "$phase" = awaiting_report ] \ + || fail "known receiver rejection aged into unretryable phase $phase" + + : > "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design known-fail \ + > "$TMP_ROOT/known-fail-retry.out" 2>&1 \ + || fail "known receiver rejection did not retry: $(cat "$TMP_ROOT/known-fail-retry.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "known receiver rejection retry did not wake the receiver" + pass "a known receiver failure stays retryable after reconciliation grace" +} + +test_known_failure_restores_retry_after_reconciliation_race() { + local home="$TMP_ROOT/reconcile-race-main" sub="$TMP_ROOT/reconcile-race-sub" + local basebin blockbin="$TMP_ROOT/reconcile-race-block" handoff i corr phase + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$blockbin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] reconcile-race - retry after concurrent reconciliation (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + basebin=$(make_fake_tmux "$TMP_ROOT/reconcile-race-fake") + cat > "$blockbin/tmux" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = send-keys ]; then + touch "$FM_RECONCILE_RACE_ENTERED" + while [ ! -f "$FM_RECONCILE_RACE_RELEASE" ]; do sleep 0.02; done + exit 1 +fi +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$blockbin/tmux" + + PATH="$blockbin:$basebin:$PATH" FM_BASE_TMUX="$basebin/tmux" FM_HOME="$home" \ + FM_RECONCILE_RACE_ENTERED="$TMP_ROOT/reconcile-race.entered" \ + FM_RECONCILE_RACE_RELEASE="$TMP_ROOT/reconcile-race.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/reconcile-race-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/reconcile-race-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design reconcile-race \ + > "$TMP_ROOT/reconcile-race.out" 2>&1 & + handoff=$! + i=0 + while [ ! -f "$TMP_ROOT/reconcile-race.entered" ]; do + kill -0 "$handoff" 2>/dev/null || fail "reconciliation-race handoff exited before backend delivery" + i=$((i + 1)) + [ "$i" -le 250 ] || fail "reconciliation-race handoff never reached backend delivery" + sleep 0.02 + done + corr=$(cut -d: -f2- "$home/state/.backlog-handoff-design.wake-pending") + FM_PENDING_REPLY_NOW=9999999999 bash -c ' + . "$1" + fm_pending_reply_reconcile_delivery "$2" "$3" + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$home/state" "$corr" \ + || fail "concurrent watcher fixture did not reconcile the aged attempt" + phase=$(sed -n 's/^phase=//p' "$home/state/pending-replies/$corr") + [ "$phase" = delivery_unknown ] || fail "aged in-flight attempt did not become delivery_unknown" + touch "$TMP_ROOT/reconcile-race.release" + if wait "$handoff"; then + fail "known backend failure after reconciliation reported success" + fi + phase=$(sed -n 's/^phase=//p' "$home/state/pending-replies/$corr") + [ "$phase" = awaiting_report ] \ + || fail "known backend failure did not restore retryable phase after reconciliation" + assert_absent "$home/state/pending-replies/.delivery-confirmed-$corr" \ + "known backend failure retained its aged attempted marker" + + : > "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design reconcile-race \ + > "$TMP_ROOT/reconcile-race-retry.out" 2>&1 \ + || fail "reconciliation-race handoff did not retry: $(cat "$TMP_ROOT/reconcile-race-retry.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "reconciliation-race retry did not wake the receiver" + pass "known failure restores retryability after concurrent reconciliation" +} + +test_move_crash_keeps_wake_pending_for_recovery() { + local home="$TMP_ROOT/move-crash-main" sub="$TMP_ROOT/move-crash-sub" + local fakebin="$TMP_ROOT/move-crash-fakebin" real_tasks rc=0 prepared_state + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] crash-item - survive the post-move crash (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_tasks=$(command -v tasks-axi) + cat > "$fakebin/tasks-axi" <<'SH' +#!/usr/bin/env bash +"$FM_REAL_TASKS_AXI" "$@" +rc=$? +case " $* " in + *" --file "*" --to "*) + if [ "$rc" -eq 0 ] && [ "${1:-}" = mv ]; then + handoff_pid=$(ps -o ppid= -p "$PPID" | tr -d '[:space:]') + kill -KILL "$handoff_pid" + sleep 1 + fi + ;; +esac +exit "$rc" +SH + chmod +x "$fakebin/tasks-axi" + + set +e + FM_REAL_TASKS_AXI="$real_tasks" PATH="$fakebin:$PATH" FM_HOME="$home" \ + "$ROOT/bin/fm-backlog-handoff.sh" design crash-item > "$TMP_ROOT/move-crash.out" 2>&1 + rc=$? + set +e + [ "$rc" -ne 0 ] || fail "post-move crash fixture unexpectedly reported success" + assert_grep 'crash-item' "$sub/data/backlog.md" "post-move crash did not leave the item durable" + assert_present "$home/state/.backlog-handoff-design.wake-pending" \ + "post-move crash lost receiver wake intent" + prepared_state=$(cat "$home/state/.backlog-handoff-design.wake-pending") + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] unrelated-move - still waiting in the main backlog (repo: alpha) + +## Done +EOF + rc=0 + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design unrelated-move \ + > "$TMP_ROOT/move-crash-unrelated.out" 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "unrelated moving handoff discarded a post-move prepared wake" + assert_contains "$(cat "$TMP_ROOT/move-crash-unrelated.out")" \ + 'belongs to a different routed batch' \ + "unrelated handoff did not surface the unresolved prepared batch" + [ "$(cat "$home/state/.backlog-handoff-design.wake-pending")" = "$prepared_state" ] \ + || fail "unrelated moving handoff changed the post-move prepared wake" + assert_grep 'unrelated-move' "$home/data/backlog.md" \ + "unrelated moving handoff changed its source item before resolving the older wake" + assert_no_grep 'unrelated-move' "$sub/data/backlog.md" \ + "unrelated moving handoff moved work despite the unresolved older wake" + + : > "$TMP_ROOT/default-tmux.log" + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design crash-item \ + > "$TMP_ROOT/move-crash-retry.out" 2>&1 \ + || fail "post-move crash recovery failed: $(cat "$TMP_ROOT/move-crash-retry.out")" + assert_grep 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log" \ + "post-move crash recovery did not wake the receiver" + assert_absent "$home/state/.backlog-handoff-design.wake-pending" \ + "confirmed crash recovery left receiver wake pending" + pass "a post-move crash preserves wake intent for an idempotent retry" +} + +test_pre_move_crash_does_not_wake_until_move_lands() { + local home="$TMP_ROOT/pre-move-crash-main" sub="$TMP_ROOT/pre-move-crash-sub" + local fakebin="$TMP_ROOT/pre-move-crash-fakebin" real_tasks rc=0 wake_count + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] pre-move-crash - wake only after durable move (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_tasks=$(command -v tasks-axi) + cat > "$fakebin/tasks-axi" <<'SH' +#!/usr/bin/env bash +case " $* " in + *" --file "*" --to "*) + if [ "${1:-}" = mv ]; then + handoff_pid=$(ps -o ppid= -p "$PPID" | tr -d '[:space:]') + kill -KILL "$handoff_pid" + sleep 1 + fi + ;; +esac +exec "$FM_REAL_TASKS_AXI" "$@" +SH + chmod +x "$fakebin/tasks-axi" + : > "$TMP_ROOT/default-tmux.log" + + set +e + FM_REAL_TASKS_AXI="$real_tasks" PATH="$fakebin:$PATH" FM_HOME="$home" \ + "$ROOT/bin/fm-backlog-handoff.sh" design pre-move-crash > "$TMP_ROOT/pre-move-crash.out" 2>&1 + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "pre-move crash fixture unexpectedly reported success" + assert_grep 'pre-move-crash' "$home/data/backlog.md" "pre-move crash changed the source backlog" + assert_no_grep 'pre-move-crash' "$sub/data/backlog.md" "pre-move crash changed the destination backlog" + assert_present "$home/state/.backlog-handoff-design.wake-pending" \ + "pre-move crash lost its prepared wake intent" + + cat > "$sub/data/backlog.md" <<'EOF' +## Queued +- [ ] unrelated-ready - already durable from another handoff (repo: alpha) + +## Done +EOF + rc=0 + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design unrelated-ready \ + > "$TMP_ROOT/pre-move-unrelated.out" 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "unrelated handoff accepted another batch's prepared wake" + assert_contains "$(cat "$TMP_ROOT/pre-move-unrelated.out")" \ + 'belongs to a different routed batch' \ + "unrelated handoff did not report the prepared batch conflict" + [ ! -s "$TMP_ROOT/default-tmux.log" ] \ + || fail "unrelated already-present work promoted another batch's prepared wake" + assert_grep 'pre-move-crash' "$home/data/backlog.md" \ + "unrelated handoff changed the prepared batch's source item" + assert_present "$home/state/.backlog-handoff-design.wake-pending" \ + "unrelated handoff discarded another batch's prepared wake" + + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design pre-move-crash \ + > "$TMP_ROOT/pre-move-crash-retry.out" 2>&1 \ + || fail "pre-move crash recovery failed: $(cat "$TMP_ROOT/pre-move-crash-retry.out")" + assert_grep 'pre-move-crash' "$sub/data/backlog.md" "pre-move crash recovery did not move the item" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log") + [ "$wake_count" -eq 1 ] || fail "pre-move crash recovery emitted $wake_count receiver wakes" + pass "a pre-move crash wakes only after retry makes the item durable" +} + +test_delivery_confirmation_crash_does_not_resend() { + local home="$TMP_ROOT/confirm-crash-main" sub="$TMP_ROOT/confirm-crash-sub" + local fakebin="$TMP_ROOT/confirm-crash-fakebin" real_sleep rc=0 wake_count + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] confirm-crash - preserve confirmed delivery (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_sleep=$(command -v sleep) + cat > "$fakebin/sleep" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = 1 ] && mkdir "$FM_CONFIRM_CRASH_ONCE" 2>/dev/null; then + handoff_pid=$(ps -o ppid= -p "$PPID" | tr -d '[:space:]') + kill -KILL "$handoff_pid" + exit 0 +fi +exec "$FM_REAL_SLEEP" "$@" +SH + chmod +x "$fakebin/sleep" + : > "$TMP_ROOT/default-tmux.log" + + set +e + PATH="$fakebin:$PATH" FM_REAL_SLEEP="$real_sleep" \ + FM_CONFIRM_CRASH_ONCE="$TMP_ROOT/confirm-crash.once" FM_SEND_SETTLE=1 \ + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design confirm-crash \ + > "$TMP_ROOT/confirm-crash.out" 2>&1 + rc=$? + set +e + [ "$rc" -ne 0 ] || fail "post-confirmation crash fixture unexpectedly reported success" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log") + [ "$wake_count" -eq 1 ] || fail "post-confirmation crash did not deliver exactly one receiver wake" + case "$(cat "$home/state/.backlog-handoff-design.wake-pending")" in + pending:*) ;; + *) fail "post-confirmation crash lost its stable delivery correlation" ;; + esac + + # Route different work before explicitly retrying the crashed invocation. The + # completed old correlation must be reconciled, but must not stand in as the + # delivery proof for this new durable move. + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] after-crash - requires its own receiver wake (repo: alpha) + +## Done +EOF + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design after-crash \ + > "$TMP_ROOT/after-confirm-crash.out" 2>&1 \ + || fail "new handoff after a confirmation crash failed: $(cat "$TMP_ROOT/after-confirm-crash.out")" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$((wake_count + 1))" ] \ + || fail "completed stale correlation suppressed or duplicated the new handoff wake" + assert_grep 'after-crash' "$sub/data/backlog.md" \ + "new item after a confirmation crash was not durably handed off" + + FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design confirm-crash \ + > "$TMP_ROOT/confirm-crash-retry.out" 2>&1 \ + || fail "post-confirmation crash recovery failed: $(cat "$TMP_ROOT/confirm-crash-retry.out")" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$((wake_count + 1))" ] \ + || fail "post-confirmation crash recovery duplicated the receiver wake" + assert_absent "$home/state/.backlog-handoff-design.wake-pending" \ + "post-confirmation crash recovery left wake state pending" + pass "a post-confirmation crash reconciles once without suppressing a later handoff wake" +} + +test_unresolved_delivery_attempt_refuses_immediate_resend() { + local home="$TMP_ROOT/attempt-crash-main" sub="$TMP_ROOT/attempt-crash-sub" + local fakebin="$TMP_ROOT/attempt-crash-fakebin" real_mv rc=0 wake_count out + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$fakebin" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] attempt-crash - do not resend an unresolved delivery (repo: alpha) + +## Done +EOF + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + real_mv=$(command -v mv) + cat > "$fakebin/mv" <<'SH' +#!/usr/bin/env bash +for arg in "$@"; do + if [ -f "$arg" ] && grep -q '^confirmed=' "$arg" 2>/dev/null; then + kill -KILL "$PPID" + exit 1 + fi +done +exec "$FM_REAL_MV" "$@" +SH + chmod +x "$fakebin/mv" + : > "$TMP_ROOT/default-tmux.log" + + set +e + PATH="$fakebin:$PATH" FM_REAL_MV="$real_mv" FM_HOME="$home" \ + "$ROOT/bin/fm-backlog-handoff.sh" design attempt-crash \ + > "$TMP_ROOT/attempt-crash.out" 2>&1 + rc=$? + set +e + [ "$rc" -ne 0 ] || fail "unresolved-attempt crash fixture unexpectedly reported success" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log") + [ "$wake_count" -eq 1 ] || fail "unresolved-attempt crash did not deliver exactly one receiver wake" + + rc=0 + out=$(FM_HOME="$home" "$ROOT/bin/fm-backlog-handoff.sh" design attempt-crash 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "immediate retry resent or accepted an unresolved delivery attempt" + assert_contains "$out" 'delivery for design is unresolved; refusing to resend correlation' \ + "immediate retry did not report the unresolved delivery boundary" + [ "$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/default-tmux.log")" -eq "$wake_count" ] \ + || fail "immediate retry duplicated the unresolved receiver wake" + pass "an unresolved delivery attempt refuses an immediate duplicate wake" +} + +test_concurrent_local_handoffs_serialize_move_and_wake() { + local home="$TMP_ROOT/concurrent-main" sub="$TMP_ROOT/concurrent-sub" + local basebin blockbin="$TMP_ROOT/concurrent-blockbin" first second i wake_count + setup_homes "$home" "$sub" + mkdir -p "$sub/data" "$blockbin" + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] concurrent-a - first routed item (repo: alpha) + +## Done +EOF + basebin=$(make_fake_tmux "$TMP_ROOT/concurrent-fake") + cat > "$blockbin/tmux" <<'SH' +#!/usr/bin/env bash +case "$*" in + *"New routed work is in your backlog."*) + if mkdir "$FM_BLOCK_WAKE_ONCE" 2>/dev/null; then + touch "$FM_BLOCK_WAKE_ENTERED" + while [ ! -f "$FM_BLOCK_WAKE_RELEASE" ]; do sleep 0.02; done + fi + ;; +esac +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$blockbin/tmux" + + PATH="$blockbin:$basebin:$PATH" FM_HOME="$home" FM_BASE_TMUX="$basebin/tmux" \ + FM_BLOCK_WAKE_ONCE="$TMP_ROOT/concurrent.once" \ + FM_BLOCK_WAKE_ENTERED="$TMP_ROOT/concurrent.entered" \ + FM_BLOCK_WAKE_RELEASE="$TMP_ROOT/concurrent.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/concurrent-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/concurrent-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design concurrent-a > "$TMP_ROOT/concurrent-a.out" 2>&1 & + first=$! + i=0 + while [ ! -f "$TMP_ROOT/concurrent.entered" ]; do + kill -0 "$first" 2>/dev/null || fail "first concurrent handoff exited before its blocked wake" + i=$((i + 1)) + [ "$i" -le 250 ] || fail "first concurrent handoff never reached its receiver wake" + sleep 0.02 + done + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] concurrent-b - second routed item (repo: alpha) + +## Done +EOF + PATH="$blockbin:$basebin:$PATH" FM_HOME="$home" FM_BASE_TMUX="$basebin/tmux" \ + FM_BLOCK_WAKE_ONCE="$TMP_ROOT/concurrent.once" \ + FM_BLOCK_WAKE_ENTERED="$TMP_ROOT/concurrent.entered" \ + FM_BLOCK_WAKE_RELEASE="$TMP_ROOT/concurrent.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/concurrent-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/concurrent-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design concurrent-b > "$TMP_ROOT/concurrent-b.out" 2>&1 & + second=$! + sleep 0.2 + assert_grep 'concurrent-b' "$home/data/backlog.md" \ + "second local handoff moved while the first still owned its wake" + touch "$TMP_ROOT/concurrent.release" + wait "$first" || fail "first serialized local handoff failed" + wait "$second" || fail "second serialized local handoff failed" + assert_grep 'concurrent-a' "$sub/data/backlog.md" "first serialized item was lost" + assert_grep 'concurrent-b' "$sub/data/backlog.md" "second serialized item was lost" + wake_count=$(grep -cF 'New routed work is in your backlog.' "$TMP_ROOT/concurrent-tmux.log") + [ "$wake_count" -eq 2 ] || fail "serialized local handoffs produced $wake_count receiver wakes" + pass "concurrent local handoffs serialize each durable move with its wake" +} + +test_local_teardown_waits_for_handoff_wake() { + local home="$TMP_ROOT/teardown-race-main" sub="$TMP_ROOT/teardown-race-sub" + local basebin blockbin="$TMP_ROOT/teardown-race-blockbin" handoff teardown i + setup_homes "$home" "$sub" + printf 'project=%s\n' "$ROOT" >> "$home/state/design.meta" + mkdir -p "$sub/data" "$blockbin" + printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md" + cat > "$home/data/backlog.md" <<'EOF' +## Queued +- [ ] teardown-race - routed while teardown starts (repo: alpha) + +## Done +EOF + basebin=$(make_fake_tmux "$TMP_ROOT/teardown-race-fake") + cat > "$blockbin/tmux" <<'SH' +#!/usr/bin/env bash +case "$*" in + *"New routed work is in your backlog."*) + touch "$FM_BLOCK_WAKE_ENTERED" + while [ ! -f "$FM_BLOCK_WAKE_RELEASE" ]; do sleep 0.02; done + ;; +esac +exec "$FM_BASE_TMUX" "$@" +SH + chmod +x "$blockbin/tmux" + PATH="$blockbin:$basebin:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_BASE_TMUX="$basebin/tmux" FM_BLOCK_WAKE_ENTERED="$TMP_ROOT/teardown-race.entered" \ + FM_BLOCK_WAKE_RELEASE="$TMP_ROOT/teardown-race.release" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-race-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-race-fake/pane.txt" \ + "$ROOT/bin/fm-backlog-handoff.sh" design teardown-race > "$TMP_ROOT/teardown-race-handoff.out" 2>&1 & + handoff=$! + i=0 + while [ ! -f "$TMP_ROOT/teardown-race.entered" ]; do + kill -0 "$handoff" 2>/dev/null || fail "teardown-race handoff exited before its blocked wake" + i=$((i + 1)) + [ "$i" -le 250 ] || fail "teardown-race handoff never reached its receiver wake" + sleep 0.02 + done + PATH="$basebin:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-race-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-race-fake/pane.txt" \ + "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-race-teardown.out" 2>&1 & + teardown=$! + sleep 0.3 + kill -0 "$teardown" 2>/dev/null \ + || fail "local teardown bypassed the in-flight handoff lock: $(cat "$TMP_ROOT/teardown-race-teardown.out")" + [ -d "$sub" ] || fail "local teardown removed the receiver home before handoff wake completed" + assert_grep 'teardown-race' "$sub/data/backlog.md" \ + "local teardown removed routed work before handoff wake completed" + touch "$TMP_ROOT/teardown-race.release" + wait "$handoff" || fail "teardown-race handoff failed after releasing its wake" + wait "$teardown" 2>/dev/null || true + pass "local teardown waits for the routed move and receiver wake" +} + +test_local_teardown_preserves_wake_when_home_removal_fails() { + local home="$TMP_ROOT/teardown-home-fail-main" sub="$TMP_ROOT/teardown-home-fail-sub" + local fakebin rm_bin="$TMP_ROOT/teardown-home-fail-rm" real_rm corr rc=0 marker rec fail_home + local marker_before="$TMP_ROOT/teardown-home-fail-marker.before" + local rec_before="$TMP_ROOT/teardown-home-fail-record.before" + setup_homes "$home" "$sub" + printf 'project=%s\n' "$ROOT" >> "$home/state/design.meta" + mkdir -p "$sub/data" "$rm_bin" + printf '## Queued\n- [ ] still-routed - preserve its wake (repo: alpha)\n\n## Done\n' > "$sub/data/backlog.md" + corr=$(FM_HOME="$home" bash -c ' + . "$1" + fm_pending_reply_create "$2" "$2/state" design "New routed work is in your backlog." + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$home") \ + || fail "could not seed teardown wake state" + marker="$home/state/.backlog-handoff-design.wake-pending" + rec="$home/state/pending-replies/$corr" + printf 'pending:%s\n' "$corr" > "$marker" + cp -p -- "$marker" "$marker_before" + cp -p -- "$rec" "$rec_before" + real_rm=$(command -v rm) + fail_home=$(cd "$sub" && pwd -P) + cat > "$rm_bin/rm" <<'SH' +#!/usr/bin/env bash +for arg in "$@"; do + [ "$arg" != "$FM_FAIL_HOME" ] || exit 1 +done +exec "$FM_REAL_RM" "$@" +SH + chmod +x "$rm_bin/rm" + fakebin=$(make_fake_tmux "$TMP_ROOT/teardown-home-fail-fake") + + set +e + PATH="$rm_bin:$fakebin:$PATH" FM_REAL_RM="$real_rm" FM_FAIL_HOME="$fail_home" \ + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-home-fail-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-home-fail-fake/pane.txt" \ + "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-home-fail.out" 2>&1 + rc=$? + set -e + [ "$rc" -ne 0 ] || fail "teardown ignored the receiver-home removal failure" + assert_present "$sub" "failed teardown did not preserve the receiver home" + assert_grep 'still-routed' "$sub/data/backlog.md" "failed teardown lost routed backlog work" + cmp -s "$marker_before" "$marker" \ + || fail "failed home removal changed the pending wake marker" + cmp -s "$rec_before" "$rec" \ + || fail "failed home removal changed the pending wake correlation" + assert_present "$home/state/design.meta" "failed teardown removed route metadata" + assert_grep '- design ' "$home/data/secondmates.md" "failed teardown removed the registry route" + + PATH="$fakebin:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \ + FM_FAKE_TMUX_LOG="$TMP_ROOT/teardown-home-fail-tmux.log" \ + FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/teardown-home-fail-fake/pane.txt" \ + "$ROOT/bin/fm-teardown.sh" design --force > "$TMP_ROOT/teardown-home-retry.out" 2>&1 \ + || fail "teardown retry did not retire the preserved wake: $(cat "$TMP_ROOT/teardown-home-retry.out")" + assert_absent "$sub" "teardown retry left the receiver home" + assert_absent "$marker" "teardown retry left the pending wake marker" + assert_absent "$rec" "teardown retry left the pending wake correlation" + assert_no_grep '- design ' "$home/data/secondmates.md" "teardown retry left the registry route" + pass "failed local home removal preserves its wake and a retry retires both" } # Exact multi-line block extract: header matching key plus following body lines @@ -632,6 +1292,17 @@ EOF pass "registry entry without (home: ...) fails cleanly with has no home" } +test_handoff_wakes_live_local_receiver +test_failed_wake_retries_when_the_item_is_already_present +test_known_receiver_failure_remains_retryable_after_grace +test_known_failure_restores_retry_after_reconciliation_race +test_move_crash_keeps_wake_pending_for_recovery +test_pre_move_crash_does_not_wake_until_move_lands +test_delivery_confirmation_crash_does_not_resend +test_unresolved_delivery_attempt_refuses_immediate_resend +test_concurrent_local_handoffs_serialize_move_and_wake +test_local_teardown_waits_for_handoff_wake +test_local_teardown_preserves_wake_when_home_removal_fails test_body_moves_when_followed_by_another_item test_body_moves_when_followed_by_section_heading test_multi_paragraph_body_with_internal_blanks_moves_whole diff --git a/tests/fm-gotmp.test.sh b/tests/fm-gotmp.test.sh index f247b7ff3dc..248d3f28c43 100755 --- a/tests/fm-gotmp.test.sh +++ b/tests/fm-gotmp.test.sh @@ -57,6 +57,7 @@ make_fake_root() { ln -s "$ROOT/bin/fm-backend.sh" "$fake/bin/fm-backend.sh" ln -s "$ROOT/bin/backends/tmux.sh" "$fake/bin/backends/tmux.sh" ln -s "$ROOT/bin/fm-tmux-lib.sh" "$fake/bin/fm-tmux-lib.sh" + ln -s "$ROOT/bin/fm-cursor-lib.sh" "$fake/bin/fm-cursor-lib.sh" ln -s "$ROOT/bin/fm-composer-lib.sh" "$fake/bin/fm-composer-lib.sh" ln -s "$ROOT/bin/fm-nm-run-lib.sh" "$fake/bin/fm-nm-run-lib.sh" # fm-lock-lib.sh: teardown sources it for the shared lock-staleness proof. @@ -81,6 +82,11 @@ make_fake_root() { ln -s "$ROOT/bin/fm-x-lib.sh" "$fake/bin/fm-x-lib.sh" ln -s "$ROOT/bin/fm-secondmate-registry-lib.sh" "$fake/bin/fm-secondmate-registry-lib.sh" ln -s "$ROOT/bin/fm-secondmate-parent-lib.sh" "$fake/bin/fm-secondmate-parent-lib.sh" + # Receiver-wake retirement sources the pending-reply library, which in turn + # requires the marker helper even for this ordinary-task teardown fixture. + ln -s "$ROOT/bin/fm-pending-reply-lib.sh" "$fake/bin/fm-pending-reply-lib.sh" + ln -s "$ROOT/bin/fm-marker-lib.sh" "$fake/bin/fm-marker-lib.sh" + ln -s "$ROOT/bin/fm-operational-input.sh" "$fake/bin/fm-operational-input.sh" # fm-guard.sh: stub (teardown calls it with `|| true`). cat > "$fake/bin/fm-guard.sh" <<'SH' #!/usr/bin/env bash @@ -141,6 +147,7 @@ test_teardown_skips_gracefully_without_tasktmp() { ln -s "$ROOT/bin/fm-backend.sh" "$fake/bin/fm-backend.sh" ln -s "$ROOT/bin/backends/tmux.sh" "$fake/bin/backends/tmux.sh" ln -s "$ROOT/bin/fm-tmux-lib.sh" "$fake/bin/fm-tmux-lib.sh" + ln -s "$ROOT/bin/fm-cursor-lib.sh" "$fake/bin/fm-cursor-lib.sh" ln -s "$ROOT/bin/fm-composer-lib.sh" "$fake/bin/fm-composer-lib.sh" ln -s "$ROOT/bin/fm-nm-run-lib.sh" "$fake/bin/fm-nm-run-lib.sh" ln -s "$ROOT/bin/fm-lock-lib.sh" "$fake/bin/fm-lock-lib.sh" @@ -162,6 +169,9 @@ test_teardown_skips_gracefully_without_tasktmp() { ln -s "$ROOT/bin/fm-x-lib.sh" "$fake/bin/fm-x-lib.sh" ln -s "$ROOT/bin/fm-secondmate-registry-lib.sh" "$fake/bin/fm-secondmate-registry-lib.sh" ln -s "$ROOT/bin/fm-secondmate-parent-lib.sh" "$fake/bin/fm-secondmate-parent-lib.sh" + ln -s "$ROOT/bin/fm-pending-reply-lib.sh" "$fake/bin/fm-pending-reply-lib.sh" + ln -s "$ROOT/bin/fm-marker-lib.sh" "$fake/bin/fm-marker-lib.sh" + ln -s "$ROOT/bin/fm-operational-input.sh" "$fake/bin/fm-operational-input.sh" cat > "$fake/bin/fm-guard.sh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-pending-reply.test.sh b/tests/fm-pending-reply.test.sh index 4457ae6bb76..5a14bafcad9 100755 --- a/tests/fm-pending-reply.test.sh +++ b/tests/fm-pending-reply.test.sh @@ -664,6 +664,60 @@ test_delivery_confirmation_fallback_reconciles() { pass "delivery confirmation fallback reconciles durably" } +test_delivery_confirmation_serializes_with_reconciliation() { + ( + local home state corr rec calls entered release confirm_pid reconcile_pid count i + home=$(setup_parent delivery-confirm-reconcile-race) + state="$home/state" + # This fixture clock is intentionally scoped to the isolated subshell. + # shellcheck disable=SC2030,SC2031 + export FM_PENDING_REPLY_NOW=5900 + corr=$(fm_pending_reply_create "$home" "$state" hibit "serialized delivery") + rec=$(fm_pending_reply_path "$state" "$corr") + calls="$home/mark-delivered.calls" + entered="$home/mark-delivered.entered" + release="$home/mark-delivered.release" + fm_pending_reply_mark_delivered() { + local pending_state=$1 pending_corr=$2 epoch=$3 pending_rec phase + printf '%s\n' "$BASHPID" >> "$calls" + : > "$entered" + while [ ! -e "$release" ]; do /bin/sleep 0.01; done + pending_rec=$(fm_pending_reply_path "$pending_state" "$pending_corr") + fm_pending_reply_set "$pending_rec" delivered_epoch "$epoch" || return 1 + phase=$(fm_pending_reply_get "$pending_rec" phase) + [ "$phase" != delivery_unknown ] \ + || fm_pending_reply_set "$pending_rec" phase awaiting_report + } + fm_pending_reply_confirm_delivery "$state" "$corr" & + # The background PID is consumed within this isolated test subshell. + # shellcheck disable=SC2031 + confirm_pid=$! + for i in $(seq 1 100); do + [ -e "$entered" ] && break + /bin/sleep 0.01 + done + [ -e "$entered" ] || fail "delivery confirmation did not reach its commit boundary" + fm_pending_reply_reconcile_delivery "$state" "$corr" & + # The background PID is consumed within this isolated test subshell. + # shellcheck disable=SC2031 + reconcile_pid=$! + /bin/sleep 0.1 + : > "$release" + wait "$confirm_pid" || fail "delivery confirmation should commit" + wait "$reconcile_pid" || fail "reconciliation should observe committed delivery" + count=$(wc -l < "$calls" | tr -d ' ') + [ "$count" = 1 ] \ + || fail "confirmation and reconciliation raced through $count delivery commits" + [ "$(fm_pending_reply_get "$rec" delivered_epoch)" = 5900 ] \ + || fail "serialized confirmation should retain delivered_epoch" + [ "$(phase_of "$state" "$corr")" = awaiting_report ] \ + || fail "serialized confirmation should retain awaiting_report phase" + [ ! -e "$(fm_pending_reply_delivery_confirmation_path "$state" "$corr")" ] \ + || fail "serialized delivery marker should be removed" + ) || fail "delivery confirmation serialization regression failed" + pass "delivery confirmation serializes with reconciliation" +} + test_unrelated_and_stale_corr_cannot_resolve() { local home state corr other home=$(setup_parent stale-corr) @@ -1197,6 +1251,7 @@ test_concurrent_escalation_yields_to_late_reply test_transport_success_is_not_reply_success test_undelivered_records_are_scan_immutable test_delivery_confirmation_fallback_reconciles +test_delivery_confirmation_serializes_with_reconciliation test_unrelated_and_stale_corr_cannot_resolve test_restart_preserves_expectation_and_parent_destination test_wrong_home_detected_not_acknowledged diff --git a/tests/fm-remote-backlog-handoff.test.sh b/tests/fm-remote-backlog-handoff.test.sh index bcfcd7dd7f0..1b95e5d425c 100755 --- a/tests/fm-remote-backlog-handoff.test.sh +++ b/tests/fm-remote-backlog-handoff.test.sh @@ -15,6 +15,7 @@ REMOTE_ROOT="$TMP_ROOT/remote-root" REMOTE="$TMP_ROOT/remote" FAKEBIN=$(fm_fakebin "$TMP_ROOT/fake") SSH_COUNT="$TMP_ROOT/ssh.count" +WAKE_LOG="$TMP_ROOT/wake.log" mkdir -p "$PARENT/data" "$PARENT/state" "$REMOTE_ROOT/bin" \ "$REMOTE/data" "$REMOTE/state" "$REMOTE/config" "$REMOTE/projects" "$REMOTE/bin" # Tear down deterministically. Releasing the blocked stages and killing the @@ -66,6 +67,19 @@ printf 'ios\n' > "$REMOTE/.fm-secondmate-home" cat > "$PARENT/data/secondmates.md" < "$PARENT/state/ios.meta" < "$WAKE_LOG" cat > "$FAKEBIN/fake-ssh" <<'SH' #!/usr/bin/env bash @@ -86,6 +100,11 @@ shift 2 argv_b64=$4 command_name=$(perl -MMIME::Base64=decode_base64 -e '$d=decode_base64($ARGV[0]); ($c)=split(/\0/, $d); print $c' "$argv_b64") case "${FM_FAKE_SSH_MODE:-normal}:$command_name" in + *:fm-remote-secondmate-control.sh) + printf '%s\n' "$command_name" >> "$FM_FAKE_REMOTE_WAKE_LOG" + [ "${FM_FAKE_REMOTE_WAKE_RC:-0}" -eq 0 ] || printf 'remote receiver wake failed\n' >&2 + exit "${FM_FAKE_REMOTE_WAKE_RC:-0}" + ;; unreachable:*) exit 255 ;; serialize:fm-backlog-receive.sh) if mkdir "$FM_FAKE_SERIALIZE_ONCE" 2>/dev/null; then @@ -112,6 +131,8 @@ handoff_env() { FM_ROOT_OVERRIDE="$ROOT" \ FM_SSH_BIN="$FAKEBIN/fake-ssh" \ FM_FAKE_SSH_COUNT="$SSH_COUNT" \ + FM_FAKE_REMOTE_WAKE_LOG="$WAKE_LOG" \ + FM_FAKE_REMOTE_WAKE_RC="${FM_FAKE_REMOTE_WAKE_RC:-0}" \ FM_FAKE_SERIALIZE_ONCE="$TMP_ROOT/serialize.once" \ FM_FAKE_SERIALIZE_ENTERED="$TMP_ROOT/serialize.entered" \ FM_FAKE_SERIALIZE_RELEASE="$TMP_ROOT/serialize.release" \ @@ -222,6 +243,8 @@ pass "ambiguous receipt leaves one durable outbox and no duplicate dispatchable out=$(handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending) assert_contains "$out" 'received: ios moved=0 already=2' "retry did not classify already-delivered keys idempotently" +[ "$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG")" -eq 1 ] \ + || fail "confirmed remote receipt did not wake its supported receiver endpoint exactly once" assert_absent "$PARENT/data/handoff/ios.outbox.md" "confirmed retry did not clean the local outbox" [ "$(grep -cF -- '- [ ] ios-a - first iOS task' "$REMOTE/data/backlog.md")" -eq 1 ] \ || fail "receipt retry duplicated ios-a" @@ -315,6 +338,69 @@ handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending >/dev/null \ || fail "pending bootstrap-visible outbox did not later converge" pass "bootstrap detects pending outbox handoffs without a journal" +write_backlog '- [ ] remote-wake-fail - receiver failure stays recoverable (repo: alpha)' +set +e +FM_FAKE_REMOTE_WAKE_RC=1 handoff_env "$ROOT/bin/fm-backlog-handoff.sh" ios remote-wake-fail \ + > "$TMP_ROOT/remote-wake-fail.out" 2>&1 +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "remote handoff claimed success after its receiver wake failed" +assert_contains "$(cat "$TMP_ROOT/remote-wake-fail.out")" 'receiver wake failed' \ + "remote receiver wake failure was not surfaced" +assert_present "$PARENT/data/handoff/ios.outbox.md" \ + "remote receiver wake failure discarded the recoverable outbox" +handoff_env "$ROOT/bin/fm-backlog-handoff.sh" --resume-pending >/dev/null \ + || fail "remote receiver wake failure did not recover through resume-pending" +assert_absent "$PARENT/data/handoff/ios.outbox.md" \ + "remote receiver wake recovery left its outbox pending" +pass "remote handoff wakes its supported endpoint or remains loudly recoverable" + +RM_FAKEBIN="$TMP_ROOT/rm-fakebin" +mkdir -p "$RM_FAKEBIN" +REAL_RM=$(command -v rm) +cat > "$RM_FAKEBIN/rm" <<'SH' +#!/usr/bin/env bash +last=${!#} +if [ "$last" = "$FM_FAIL_RM_PATH" ]; then + exit 1 +fi +exec "$FM_REAL_RM" "$@" +SH +chmod +x "$RM_FAKEBIN/rm" +write_backlog '- [ ] cleanup-retry - confirmed wake survives cleanup retry (repo: alpha)' +wakes_before=$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG") +set +e +PATH="$RM_FAKEBIN:$PATH" FM_REAL_RM="$REAL_RM" \ + FM_FAIL_RM_PATH="$PARENT/data/handoff/ios.outbox.md" \ + handoff_env "$ROOT/bin/fm-backlog-handoff.sh" ios cleanup-retry \ + > "$TMP_ROOT/cleanup-retry.out" 2>&1 +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "remote handoff ignored local outbox cleanup failure" +assert_present "$PARENT/data/handoff/ios.outbox.md" \ + "remote cleanup failure did not preserve the outbox" +case "$(cat "$PARENT/state/.backlog-handoff-ios.wake-pending")" in + confirmed:*) ;; + *) fail "remote cleanup failure did not preserve confirmed wake state" ;; +esac +wakes_after=$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG") +[ "$wakes_after" -eq $((wakes_before + 1)) ] \ + || fail "remote cleanup failure did not perform exactly one receiver wake" +write_backlog '- [ ] after-cleanup - fresh work after confirmed cleanup failure (repo: alpha)' +handoff_env "$ROOT/bin/fm-backlog-handoff.sh" ios after-cleanup >/dev/null \ + || fail "fresh handoff did not converge an older confirmed cleanup failure" +[ "$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG")" -eq $((wakes_after + 1)) ] \ + || fail "fresh handoff reused the older confirmed wake instead of waking its receiver" +[ "$(grep -cF cleanup-retry "$REMOTE/data/backlog.md")" -eq 1 ] \ + || fail "cleanup recovery lost or duplicated the older delivered item" +[ "$(grep -cF after-cleanup "$REMOTE/data/backlog.md")" -eq 1 ] \ + || fail "fresh handoff after cleanup recovery was lost or duplicated" +assert_absent "$PARENT/data/handoff/ios.outbox.md" \ + "fresh handoff left the recovered outbox pending" +assert_absent "$PARENT/state/.backlog-handoff-ios.wake-pending" \ + "fresh handoff left confirmed wake state behind" +pass "fresh remote work gets a new wake after confirmed cleanup recovery" + write_backlog '- [ ] route-race - remains dispatchable through retirement (repo: alpha)' registry_lock="$PARENT/state/.secondmate-registry.lock" handoff_lock="$PARENT/state/.backlog-handoff-ios.lock" diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index 9e6bfbba4d4..e7e84a95407 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -1149,6 +1149,19 @@ assert_present "$REMOTE_HOME" "unsafe pending-replies retirement removed the rem assert_present "$TMP_ROOT/external-pending/escape" "unsafe retirement removed an external pending reply" rm -f "$PARENT/state/pending-replies" mv "$PARENT/state/pending-replies.safe" "$PARENT/state/pending-replies" +retired_wake_corr=$(FM_HOME="$PARENT" bash -c ' + . "$1" + fm_pending_reply_create "$2" "$2/state" ios "New routed work is in your backlog." +' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$PARENT") \ + || fail "could not seed remote receiver wake retirement state" +retired_wake_rec="$PARENT/state/pending-replies/$retired_wake_corr" +FM_HOME="$PARENT" bash -c ' + . "$1" + fm_pending_reply_set "$2" phase resolved + fm_pending_reply_set "$2" delivered_epoch 1 +' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$retired_wake_rec" \ + || fail "could not settle remote receiver wake retirement state" +printf 'confirmed:%s\n' "$retired_wake_corr" > "$PARENT/state/.backlog-handoff-ios.wake-pending" handoff_lock="$PARENT/state/.backlog-handoff-ios.lock" FM_HOME="$PARENT" /bin/bash -c ' . "$1" @@ -1199,6 +1212,9 @@ if ! wait "$teardown_pid"; then fi assert_absent "$REMOTE_HOME" "remote retirement did not remove the remote home" assert_absent "$PARENT/state/ios.meta" "remote retirement did not remove parent metadata" +assert_absent "$PARENT/state/.backlog-handoff-ios.wake-pending" \ + "remote retirement left receiver wake state that could poison a replacement route" +assert_absent "$retired_wake_rec" "remote retirement left the retired receiver wake correlation" assert_no_grep '- ios ' "$PARENT/data/secondmates.md" "remote retirement did not remove the registry route" jq -e --arg workspace "$SIBLING_WORKSPACE" --arg pane "$SIBLING_PANE" ' any(.workspaces[]; .workspace_id == $workspace and .label == "2ndmate-macos") diff --git a/tests/fm-secondmate-lifecycle-e2e.test.sh b/tests/fm-secondmate-lifecycle-e2e.test.sh index 9c9555f1cf8..c4cb31d06f1 100755 --- a/tests/fm-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-secondmate-lifecycle-e2e.test.sh @@ -171,7 +171,9 @@ phase_handoff() { - [x] old-task - shipped thing - local main (merged 2026-06-19) EOF local out before - out=$(FM_HOME="$HOME_DIR" "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y) \ + out=$(PATH="$FAKEBIN:$PATH" FM_HOME="$HOME_DIR" FM_FAKE_TMUX_LOG="$LOG" \ + FM_FAKE_TMUX_CAPTURE="$PANE" \ + "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y) \ || fail "handoff failed for in-scope items" assert_contains "$out" "handed off 2 item(s) to design" "handoff did not report the moved items" @@ -187,7 +189,9 @@ EOF # Idempotent: a second handoff neither errors nor duplicates, and leaves main alone. before=$(cat "$HOME_DIR/data/backlog.md") - FM_HOME="$HOME_DIR" "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y >/dev/null 2>&1 \ + PATH="$FAKEBIN:$PATH" FM_HOME="$HOME_DIR" FM_FAKE_TMUX_LOG="$LOG" \ + FM_FAKE_TMUX_CAPTURE="$PANE" \ + "$ROOT/bin/fm-backlog-handoff.sh" design feat-x feat-y >/dev/null 2>&1 \ || fail "idempotent re-run failed" [ "$(grep -cF -- '- [ ] feat-x - add feature x (repo: alpha)' "$SUB/data/backlog.md")" -eq 1 ] \ || fail "idempotent re-run duplicated feat-x in the subhome backlog" @@ -210,7 +214,20 @@ phase_recovery() { } phase_teardown() { - local teardown_out + local teardown_out corr rec + corr=$(FM_HOME="$HOME_DIR" bash -c ' + . "$1" + fm_pending_reply_create "$2" "$2/state" design "New routed work is in your backlog." + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$HOME_DIR") \ + || fail "could not seed receiver wake retirement state" + rec="$HOME_DIR/state/pending-replies/$corr" + FM_HOME="$HOME_DIR" bash -c ' + . "$1" + fm_pending_reply_set "$2" phase resolved + fm_pending_reply_set "$2" delivered_epoch 1 + ' _ "$ROOT/bin/fm-pending-reply-lib.sh" "$rec" \ + || fail "could not settle receiver wake retirement state" + printf 'confirmed:%s\n' "$corr" > "$HOME_DIR/state/.backlog-handoff-design.wake-pending" : > "$LOG" teardown_out=$(PATH="$FAKEBIN:$PATH" FM_HOME="$HOME_DIR" FM_FAKE_TMUX_LOG="$LOG" FM_FAKE_TMUX_CAPTURE="$PANE" \ "$ROOT/bin/fm-teardown.sh" design 2>&1) \ @@ -219,6 +236,9 @@ phase_teardown() { && fail "secondmate teardown emitted a main-backlog completion reminder" assert_absent "$SUB" "teardown did not remove the retired secondmate home" assert_absent "$HOME_DIR/state/design.meta" "teardown did not clear the parent meta" + assert_absent "$HOME_DIR/state/.backlog-handoff-design.wake-pending" \ + "teardown left receiver wake state that could poison a replacement route" + assert_absent "$rec" "teardown left the retired receiver wake correlation" assert_no_grep '- design ' "$HOME_DIR/data/secondmates.md" "teardown did not remove the registry route" # The parent's source projects are untouched (no write through a parent home). assert_present "$HOME_DIR/projects/alpha" "teardown disturbed a parent project" diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index 0a3619ce0ed..2b994fcbe05 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -235,6 +235,208 @@ test_drain_dedupes_obvious_duplicates() { # plain drain-and-handle turn that runs no other supervision script. It must warn # when work is in flight with no live watcher, and stay silent right after a # normal fire from a live watcher with a fresh beacon, so it never false-alarms. +test_secondmate_foreign_queue_stall_is_one_shot_and_read_only() { + local dir state sub fakebin out row_before row_after stall_count + dir=$(make_case secondmate-foreign-stall) + state="$dir/state" + sub="$dir/secondmate" + mkdir -p "$sub/state" "$sub/data" "$sub/bin" + printf '# Firstmate\n' > "$sub/AGENTS.md" + printf 'mate\n' > "$sub/.fm-secondmate-home" + printf 'window=firstmate:fm-mate\nkind=secondmate\nharness=claude\nbackend=tmux\nhome=%s\n' \ + "$sub" > "$state/mate.meta" + printf '%s\t7\tcheck\trouted\tcheck: routed row\n' "$(( $(date +%s) - 10 ))" > "$sub/state/.wake-queue" + row_before="$dir/foreign-before" + row_after="$dir/foreign-after" + cp "$sub/state/.wake-queue" "$row_before" + fakebin="$dir/fakebin" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +case "${1:-}" in + list-windows) printf '%s\n' "${FM_FAKE_TMUX_WINDOW:-}" ;; + capture-pane) cat "${FM_FAKE_TMUX_CAPTURE:-/dev/null}" ;; + display-message) printf '0\n' ;; + *) exit 0 ;; +esac +SH + chmod +x "$fakebin/tmux" + out="$dir/watch.out" + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 3 > "$out" 2> "$dir/watch.err" || true + grep -F 'check: secondmate wake-loop stalled: mate=mate row=7' "$out" >/dev/null \ + || fail "an aged foreign row did not wake the parent checkpoint: $(cat "$out"); err=$(cat "$dir/watch.err"); meta=$(cat "$state/mate.meta"); foreign=$(cat "$sub/state/.wake-queue")" + [ -s "$state/.wake-queue" ] || fail "the parent notification was not durable" + stall_count=$(grep -c 'secondmate-wake-loop-mate-' "$state/.wake-queue" || true) + [ "$stall_count" -eq 1 ] || fail "the first parent checkpoint did not publish exactly one stall notification" + + cmp -s "$row_before" "$sub/state/.wake-queue" \ + || fail "foreign queue row changed during read-only stall detection" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2> "$dir/drain.err" \ + || fail "parent drain failed after the stall notification" + ack_drain_err "$state" "$dir/drain.err" \ + || fail "parent stall notification could not be acknowledged" + + sleep 1 + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$dir/watch-second.out" 2> "$dir/watch-second.err" || true + [ ! -s "$state/.wake-queue" ] || { + stall_count=$(grep -c 'secondmate-wake-loop-mate-' "$state/.wake-queue" || true) + [ "$stall_count" -eq 0 ] || fail "repeated checkpoint re-published the same stall notification" + } + cp "$sub/state/.wake-queue" "$row_after" + cmp -s "$row_before" "$row_after" || fail "foreign queue changed after idempotent re-check" + + : > "$sub/state/.wake-queue" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$dir/watch-empty.out" 2> "$dir/watch-empty.err" || true + ! grep -F 'secondmate wake-loop stalled' "$dir/watch-empty.out" >/dev/null \ + || fail "an empty foreign queue produced a stall notification" + + printf '%s\t8\tcheck\thealthy\tcheck: healthy row\n' "$(date +%s)" > "$sub/state/.wake-queue" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=60 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$dir/watch-healthy.out" 2> "$dir/watch-healthy.err" || true + ! grep -F 'secondmate wake-loop stalled' "$dir/watch-healthy.out" >/dev/null \ + || fail "a healthy foreign queue produced a stall notification" + pass "foreign secondmate queue stalls notify once, remain byte-stable, and stay quiet when empty or healthy" +} + +test_secondmate_stall_marker_rejects_symlink() { + local dir state sub fakebin marker outside expected + dir=$(make_case secondmate-stall-marker-symlink) + state="$dir/state" + sub="$dir/secondmate" + mkdir -p "$sub/state" + printf 'mate\n' > "$sub/.fm-secondmate-home" + printf 'window=firstmate:fm-mate\nkind=secondmate\nhome=%s\n' "$sub" > "$state/mate.meta" + printf '%s\t7\tcheck\trouted\tcheck: routed row\n' "$(( $(date +%s) - 10 ))" > "$sub/state/.wake-queue" + outside="$dir/outside" + expected='must remain unchanged' + printf '%s\n' "$expected" > "$outside" + marker="$state/.secondmate-wake-stall-mate" + ln -s "$outside" "$marker" + fakebin="$dir/fakebin" + cat > "$fakebin/tmux" <<'SH' +#!/usr/bin/env bash +case "${1:-}" in + list-windows) printf '%s\n' 'firstmate:fm-mate' ;; + capture-pane) : ;; + display-message) printf '0\n' ;; + *) exit 0 ;; +esac +SH + chmod +x "$fakebin/tmux" + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 \ + FM_SIGNAL_GRACE=0 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 \ + > "$dir/watch.out" 2> "$dir/watch.err" || true + [ "$(cat "$outside")" = "$expected" ] || fail "stall marker write followed an unsafe symlink" + [ -L "$marker" ] || fail "stall marker write replaced rather than rejected an unsafe path" + [ ! -s "$state/.wake-queue" ] || fail "unsafe stall marker path still published a parent notification" + pass "secondmate stall markers reject symlinks without touching their targets" +} + +test_acknowledged_stall_publication_survives_pre_marker_crash() { + local dir state sub fakebin out epoch row_before + dir=$(make_case secondmate-stall-crash) + state="$dir/state" + sub="$dir/secondmate" + mkdir -p "$sub/state" "$sub/data" + printf 'mate\n' > "$sub/.fm-secondmate-home" + printf 'window=firstmate:fm-mate\nkind=secondmate\nharness=claude\nbackend=tmux\nhome=%s\n' \ + "$sub" > "$state/mate.meta" + epoch=$(( $(date +%s) - 10 )) + printf '%s\t7\tcheck\trouted\tcheck: routed row\n' "$epoch" > "$sub/state/.wake-queue" + row_before="$dir/foreign-before" + cp "$sub/state/.wake-queue" "$row_before" + append_wake "$state" check "secondmate-wake-loop-mate-$epoch-7" \ + "check: secondmate wake-loop stalled: mate=mate row=7 age=10s" \ + || fail "could not seed the pre-marker crash publication" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2> "$dir/drain.err" \ + || fail "pre-marker crash publication could not be drained" + ack_drain_err "$state" "$dir/drain.err" \ + || fail "pre-marker crash publication could not be acknowledged" + + fakebin="$dir/fakebin" + out="$dir/watch.out" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='firstmate:fm-mate' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 > "$out" 2> "$dir/watch.err" || true + ! grep -F 'secondmate wake-loop stalled' "$out" >/dev/null \ + || fail "an acknowledged publication was duplicated after the pre-marker crash state" + [ ! -s "$state/.wake-queue" ] \ + || fail "the replacement watcher re-published an acknowledged stall notification" + cmp -s "$row_before" "$sub/state/.wake-queue" \ + || fail "pre-marker crash recovery changed the foreign queue row" + pass "stall publication acknowledgement closes the pre-marker crash window" +} + +test_empty_prefix_mate_preserves_other_mate_receipt() { + local dir state empty stalled fakebin epoch row_before round + dir=$(make_case secondmate-prefix-receipt) + state="$dir/state" + empty="$dir/ios" + stalled="$dir/ios-ui" + mkdir -p "$empty/state" "$stalled/state" + printf 'ios\n' > "$empty/.fm-secondmate-home" + printf 'ios-ui\n' > "$stalled/.fm-secondmate-home" + printf 'window=firstmate:fm-ios\nkind=secondmate\nhome=%s\n' "$empty" > "$state/ios.meta" + printf 'window=firstmate:fm-ios-ui\nkind=secondmate\nhome=%s\n' "$stalled" > "$state/ios-ui.meta" + : > "$empty/state/.wake-queue" + epoch=$(( $(date +%s) - 10 )) + printf '%s\t9\tcheck\trouted\tcheck: routed row\n' "$epoch" > "$stalled/state/.wake-queue" + row_before="$dir/foreign-before" + cp "$stalled/state/.wake-queue" "$row_before" + append_wake "$state" check "secondmate-wake-loop-ios-ui-$epoch-9" \ + "check: secondmate wake-loop stalled: mate=ios-ui row=9 age=10s" \ + || fail "could not seed the ios-ui stall publication" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$dir/drain.out" 2> "$dir/drain.err" \ + || fail "ios-ui stall publication could not be drained" + ack_drain_err "$state" "$dir/drain.err" \ + || fail "ios-ui stall publication could not be acknowledged" + + fakebin="$dir/fakebin" + round=1 + while [ "$round" -le 2 ]; do + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_FAKE_TMUX_WINDOW='' \ + FM_FAKE_TMUX_LOG="$dir/tmux.log" FM_FAKE_TMUX_CAPTURE="$dir/fake-tmux/pane.txt" \ + FM_SECONDMATE_WAKE_STALL_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=0 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$ROOT/bin/fm-watch-checkpoint.sh" --seconds 2 \ + > "$dir/watch-$round.out" 2> "$dir/watch-$round.err" || true + ! grep -F 'secondmate wake-loop stalled' "$dir/watch-$round.out" >/dev/null \ + || fail "empty ios queue erased ios-ui idempotency on checkpoint $round" + round=$((round + 1)) + done + [ ! -s "$state/.wake-queue" ] \ + || fail "overlapping mate ids re-published the acknowledged ios-ui stall" + cmp -s "$row_before" "$stalled/state/.wake-queue" \ + || fail "overlapping mate receipt checks changed the foreign row" + pass "empty prefix mate cleanup preserves another mate's stall receipt" +} + test_drain_asserts_watcher_liveness() { local dir state err identity dir=$(make_case drain-liveness) @@ -793,6 +995,10 @@ test_historical_annotation_skips_announced_status() { } test_self_held_lock_reclaims_instead_of_deadlocking +test_secondmate_foreign_queue_stall_is_one_shot_and_read_only +test_secondmate_stall_marker_rejects_symlink +test_acknowledged_stall_publication_survives_pre_marker_crash +test_empty_prefix_mate_preserves_other_mate_receipt test_self_announced_append_guards test_historical_annotation_skips_announced_status test_concurrent_append_and_drain From 822a9902494b628ef92c538f40112bd79757271e Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:58:29 -0700 Subject: [PATCH 09/12] fix: make macOS inbox test path portable (#2857) --- bin/fm-inbox.sh | 10 +++++----- tests/fm-tool-update-check.test.sh | 6 +++--- tests/fm-voice-relay.test.sh | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bin/fm-inbox.sh b/bin/fm-inbox.sh index 3f967fd80f2..f314a12f7a1 100755 --- a/bin/fm-inbox.sh +++ b/bin/fm-inbox.sh @@ -168,10 +168,12 @@ queue_note() { [ -n "${body//[[:space:]]/}" ] || die "refusing to queue an empty note" mkdir -p "$INBOX" - local tmp id summary + local tmp id summary staging_name tmp=$(mktemp "$INBOX/.staging-XXXXXX") + staging_name=$(basename "$tmp") + id="$(date +%s)-${staging_name#.staging-}" { - printf 'id=PENDING\n' + printf 'id=%s\n' "$id" printf 'at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'source=%s\n' "$source" [ -z "$extra" ] || printf '%s\n' "$extra" @@ -179,9 +181,7 @@ queue_note() { printf '%s\n' "$body" } >"$tmp" - id="$(date +%s)-$(basename "$tmp" | sed 's/^\.staging-//')" - # Rewrite the id line now that we know it, then publish atomically. - sed -i "s/^id=PENDING$/id=$id/" "$tmp" + # Publish the completed note atomically. mv "$tmp" "$INBOX/$id.note" # One-line summary for the wake payload; the full body stays in the file. diff --git a/tests/fm-tool-update-check.test.sh b/tests/fm-tool-update-check.test.sh index 2f8843a821d..b30bc049f30 100755 --- a/tests/fm-tool-update-check.test.sh +++ b/tests/fm-tool-update-check.test.sh @@ -131,7 +131,7 @@ test_path_skew_is_reported_from_every_copy() { assert_contains "$report" "0.8.2 is installed at $fresh/$TOOL" "the report does not name the newer installed copy, so no other PATH copy was asked for its version" assert_not_contains "$report" "update available" "PATH skew must not be reported as a published update" assert_contains "$report" "$(printf 'tool updates:')" "the report is missing its one-line prefix" - [ "$(wc -l < "$out")" = 1 ] || fail "the report must be exactly one line for the wake record" + [ "$(wc -l < "$out" | tr -d '[:space:]')" = 1 ] || fail "the report must be exactly one line for the wake record" pass "PATH skew is reported by asking every copy on PATH for its own version" } @@ -311,7 +311,7 @@ test_one_broken_pattern_does_not_blind_the_rest_of_the_sweep() { report=$(cat "$out") assert_contains "$report" "herdr update not in effect: PATH resolves 0.8.0 at $stale/$TOOL" "a broken pattern on another tool suppressed the PATH skew report" assert_contains "$report" "no-mistakes check failed: announce_pattern is not a usable extended regular expression" "the tool whose pattern cannot be used was not named" - [ "$(wc -l < "$out")" = 1 ] || fail "the report must stay exactly one line" + [ "$(wc -l < "$out" | tr -d '[:space:]')" = 1 ] || fail "the report must stay exactly one line" pass "a broken pattern is reported for its own tool and the rest of the sweep still reports" } @@ -690,7 +690,7 @@ test_an_overlong_report_says_it_was_cut() { run_check "$home" "$PATH" "$out" report=$(cat "$out") assert_contains "$report" "[truncated]" "an over-long report was cut without saying so" - [ "$(wc -l < "$out")" = 1 ] || fail "the cut report must still be exactly one line" + [ "$(wc -l < "$out" | tr -d '[:space:]')" = 1 ] || fail "the cut report must still be exactly one line" pass "an over-long report is cut with the shared truncation marker" } diff --git a/tests/fm-voice-relay.test.sh b/tests/fm-voice-relay.test.sh index f9c57543aed..99645ec488a 100755 --- a/tests/fm-voice-relay.test.sh +++ b/tests/fm-voice-relay.test.sh @@ -3646,7 +3646,7 @@ pass "the configured read scope is honoured" # The point of the boundary: real work is queued for firstmate, not done by the # voice agent. It reuses bin/fm-inbox.sh rather than carrying a second queue. -before=$(find "$HOME_FIXTURE/state" -maxdepth 2 -name '*.note' | wc -l) +before=$(find "$HOME_FIXTURE/state" -maxdepth 2 -name '*.note' | wc -l | tr -d '[:space:]') [ "$before" = 0 ] || fail "fixture should start with an empty inbox" handed=$(FM_HOME="$HOME_FIXTURE" python3 "$ROOT/bin/fm_voice_records.py" queue \ @@ -3656,7 +3656,7 @@ assert_contains "$handed" '"queued": true' "handover should report the request q assert_contains "$handed" 'did not do the work yourself' \ "handover should tell the model it handed over rather than acted" -notes=$(find "$HOME_FIXTURE/state/inbox" -maxdepth 1 -name '*.note' | wc -l) +notes=$(find "$HOME_FIXTURE/state/inbox" -maxdepth 1 -name '*.note' | wc -l | tr -d '[:space:]') [ "$notes" = 1 ] || fail "handover should leave exactly one note, found $notes" note_file=$(find "$HOME_FIXTURE/state/inbox" -maxdepth 1 -name '*.note' | head -1) assert_grep 'Refactor the login module' "$note_file" \ @@ -3687,7 +3687,7 @@ FM_STATE_OVERRIDE="$alt_state" python3 "$ROOT/bin/fm_voice_records.py" queue \ "Chase the flaky retry test" --home "$alt_home" >/dev/null \ || fail "handover with an overridden state directory failed" -moved=$(find "$alt_state/inbox" -maxdepth 1 -name '*.note' | wc -l) +moved=$(find "$alt_state/inbox" -maxdepth 1 -name '*.note' | wc -l | tr -d '[:space:]') [ "$moved" = 1 ] || \ fail "the queue should write into the overridden state directory, found $moved" [ ! -e "$alt_home/state/inbox" ] || \ From e46df1a55a9edbf242921c0b00703a388d6c7635 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:40:39 -0700 Subject: [PATCH 10/12] feat(bin): deliver local steers through durable task inboxes (#2856) * feat(bin): steer local tasks by durable inbox record plus constant doorbell Stage 1 (local steers) of the captain-adopted reframe in data/fm-send-reliability-reframe-s1/report.md: an ordinary fm-send text steer to a task recorded in this home is appended as a sequenced durable record under state/.inbox/ and the terminal receives only one constant self-describing doorbell line, best-effort. The worker acknowledges by moving the record into handled/; the watcher re-rings an unacknowledged message on an idle pane and escalates once as an ordinary stale wake. --resolve-key closes decisions at enqueue time, because the durable enqueue IS delivery to the task's record. bin/fm-task-inbox-lib.sh owns the record format, doorbell line, and re-ring ladder. The typed plane remains for what must reach the terminal itself: lifecycle keys, harness-native slash and codex $-skill invocations, explicit backend targets, and the remote secondmate leg (unchanged until the remote inbox leg ships separately). The composer classifier is demoted from delivery proof to an advisory ring guard that skips only on a proven pending verdict. Verified live against claude, codex, opencode, pi, grok, and muse: each real worker read its record, acted, and acked with the mv (docs/verification/runtime-backends.md "Steering-inbox doorbell"). * docs(verification): flag the grok 1.0.5 composer-matrix staleness observed by the doorbell run * test(captain-hold): read the chat-channel answer from the durable inbox record * test: migrate fm-control's marker contrast to the inbox record and fix macOS wc padding in the tool-update suite * no-mistakes(review): Harden inbox locking, teardown races, and acknowledgements * no-mistakes(review): Serialize watcher actions with inbox acknowledgements * no-mistakes(review): Bound metadata locking and tighten acknowledgement rechecks * no-mistakes(review): Preserve exact inbox bytes and harden delivery recovery * no-mistakes(review): Harden watcher bookkeeping against concurrent inbox teardown * no-mistakes(document): Update inbox and typed-plane documentation * no-mistakes: apply CI fixes * no-mistakes: apply CI fixes * revert(pipeline): keep parser-native secondmate marking and the both-failed exit out of stage 1 The CI monitor's fix changed the secondmate marking contract for parser-native invocations (appending the marker after the text) and softened the both-commit-and-marker-failed branch to exit 0. The merge authority ruled the marking question out of scope for this stage-1 transport PR (follow-up: fm-send-secondmate-harness-invocation-r1) and ruled the both-failed case a loud nonzero local failure. Restore both, keeping the monitor's legitimate migrations and hardening. * no-mistakes(document): Document inbox and typed-plane boundaries * no-mistakes(document): Scope backend transport docs to typed plane * no-mistakes(document): Clarify inbox attempt-budget documentation * no-mistakes: apply CI fixes * fix(send): the durable record alone governs the inbox exit status Captain-refined ruling on the F2/Greptile finding: the durable inbox record is what delivers the steer, so pending-reply bookkeeping trouble after a successful enqueue never exits nonzero - a resend-inviting status would make automated callers enqueue the delivered instruction again under a new sequence. With the recovery marker stored the watcher reconciles silently; with the commit and marker both lost the send surfaces a distinct reply-tracking-degraded do-not-resend warning and still exits 0. Nonzero remains only where nothing was delivered (or a decision close needs its manual command). Regression: record durable + both bookkeeping writes lost -> exit 0, one record, no duplicate. * no-mistakes(review): Preserve inbox ordering with drain-all doorbells * no-mistakes(review): Surface unwritable inbox ladder bookkeeping * no-mistakes(review): Silence ladder failures after inbox acknowledgement * no-mistakes(document): Update steering inbox documentation * no-mistakes: apply CI fixes --- .agents/skills/afk/SKILL.md | 4 +- .agents/skills/firstmate-orca/SKILL.md | 8 +- .agents/skills/harness-adapters/SKILL.md | 8 +- .../skills/stuck-crewmate-recovery/SKILL.md | 2 +- AGENTS.md | 4 +- bin/fm-brief.sh | 24 + bin/fm-control.sh | 4 + bin/fm-send.sh | 238 ++++++++-- bin/fm-task-inbox-lib.sh | 315 +++++++++++++ bin/fm-teardown.sh | 4 + bin/fm-test-run.sh | 14 +- bin/fm-watch.sh | 79 ++++ docs/architecture.md | 2 +- docs/cmux-backend.md | 3 +- docs/configuration.md | 14 +- docs/herdr-backend.md | 6 +- docs/orca-backend.md | 3 +- docs/scripts.md | 3 +- docs/tmux-backend.md | 2 +- docs/verification/runtime-backends.md | 44 +- docs/zellij-backend.md | 3 +- tests/fm-backend-herdr.test.sh | 6 +- tests/fm-backend-orca.test.sh | 16 +- tests/fm-backlog-handoff.test.sh | 188 +++++--- tests/fm-captain-hold-lifecycle.test.sh | 5 +- tests/fm-control.test.sh | 5 +- tests/fm-daemon.test.sh | 6 +- tests/fm-gate-refuse.test.sh | 10 +- tests/fm-pending-reply.test.sh | 24 +- tests/fm-secondmate-harness.test.sh | 89 ++-- tests/fm-secondmate-lifecycle-e2e.test.sh | 17 +- tests/fm-secondmate-sync.test.sh | 40 +- tests/fm-send-inbox-doorbell-live-e2e.test.sh | 206 +++++++++ tests/fm-send-inbox.test.sh | 352 ++++++++++++++ tests/fm-send-popup-settle.test.sh | 49 +- tests/fm-send-remote-delivery.test.sh | 25 +- tests/fm-send-resolve-key.test.sh | 75 ++- tests/fm-send-secondmate-marker.test.sh | 63 ++- tests/fm-send-settle.test.sh | 19 +- tests/fm-send-strict.test.sh | 16 +- tests/fm-startup-memory-budget.test.sh | 12 +- tests/fm-task-inbox.test.sh | 430 ++++++++++++++++++ 42 files changed, 2135 insertions(+), 302 deletions(-) create mode 100644 bin/fm-task-inbox-lib.sh create mode 100644 tests/fm-send-inbox-doorbell-live-e2e.test.sh create mode 100644 tests/fm-send-inbox.test.sh create mode 100644 tests/fm-task-inbox.test.sh diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index e884db150c6..058a1947844 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -125,9 +125,7 @@ For tmux that confirmation is normally a proven cleared composer from the shared Without that baseline, busy state never converts an `unknown` composer into confirmation. For herdr, idle-baseline submits first seek native agent-state showing a real turn started, then use the shared classifier when native state remains idle: a cleared composer confirms delivery, while pending text retries Enter and reaches the shared busy-queue verdict only after the retry budget. A bordered-empty or ghost-only composer is recognized as empty where that backend uses composer confirmation, rather than mistaken for a swallowed Enter. -`fm-send.sh` uses the same primitive and exits non-zero -when a steer's Enter is positively swallowed, so firstmate learns an instruction -did not land instead of leaving it unsubmitted. +`fm-send.sh` uses the same primitive only on its typed plane and exits non-zero when that plane's Enter is positively swallowed; ordinary local text steers use the durable inbox and do not treat doorbell submission as delivery proof. **Busy-queued Enter exception (opencode 1.18.4).** OpenCode keeps queued text visible while it is mid-turn, so tmux and herdr delegate the final delivery decision to `fm_composer_queued_enter_verdict` in `bin/fm-composer-lib.sh` rather than treating visible text alone as a swallowed Enter. The daemon still clears its buffer only on the backend's `empty` success verdict; [`docs/tmux-backend.md`](../../../docs/tmux-backend.md) and [`docs/herdr-backend.md`](../../../docs/herdr-backend.md) own the backend-specific confirmation signals. diff --git a/.agents/skills/firstmate-orca/SKILL.md b/.agents/skills/firstmate-orca/SKILL.md index c6c23b07121..939f6698b9b 100644 --- a/.agents/skills/firstmate-orca/SKILL.md +++ b/.agents/skills/firstmate-orca/SKILL.md @@ -52,15 +52,15 @@ Do not manually patch metadata to make an externally-created Orca terminal look ## Supervision Use `bin/fm-peek.sh`, `bin/fm-send.sh`, `bin/fm-crew-state.sh`, and `bin/fm-teardown.sh` for routine operation. -For steer messages, send short lines through `bin/fm-send.sh '...'`; the stable `fm-` alias also works. -Put long instructions in the task brief or a temporary file and point the crewmate at that file. +For steer messages, use `bin/fm-send.sh '...'`; the stable `fm-` alias also works, and ordinary local text steers may contain newlines because they ride the durable inbox. +Keep initial scope in the task brief; a temporary file remains useful when the instruction includes supporting material the worker should inspect separately. When supervising, treat `state/.meta` as the routing record and Orca's own ids as backend implementation details. The stable firstmate alias is `fm-`. The recorded `terminal=` and `orca_worktree_id=` fields are what backend helpers use under the hood. -If `fm-send` fails to submit, do not immediately repeat the same long instruction. -Peek first, then decide whether the target is busy, waiting on a prompt, stuck behind a popup, or genuinely wedged. +If an ordinary steer fails to enqueue, or a typed-plane `fm-send` fails to submit, do not immediately repeat the instruction. +Read the reported failure and peek first, then decide whether the record exists or the target is busy, waiting on a prompt, stuck behind a popup, or genuinely wedged. For harness-specific interrupts or exits, load `harness-adapters`. ## Recovery diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 1b3c36ecc49..d20a7dfaa62 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -258,9 +258,7 @@ If a pane shows the exit banner, relaunch with `--continue` to resume the sessio While opencode is mid-turn, the composer accepts Enter as a "send when the turn ends" keystroke but does not clear the typed text from the composer until the turn actually finishes. -Without a conversion, every `fm-send` to a busy opencode pane exits non-zero on a -false "Enter swallowed", and every daemon escalation that lands while the -primary is mid-turn is treated as wedged. +Without a conversion, every typed-plane `fm-send` to a busy opencode pane exits non-zero on a false "Enter swallowed", and every daemon escalation that lands while the primary is mid-turn is treated as wedged. Both tmux and herdr delegate this exception to the one policy in `fm_composer_queued_enter_verdict` (`bin/fm-composer-lib.sh`), with backend-specific signals documented in `docs/tmux-backend.md` and `docs/herdr-backend.md`. Regression coverage is `tests/fm-tmux-submit-busy.test.sh`, `tests/fm-composer-lib.test.sh`, and `tests/fm-backend-herdr.test.sh`; the live Herdr Claude guard is `FM_HERDR_SUBMIT_CONFIRM_LIVE=1 tests/fm-herdr-submit-confirm-live-e2e.test.sh`. @@ -407,8 +405,8 @@ Match that TOKEN and never the spinner verb: the same version rendered `Working` **Delivery confirmation is verified on tmux and Herdr only.** Herdr reports a Cursor pane `blocked` in EVERY state - idle, mid-turn, and after - so its native idle-baseline submit path is unreachable for Cursor and the composer branch runs instead; that branch reads a mid-turn row carrying the placeholder beside `ctrl+c to stop`, which is `pending`. `bin/backends/herdr.sh` therefore confirms a Cursor submit from a rendered-footer idle-to-busy transition, taking the baseline before the first Enter so an already-busy pane never confirms. -Zellij, cmux, and Orca share a submit core that never consults that footer, so a Cursor steer there LANDS but `bin/fm-send.sh` reports delivery unconfirmed and exits non-zero. -Treat that as a known limitation of those three backends rather than a lost message: the steer is in the pane and the worker's own recorded state still comes from its transcript fold. +Zellij, cmux, and Orca share a submit core that never consults that footer, so a typed-plane Cursor send there (a harness-native invocation or an explicit backend target; ordinary text steers ride the durable inbox and exit 0 at enqueue) LANDS but `bin/fm-send.sh` reports delivery unconfirmed and exits non-zero. +Treat that as a known limitation of those three backends rather than a lost message: the text is in the pane and the worker's own recorded state still comes from its transcript fold. Teaching the shared core the same transition is deliberately separate work, because it changes the submit path for every harness on those three backends and needs its own live validation on each. The composer's reverse-video placeholder remnant is taught to the ONE fleet-wide screen classifier in `bin/fm-composer-lib.sh`, not to any adapter. diff --git a/.agents/skills/stuck-crewmate-recovery/SKILL.md b/.agents/skills/stuck-crewmate-recovery/SKILL.md index b9b94b27d43..64d809c798d 100644 --- a/.agents/skills/stuck-crewmate-recovery/SKILL.md +++ b/.agents/skills/stuck-crewmate-recovery/SKILL.md @@ -43,7 +43,7 @@ If the worktree or ownership cannot be reconciled safely, leave all state intact Escalate in order: -1. Peek the pane. +1. Peek the pane, and check the task's steering inbox (`state/.inbox/`) for unhandled `*.msg` records - a stale wake naming an unread firstmate instruction means the worker never acknowledged a durable steer, and the record itself shows exactly what was intended. 2. If the crewmate is waiting on a question its brief already answers, answer in one line via `FM_HOME= bin/fm-send.sh` from an active firstmate session unless `FM_HOME` is already set to the active firstmate home. 3. If the crewmate is confused or looping, interrupt with `FM_HOME= bin/fm-control.sh interrupt`, then redirect with one corrective line through `fm-send`. 4. If the crewmate is genuinely wedged after redirection, relaunch it with `FM_HOME= bin/fm-control.sh relaunch --note ''`, which stops the agent, carries the brief plus that note into a replacement in the same local copy, and restores the prior record if the replacement cannot start. diff --git a/AGENTS.md b/AGENTS.md index 06685cee0dd..07d5c73f661 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,6 +95,7 @@ state/ runtime records and signals; gitignored .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown .muse-session muse busy-source binding (sessions root plus task worktree) written by fm-spawn; removed by teardown .cursor-session cursor busy-source binding (projects root, task worktree, prior conversations) written by fm-spawn; removed by teardown + .inbox/ durable steering inbox: sequenced firstmate instruction records the worker acknowledges by moving them into its handled/ subdirectory; written by fm-send, re-rung and escalated by the watcher, removed by teardown (bin/fm-task-inbox-lib.sh) .meta task metadata; each producer script's header owns its exact fields and mutation contract, with docs/configuration.md routing operator-facing backend and trace-context details .herdr-presentation quarantinable attempt and restart-binding journal for Herdr's optional visual projection; never task or endpoint authority; see docs/herdr-backend.md "Presentation spaces" .check.sh authenticated slow poll; the watcher dispatches validated PR data and the byte-identified Relay shim through trusted repository scripts, runs registered custom checks from hash-validated private snapshots, and rejects every other state check without execution @@ -301,7 +302,8 @@ The spawn must resolve a genuine isolated task worktree distinct from the primar After spawning, confirm the worker is processing the brief, handle any trust dialog through `harness-adapters`, and record ship or scout work as under way. A persistent secondmate is recorded in the secondmate registry and runtime state, never as a backlog work item. -Steer a worker with short single-line messages through fail-closed `fm-send`; put long instructions in a file. +Steer a worker with ordinary local text through fail-closed `fm-send`: the message becomes a durable record in the task's steering inbox (multi-line text is legal) and the worker's terminal receives only a constant doorbell line, with the watcher re-ringing an unacknowledged message and escalating a stuck one (`bin/fm-task-inbox-lib.sh`; `bin/fm-send.sh` owns the typed-plane carve-outs). +A remote secondmate steer still crosses the typed terminal channel unchanged until the remote inbox leg lands, so keep those messages short and single-line. When a steer answers an open keyed decision or blocker, pass `fm-send`'s `--resolve-key` so the answer itself closes that decision record at answer time, identically for local and remote workers (contract: `bin/fm-send.sh` header). `fm-send` is the data plane for text the worker should read; never use its key or text paths for interrupt, exit, or other lifecycle control, because routing-marked lifecycle text becomes chat the worker reasons about instead of executing. Drive a worker's lifecycle through `bin/fm-control.sh interrupt|exit|relaunch`, which owns the per-runtime mechanics, verifies each action, and never tears down or discards anything ([`docs/agent-control.md`](docs/agent-control.md)). diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index 3528fd3866b..116cfd23e9a 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -49,6 +49,9 @@ # declared-external-wait verb (FM_CLASSIFY_PAUSED_VERB, default "paused") from # "blocked:": pause for a known external wait expected to clear on its own, # blocked when firstmate must act. +# Every scaffold also carries the steering-inbox receive-and-ack section: +# process state/.inbox/*.msg in order and acknowledge each by moving it to +# handled/ (record, doorbell, and ladder owned by bin/fm-task-inbox-lib.sh). # Ship tasks include a project-memory section so durable project-intrinsic # learnings can be committed to AGENTS.md through the project's delivery path; # it carries the AGENTS.md authoring bar (widely useful knowledge only, pointers @@ -177,6 +180,20 @@ shell_quote() { } STATUS_FILE=$(shell_quote "$STATE/$ID.status") +INBOX_DIR=$(shell_quote "$STATE/$ID.inbox") + +# The receive-and-ack half of the steering-inbox contract, included in every +# scaffold kind. The record format, doorbell line, and re-ring ladder are +# owned by bin/fm-task-inbox-lib.sh; the doorbell itself is self-describing, +# so this section is reinforcement for the natural-checkpoint habit, not the +# only carrier of the instruction. +IFS= read -r -d '' INBOX_SECTION <> "$RELAUNCH_BRIEF" \ || die "could not append the progress note to task $ID's instructions" diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 99594b03506..ec31502ece8 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Send one line of literal text to a crewmate endpoint, then Enter. +# Steer a task by durable record: write the message into the task's steering +# inbox and ring a constant doorbell line into its terminal, best-effort. # Usage: fm-send.sh [--resolve-key ]... # may be an exact task id, a legacy fm- task label resolved # through this home's state/.meta, or an explicit well-formed backend @@ -10,41 +11,89 @@ # Key support is backend-specific: tmux/herdr support Escape, Enter, and C-c; # Orca currently supports Enter and C-c only, and rejects Escape. # -# Text submission is verified: the line is typed ONCE, then Enter is sent and -# retried (Enter only, never retyped) until the target backend confirms a -# submit or reports an inconclusive send. If a swallowed Enter is positively -# confirmed, fm-send exits NON-ZERO so the caller knows the steer did not land -# instead of silently leaving an unsubmitted instruction. -# Exit status contract: 0 = submit confirmed (or, for a remote secondmate -# target, delivered with confirmation pending - see the remote paragraph); -# 3 = the text was typed into the live endpoint and Enter was sent, but the -# submit read-back stayed unconfirmed (verify the pane before any resend, and -# never re-type blindly; a marked request's pending-reply expectation stays -# armed because this outcome is not a proven failure); any other nonzero = the -# send failed and nothing may be assumed delivered. -# Submission dispatches through the target's recorded backend; the tmux adapter -# shares its composer/submit core with the away-mode daemon via bin/fm-tmux-lib.sh. -# Tune with FM_SEND_RETRIES (default 3) / FM_SEND_SLEEP (0.4). -# Slash commands, and codex `$...` skill invocations resolved through harness -# meta, get a longer pre-Enter settle so completion popups do not swallow Enter. +# Two data planes: +# +# INBOX - the default for text to a task recorded in this home. The message is +# appended as a durable sequenced record under state/.inbox/ (newlines are +# legal), and the terminal receives only one short constant self-describing +# doorbell line plus Enter, best-effort. The durable record IS the delivery, +# so the record's fate alone governs the exit: 0 = the steer is durably sent +# (recorded); nonzero = nothing was delivered and a resend is appropriate +# (unresolvable target, an endpoint that cannot be locked and revalidated or +# that retired or changed, an unwritable record) or a decision-close append +# failed after delivery (the error then carries the exact manual close). +# Pending-reply bookkeeping trouble after a durable enqueue NEVER exits +# nonzero: with the recovery marker stored the watcher reconciles it silently, +# and with both the commit and the marker lost the send prints a distinct +# "reply-tracking-degraded (steer delivered, do not resend)" warning instead, +# because a resend-inviting status there would duplicate a delivered +# instruction. There is no delivered-unconfirmed +# outcome on this plane: "did the doorbell land" is no longer the question - +# "was the message acted on" is, and that is answered asynchronously by the +# worker's acknowledgement move into handled/, with the watcher re-ringing an +# unacknowledged message and escalating a stuck one. bin/fm-task-inbox-lib.sh +# owns the record format, the doorbell line, and the re-ring ladder. The +# composer pre-check before the ring is ADVISORY only: when the composer +# visibly holds pending text the ring is skipped with a notice and the watcher +# re-rings later; no composer verdict is delivery proof on this plane, and a +# failed ring never fails the send. +# +# TYPED - everything that must reach the terminal itself: a harness-native +# invocation (a leading "/", or a leading "$" to a codex target) must reach +# the harness's own parser; an explicit backend target names an endpoint, not +# a task, and stays typed even when local metadata happens to match it (the +# same boundary that keeps it unmarked and outside --resolve-key); and a +# remote secondmate steer still crosses fm-on.sh unchanged until the remote +# inbox leg lands. These type the literal +# text through the target backend's verified submit core: typed ONCE, then +# Enter retried (never retyped) until the backend confirms a submit or reports +# an inconclusive send. Typed-plane exit contract: 0 = submit confirmed (or, +# for a remote secondmate target, delivered with confirmation pending - see +# the remote paragraph); 3 = the text was typed into the live endpoint and +# Enter was sent, but the submit read-back stayed unconfirmed (verify the pane +# before any resend, and never re-type blindly; a marked request's +# pending-reply expectation stays armed because this outcome is not a proven +# failure); any other nonzero = the send failed and nothing may be assumed +# delivered. Submission dispatches through the target's recorded backend; the +# tmux adapter shares its composer/submit core with the away-mode daemon via +# bin/fm-tmux-lib.sh. Tune with FM_SEND_RETRIES (default 3) / FM_SEND_SLEEP +# (0.4). Slash commands, and codex `$...` skill invocations resolved through +# harness meta, get a longer pre-Enter settle so completion popups do not +# swallow Enter. +# +# Stage-1 compatibility boundary: classification uses the original pre-marker +# text, but secondmate marking still precedes every typed submission. Therefore +# a marked parser-native secondmate invocation intentionally reaches the harness +# as marker-prefixed chat rather than executing as a parser command. This is a +# pre-existing interaction retained for byte compatibility in this local-inbox +# stage; do not move the marker behind the invocation or omit it here. Follow-up +# fm-send-secondmate-harness-invocation-r1 owns that behavior. # # From-firstmate marker: when the resolved target is a task selector whose meta -# records kind=secondmate, the text uses the live-charter-compatible +# records kind=secondmate, the message uses the live-charter-compatible # from-firstmate carrier owned by bin/fm-operational-input.sh so the secondmate # routes its reply via its status file or a status-pointed doc instead of -# stranding it in chat the main firstmate never reads. A crewmate/scout target, +# stranding it in chat the main firstmate never reads. On the inbox plane the +# marker travels verbatim inside the recorded body. A crewmate/scout target, # an explicit backend-target escape-hatch target, and the --key path are never # marked - their behavior is unchanged. # # Parent-owned pending-reply expectation: every newly marked secondmate request # also receives a privacy-safe correlation id and a durable parent record under # state/pending-replies/ before delivery (bin/fm-pending-reply-lib.sh). Delivery -# success and reply success are separate facts: a successful submit never -# resolves the expectation, and an unconfirmed submit (exit 3) keeps it armed -# rather than dropping it; only a proven send failure discards it. Set -# FM_PENDING_REPLY_EXISTING_CORR= when re-sending a recovery request for an -# already-open expectation so a second record is not created. Direct unmarked -# captain input never creates one. +# success and reply success are separate facts: delivery never resolves the +# expectation. On the inbox plane the durable enqueue IS delivery to the task's +# record, so the expectation is marked delivered at enqueue time; when that +# bookkeeping commit fails after its durable recovery marker is stored, the +# send remains successful and watcher reconciliation owns the repair, and when +# the commit and marker are BOTH lost the send still remains successful with a +# reply-tracking-degraded warning naming the expectation an operator must +# inspect (it can no longer reconcile or escalate on its own). Only a +# failed enqueue discards the expectation. On the typed plane an unconfirmed submit (exit 3) keeps +# it armed rather than dropping it, and only a proven send failure discards it. +# Set FM_PENDING_REPLY_EXISTING_CORR= when re-sending a recovery request +# for an already-open expectation so a second record is not created. Direct +# unmarked captain input never creates one. # # Remote secondmate delivery: the send crosses fm-on.sh to a host-local leg # (bin/fm-remote-secondmate-control.sh cmd_send) that runs this same verified @@ -62,16 +111,19 @@ # # Decision closure (answerer-closes): pass --resolve-key (repeatable, # before the message) when this send answers an open keyed needs-decision: or -# blocked: record in the target task's state/.status. After the submit is -# confirmed, fm-send itself appends the closing -# "resolved [key=]: answered: " line to that status file, -# so the captain-facing OPEN DECISIONS record closes at answer time and never -# depends on the busy worker writing a matching resolved line. The close is a -# LOCAL append for every target kind - crewmate, scout, local secondmate, and -# remote secondmate alike - because the open-decision ledger fm-wake-drain -# folds lives in this home's own state dir (a remote mate's escalations reach -# it through the parent-replies ingest); only the answer message crosses the -# backend or remote transport. +# blocked: record in the target task's state/.status. fm-send itself +# appends the closing "resolved [key=]: answered: " line +# to that status file, so the captain-facing OPEN DECISIONS record closes at +# answer time and never depends on the busy worker writing a matching resolved +# line. On the inbox plane the close happens at ENQUEUE time, because enqueue +# is durable delivery to the task's record; the worker reading the answer late +# is covered by the acknowledgement re-ring ladder. On the typed plane it +# still waits for the confirmed submit. The close is a LOCAL append for every +# target kind - crewmate, scout, local secondmate, and remote secondmate alike +# - because the open-decision ledger fm-wake-drain folds lives in this home's +# own state dir (a remote mate's escalations reach it through the +# parent-replies ingest); only the answer message crosses the backend or +# remote transport. # # Chat is also a channel that carries keyed captain answers, so the same flag # feeds bin/fm-captain-hold.sh's one keyed-answer intake for any key that names @@ -97,12 +149,13 @@ # refused with --key, with an explicit backend target (no task ledger in this # home), and with an empty message. # -# After a successful text submit fm-send pauses FM_SEND_SETTLE seconds (default 1, -# 0 disables) before returning: submit confirmation only proves the text was -# accepted, but the harness needs a beat to spin up the turn before its busy -# footer appears, so an immediate peek would otherwise see the stale idle pane. -# The pause is fm-send-only; the shared submit core (used by the away-mode daemon, -# which only needs "submitted") does not pay it, and the --key path is unaffected. +# After a successful TYPED-plane submit fm-send pauses FM_SEND_SETTLE seconds +# (default 1, 0 disables) before returning: submit confirmation only proves the +# text was accepted, but the harness needs a beat to spin up the turn before its +# busy footer appears, so an immediate peek would otherwise see the stale idle +# pane. The pause is typed-plane-only; the inbox plane, the shared submit core +# (used by the away-mode daemon, which only needs "submitted"), and the --key +# path do not pay it. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -143,6 +196,8 @@ fi . "$SCRIPT_DIR/fm-line-cap-lib.sh" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-task-inbox-lib.sh +. "$SCRIPT_DIR/fm-task-inbox-lib.sh" FM_GUARD_CONTINUE_LINE='This is a supervision warning only; the requested message WILL still be sent.' "$SCRIPT_DIR/fm-guard.sh" || true @@ -461,8 +516,9 @@ if [ -n "$RESOLVE_KEYS" ]; then done fi -# Close each answered decision in this home's ledger, only after delivery is -# fully confirmed. An append failure exits nonzero with the manual close +# Close each answered decision in this home's ledger, only after the answer is +# durably sent: enqueued on the inbox plane, submit-confirmed on the typed +# plane. An append failure exits nonzero with the manual close # command; the decision then stays open and re-surfaces, never silently lost. # The close is this home's own bookkeeping, written by the very turn that # answered the decision, so it goes through the guarded self-announced append @@ -568,6 +624,100 @@ else exit 1 fi fi + # Data-plane selection (see the header): text addressed to a task selector + # resolved through this home's metadata rides the inbox plane, unless it is + # a harness-native invocation that must reach the harness's own parser - a + # leading "/" (slash command), or a leading "$" to a codex target (skill + # invocation). Remote targets keep the typed transport until the remote + # inbox leg lands, and an explicit backend target stays typed even when it + # happens to match local metadata: it names an endpoint, not a task, the + # same boundary that keeps it unmarked and outside --resolve-key. + # Classification reads the pre-marker text so a marked secondmate request + # and a plain crewmate steer classify identically. It deliberately does NOT + # promise that a marked parser-native secondmate request executes as a parser + # command: the pre-existing marker-first wire bytes are retained in stage 1. + INBOX_PLANE=0 + if [ "$TARGET_BACKEND" != remote ] && [ -n "$TARGET_SELECTOR" ]; then + case "$RESOLVE_ANSWER_TEXT" in + /*) ;; + \$*) [ "$TARGET_HARNESS" = codex ] || INBOX_PLANE=1 ;; + *) INBOX_PLANE=1 ;; + esac + fi + if [ "$INBOX_PLANE" = 1 ]; then + INBOX_TASK_ID=$(fm_send_id_from_meta "$TARGET_META") + INBOX_META_LOCK=$(fm_meta_lock_path "$TARGET_META") || exit 1 + if ! fm_task_inbox_lock_acquire "$INBOX_META_LOCK"; then + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + fi + echo "error: steer not sent to $INBOX_TASK_ID: its task metadata could not be locked for final delivery validation" >&2 + exit 1 + fi + CURRENT_INBOX_TARGET= + CURRENT_INBOX_BACKEND= + if [ -f "$TARGET_META" ]; then + CURRENT_INBOX_TARGET=$(fm_backend_target_of_meta "$TARGET_META") + CURRENT_INBOX_BACKEND=$(fm_backend_of_meta "$TARGET_META") + fi + if [ "$CURRENT_INBOX_TARGET" != "$T" ] \ + || [ "$CURRENT_INBOX_BACKEND" != "$TARGET_BACKEND" ] \ + || [ -n "$(fm_meta_get "$TARGET_META" remote_host)" ]; then + fm_lock_release "$INBOX_META_LOCK" + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + fi + echo "error: steer not sent to $INBOX_TASK_ID: the task retired or changed endpoint during target resolution" >&2 + exit 1 + fi + if ! INBOX_RECORD=$(fm_task_inbox_write "$STATE" "$INBOX_TASK_ID" "$MESSAGE"); then + fm_lock_release "$INBOX_META_LOCK" + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + fi + echo "error: steer not sent to $INBOX_TASK_ID: its inbox record could not be written under $STATE/$INBOX_TASK_ID.inbox" >&2 + exit 1 + fi + fm_lock_release "$INBOX_META_LOCK" + # Enqueue IS durable delivery to the task's record: mark the pending + # expectation delivered now, without resolving it - only a correlated + # parent report acknowledges the request. + if [ -n "$PENDING_REPLY_CORR" ]; then + if fm_pending_reply_confirm_delivery "$STATE" "$PENDING_REPLY_CORR"; then + : + else + delivery_commit_status=$? + if [ "$delivery_commit_status" = 2 ]; then + echo "notice: the steer was recorded at $INBOX_RECORD, but its pending-reply delivery commit failed; a durable recovery marker was stored and the watcher will reconcile it. Do not resend." >&2 + else + # Both the commit and its recovery marker failed. The durable inbox + # record is what delivers the steer, so the send still SUCCEEDED: + # a nonzero here would read as undelivered to every automated caller + # and invite a duplicate enqueue - the exact defect this plane + # removes. Surface the degradation as its own distinct, + # non-resend-inviting condition instead: reply tracking for this + # request may not resolve or escalate on its own until an operator + # inspects it. + echo "warning: reply-tracking-degraded (steer delivered, do not resend): the steer was durably recorded at $INBOX_RECORD, but its pending-reply delivery commit and recovery marker both failed, so the reply expectation for this request may not reconcile on its own. Inspect $STATE." >&2 + fi + fi + fi + # The answer is durably sent: close each answered decision at enqueue time + # (answerer-closes; see the header contract). + if [ -n "$RESOLVE_KEYS" ]; then + fm_send_close_resolved_keys "$RESOLVE_ANSWER_TEXT" || exit 1 + fm_send_feed_resolved_holds "$RESOLVE_ANSWER_TEXT" || exit 1 + fi + # Ring the doorbell, best-effort: no ring outcome changes the exit status, + # because the watcher's re-ring ladder owns loss detection from here. + ring_rc=0 + fm_task_inbox_ring "$TARGET_BACKEND" "$T" "$INBOX_RECORD" "$EXPECTED_LABEL" || ring_rc=$? + case "$ring_rc" in + 1) echo "fm-send: doorbell skipped (composer visibly holds pending text); the steer is durably recorded at $INBOX_RECORD and the watcher will re-ring" >&2 ;; + 2) echo "fm-send: doorbell did not reach $T; the steer is durably recorded at $INBOX_RECORD and the watcher will re-ring" >&2 ;; + esac + exit 0 + fi # Slash commands open a completion popup in some TUIs (verified on codex); # submitting too fast selects nothing, so give the popup time to settle before # the (retried) Enter. Codex opens the same kind of popup for a `$` diff --git a/bin/fm-task-inbox-lib.sh b/bin/fm-task-inbox-lib.sh new file mode 100644 index 00000000000..a6543e1af35 --- /dev/null +++ b/bin/fm-task-inbox-lib.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +# fm-task-inbox-lib.sh - the per-task steering inbox: durable records plus a +# constant doorbell. +# +# ONE owner of the steering-inbox contract: the record format, sequence +# allocation, the handled/ acknowledgement, the self-describing doorbell line, +# and the watcher's re-ring ladder policy. bin/fm-send.sh writes and rings, +# bin/fm-watch.sh polls and re-rings, and the brief scaffold (bin/fm-brief.sh) +# tells the worker how to read and acknowledge; none of them restates the +# format. +# +# Design (captain-adopted, data/fm-send-reliability-reframe-s1/report.md): the +# payload moves to the filesystem, which is reliable; the terminal carries only +# a short constant doorbell line, which does not need to be reliable because +# ringing it again is free. A duplicated doorbell is a no-op by construction +# (the worker finds the inbox empty or already handled), a swallowed doorbell +# is detected by the absence of the worker's acknowledgement and re-rung on a +# bounded schedule, and a worker that never acknowledges surfaces through the +# ordinary stale wake into stuck-crewmate-recovery. +# +# Layout under : +# .inbox/NNN.msg one durable steer, numeric sequence, atomic rename +# .inbox/handled/ the worker's `mv` here IS the acknowledgement +# .inbox/.seq.lock serializes sequence allocation across writers +# (the session and the away daemon) +# .inbox/.ring-state watcher re-ring ladder: "\t\t" +# .inbox/.escalated oldest-message name already surfaced as stale, +# so later polls suppress another escalation +# +# Record format (fm_task_inbox_write / fm_task_inbox_body): +# schema=fm-task-inbox.v1 +# at= +# -- +# +# +# Sequence numbers are never reused within a task: allocation scans both the +# inbox root and handled/, so a message is processed at most once per worker +# lifetime even if every doorbell is duplicated. Concurrent writers serialize +# on .seq.lock; the worst racing outcome is ordering, never loss. +# +# Re-ring ladder (fm_task_inbox_due_action): an unhandled message older than +# FM_TASK_INBOX_GRACE_SECS is due one delivery attempt per grace period; an +# attempt may ring or be skipped to protect proven pending composer text. After +# FM_TASK_INBOX_RING_MAX attempts without an acknowledgement it escalates. The +# caller owns the busy check (a busy pane just waits - the record is durable and +# the worker reaches a turn boundary) and the wake emission; this library owns +# only the schedule. If attempt bookkeeping cannot be persisted while the record +# remains unhandled, the caller surfaces that failure instead of retrying +# silently; a concurrently removed inbox is a quiet no-op. Escalation +# deliberately queues the wake before writing the +# deduplication marker: normal polls surface a message once, while a crash or +# marker failure may produce a rare duplicate rather than silently lose a wake. +# +# fm_task_inbox_ring requires bin/fm-backend.sh's dispatch (sourced below); the +# other helpers are dependency-light. Sourced by bin/fm-send.sh, bin/fm-watch.sh, +# and tests. No side effects on source beyond its sourced libraries. +# +# Tunables (env): +# FM_TASK_INBOX_GRACE_SECS default 90; delivery-attempt grace and spacing +# FM_TASK_INBOX_RING_MAX default 3; delivery attempts before escalation + +_FM_TASK_INBOX_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Both dependencies are canonical lint roots in their own right. Keep them as +# analysis boundaries here so ShellCheck's external-source traversal does not +# recursively duplicate the full backend graph for every inbox consumer. +# shellcheck source=/dev/null +. "$_FM_TASK_INBOX_LIB_DIR/fm-wake-lib.sh" +# shellcheck source=/dev/null +. "$_FM_TASK_INBOX_LIB_DIR/fm-backend.sh" + +FM_TASK_INBOX_SCHEMA='fm-task-inbox.v1' +FM_TASK_INBOX_GRACE_DEFAULT=90 +FM_TASK_INBOX_RING_MAX_DEFAULT=3 +FM_TASK_INBOX_LOCK_WAIT_DEFAULT=5 + +fm_task_inbox_grace_secs() { + local g=${FM_TASK_INBOX_GRACE_SECS:-$FM_TASK_INBOX_GRACE_DEFAULT} + case "$g" in ''|*[!0-9]*) g=$FM_TASK_INBOX_GRACE_DEFAULT ;; esac + printf '%s' "$g" +} + +fm_task_inbox_ring_max() { + local m=${FM_TASK_INBOX_RING_MAX:-$FM_TASK_INBOX_RING_MAX_DEFAULT} + case "$m" in ''|*[!0-9]*) m=$FM_TASK_INBOX_RING_MAX_DEFAULT ;; esac + printf '%s' "$m" +} + +fm_task_inbox_dir() { # + printf '%s/%s.inbox' "$1" "$2" +} + +fm_task_inbox_handled_dir() { # + printf '%s/%s.inbox/handled' "$1" "$2" +} + +# Numeric sequence of one record basename, or fail for a non-record name. +fm_task_inbox_seq_of() { # + local n=${1%.msg} + [ "$n" != "$1" ] || return 1 + case "$n" in ''|*[!0-9]*) return 1 ;; esac + printf '%s' "$((10#$n))" +} + +# Next unused sequence, scanning the inbox root AND handled/ so an +# acknowledged sequence is never reissued. Caller must hold .seq.lock. +fm_task_inbox_next_seq() { # + local dir=$1 max=0 d f n + for d in "$dir" "$dir/handled"; do + for f in "$d"/*.msg; do + [ -e "$f" ] || continue + n=$(fm_task_inbox_seq_of "${f##*/}") || continue + [ "$n" -le "$max" ] || max=$n + done + done + printf '%03d' "$((max + 1))" +} + +fm_task_inbox_lock_acquire() { # + local lock=$1 wait=${FM_TASK_INBOX_LOCK_WAIT_SECS:-$FM_TASK_INBOX_LOCK_WAIT_DEFAULT} + local deadline probe + case "$wait" in ''|*[!0-9]*) wait=$FM_TASK_INBOX_LOCK_WAIT_DEFAULT ;; esac + probe=$(mktemp "${lock%/*}/.lock-probe.XXXXXX") || return 1 + rm -f "$probe" || return 1 + if [ ! -e "$lock" ] && [ ! -L "$lock" ]; then + fm_lock_try_create "$lock" && return 0 + [ -e "$lock" ] || [ -L "$lock" ] || return 1 + fi + deadline=$(( $(date +%s) + wait )) + while ! fm_lock_try_acquire "$lock"; do + [ "$(date +%s)" -lt "$deadline" ] || return 1 + sleep 0.1 + done +} + +# Durably enqueue one steer: temp-write, then atomic rename into the next +# sequence slot. Prints the record path. Fails without a partial record. +fm_task_inbox_write() { # + local state=$1 task=$2 text=$3 dir lock seq tmp rec status=0 + dir=$(fm_task_inbox_dir "$state" "$task") + mkdir -p "$dir/handled" || return 1 + lock="$dir/.seq.lock" + fm_task_inbox_lock_acquire "$lock" || return 1 + seq=$(fm_task_inbox_next_seq "$dir") + rec="$dir/$seq.msg" + if tmp=$(mktemp "$dir/.staging.XXXXXX"); then + { + printf 'schema=%s\n' "$FM_TASK_INBOX_SCHEMA" + printf 'at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf -- '--\n' + printf '%s' "$text" + } > "$tmp" && mv "$tmp" "$rec" || status=1 + [ "$status" -eq 0 ] || rm -f "$tmp" + else + status=1 + fi + fm_lock_release "$lock" + [ "$status" -eq 0 ] || return 1 + printf '%s' "$rec" +} + +# The exact enqueued text back out of a record. +fm_task_inbox_body() { # + local line + [ -f "$1" ] || return 1 + while IFS= read -r line; do + if [ "$line" = -- ]; then + cat + return 0 + fi + done < "$1" + return 1 +} + +# The constant self-describing doorbell line for the inbox containing a record. +# Self-describing on purpose: a worker whose brief predates the inbox contract +# still receives the complete instruction in the line itself. +fm_task_inbox_doorbell_line() { # + local dir=${1%/*} abs + abs=$(cd "$dir" 2>/dev/null && pwd) || abs=$dir + printf 'Firstmate instruction waiting: list %s/*.msg and, in numeric order, read and act on each, then mv each handled file to %s/handled/.' \ + "$abs" "$abs" +} + +# Ring the doorbell, best-effort: one advisory composer pre-check, then the +# backend's submit machinery with a minimal retry budget, verdict discarded. +# Returns 0 rang, 1 skipped because the composer PROVENLY holds pending text +# (the watcher re-rings later), 2 the backend send failed. No return value is +# delivery proof; the acknowledgement move is the only delivery signal. +# The skip is deliberately narrow: only an exact `pending` verdict defers, +# because there our Enter could submit someone's real half-typed content. +# `pending-unproven` and `unknown` still ring - the worst outcome is a garbled +# CONSTANT line the worker recovers semantically, while skipping on ambiguous +# verdicts would starve a harness whose idle screen the classifier cannot +# positively identify (that classifier is advisory here by design). +fm_task_inbox_ring() { # [expected-label] + local backend=$1 target=$2 rec=$3 label=${4:-} line cstate verdict + line=$(fm_task_inbox_doorbell_line "$rec") + cstate=$(fm_backend_composer_state "$backend" "$target" "$label" 2>/dev/null) || cstate=unknown + case "$cstate" in + pending) return 1 ;; + esac + if ! verdict=$(fm_backend_send_text_submit "$backend" "$target" "$line" 1 0.4 0.3 "$label" 2>/dev/null); then + return 2 + fi + # The verdict is read only to report a failed keystroke; every other value + # (empty, pending, unknown, ...) is deliberately ignored, never proof. + [ "$verdict" != send-failed ] || return 2 + return 0 +} + +# Oldest unhandled record by sequence, or fail when the inbox is empty. +fm_task_inbox_oldest_unhandled() { # + local dir best='' best_n=0 f n + dir=$(fm_task_inbox_dir "$1" "$2") + for f in "$dir"/*.msg; do + [ -e "$f" ] || continue + n=$(fm_task_inbox_seq_of "${f##*/}") || continue + if [ -z "$best" ] || [ "$n" -lt "$best_n" ]; then + best=$f + best_n=$n + fi + done + [ -n "$best" ] || return 1 + printf '%s' "$best" +} + +# The re-ring ladder decision for one task. Prints exactly one of: +# quiet nothing due (healthy, within grace or spacing, +# or already escalated for the current oldest) +# ring one doorbell re-ring is due +# escalate attempt budget spent; surface as stale +# An empty inbox also resets the ladder bookkeeping so the next message starts +# a fresh ladder. +fm_task_inbox_due_action() { # + local dir oldest base now grace max ladder rec_base count last + dir=$(fm_task_inbox_dir "$1" "$2") + if ! oldest=$(fm_task_inbox_oldest_unhandled "$1" "$2"); then + rm -f "$dir/.ring-state" "$dir/.escalated" 2>/dev/null || true + printf 'quiet' + return 0 + fi + base=${oldest##*/} + grace=$(fm_task_inbox_grace_secs) + if [ "$(fm_path_age "$oldest")" -lt "$grace" ]; then + printf 'quiet' + return 0 + fi + count=0 + last=0 + ladder=$(cat "$dir/.ring-state" 2>/dev/null || true) + IFS=$(printf '\t') read -r rec_base count last </dev/null || true + fi + case "$count" in ''|*[!0-9]*) count=0 ;; esac + case "$last" in ''|*[!0-9]*) last=0 ;; esac + if [ "$(cat "$dir/.escalated" 2>/dev/null || true)" = "$base" ]; then + printf 'quiet' + return 0 + fi + max=$(fm_task_inbox_ring_max) + if [ "$count" -ge "$max" ]; then + printf 'escalate %s %s' "$oldest" "$count" + return 0 + fi + now=$(date +%s) + if [ "$((now - last))" -lt "$grace" ]; then + printf 'quiet' + return 0 + fi + printf 'ring %s' "$oldest" +} + +# Advance the ladder after a delivery attempt. A failed ring or a composer- +# protected skip still consumes budget so neither a dead pane nor permanently +# blocked composer can retry silently forever. A concurrently removed inbox is +# a successful no-op; otherwise failure means the caller must surface the +# unwritable ladder while the record remains unhandled. +fm_task_inbox_record_ring() { # + local dir base ladder rec_base count last + dir=$(fm_task_inbox_dir "$1" "$2") + base=${3##*/} + count=0 + ladder=$(cat "$dir/.ring-state" 2>/dev/null || true) + IFS=$(printf '\t') read -r rec_base count last < "$dir/.ring-state"; } 2>/dev/null; then + [ -d "$dir" ] || return 0 + return 1 + fi +} + +# Mark the current oldest as escalated after its stale wake is durably queued, +# suppressing another wake on later polls. Wake-before-marker ordering favors +# at-least-once recovery: a crash or marker failure can cause a rare duplicate; +# stuck-crewmate-recovery owns the message from here. +fm_task_inbox_record_escalated() { # + local dir + dir=$(fm_task_inbox_dir "$1" "$2") + [ -d "$dir" ] || return 0 + if ! { printf '%s\n' "${3##*/}" > "$dir/.escalated"; } 2>/dev/null; then + [ -d "$dir" ] || return 0 + return 1 + fi +} diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index d83ce4d0567..595c73e6a13 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -2803,6 +2803,10 @@ rm -f "$STATE/$ID.turn-ended" "$STATE/$ID.meta" \ "$STATE/$ID.muse-session-current" "$STATE/$ID.cursor-session" \ "$STATE/$ID.control-relaunch" "$STATE/$ID.control-relaunch.meta-prior" \ "$STATE/$ID.control-relaunch.brief-prior" "$STATE/$ID.control-relaunch.note" +# The steering inbox (bin/fm-task-inbox-lib.sh) is runtime state for the +# retired endpoint; teardown only runs after landing is confirmed, so any +# leftover unhandled steer here is moot rather than unlanded work. +rm -rf "$STATE/$ID.inbox" fm_lock_release "$META_LOCK" META_LOCK_HELD=0 if [ "$KIND" != scout ] && [ "$KIND" != secondmate ] && [ "$MODE" != local-only ]; then diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 3a9e1d7ae9f..d8739f206e9 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -157,7 +157,7 @@ family_for_basename() { fm-wake-drain-unread-status.test.sh|\ fm-tool-update-check.test.sh|\ fm-wake-queue.test.sh|fm-watch-arm.test.sh|fm-watch-checkpoint.test.sh|fm-watch-recovery-loop.test.sh|\ - fm-watch-triage.test.sh|\ + fm-watch-triage.test.sh|fm-task-inbox.test.sh|\ fm-watcher-lock.test.sh|fm-inactive-reconcile.test.sh) printf '%s\n' watcher-wake-lock ;; @@ -196,13 +196,15 @@ family_for_basename() { fm-opencode-primary-live-e2e.test.sh|fm-pi-primary-live-e2e.test.sh|\ fm-sessionstart-hook-live-e2e.test.sh|fm-sessionstart-instruction-refresh-live-e2e.test.sh|\ fm-quota-array-dispatch-live-e2e.test.sh|fm-send-secondmate-marker-herdr-e2e.test.sh|\ + fm-send-inbox-doorbell-live-e2e.test.sh|\ fm-herdr-submit-confirm-live-e2e.test.sh) printf '%s\n' live-harness-optin ;; fm-backend-herdr.test.sh|fm-backend-tmux-smoke.test.sh|fm-backend.test.sh|\ fm-tmux-agent-liveness.test.sh|\ fm-control.test.sh|fm-control-relaunch.test.sh|\ - fm-herdr-session-cleanup.test.sh|fm-send-resolve-key.test.sh|fm-send-strict.test.sh|fm-spawn-batch.test.sh|\ + fm-herdr-session-cleanup.test.sh|fm-send-resolve-key.test.sh|fm-send-strict.test.sh|\ + fm-send-inbox.test.sh|fm-spawn-batch.test.sh|\ fm-spawn-dispatch-profile.test.sh|\ fm-trace-context-spawn.test.sh|fm-spawn-worktree-settle.test.sh|\ fm-teardown-endpoint-safety.test.sh) @@ -1003,6 +1005,14 @@ families_for_changed_path() { printf '%s\n' backend-dispatch printf '%s\n' pure-contract-unit ;; + bin/fm-task-inbox-lib.sh) + # The steering-inbox record/doorbell/ladder owner: fm-send's data plane + # (backend-dispatch), the watcher's re-ring check (watcher-wake-lock), + # and the live doorbell guard against real harnesses. + printf '%s\n' backend-dispatch + printf '%s\n' watcher-wake-lock + printf '%s\n' live-harness-optin + ;; bin/fm-bearings-snapshot.sh|bin/fm-fleet-snapshot.sh|bin/fm-fleet-view.sh) printf '%s\n' snapshot-bearings ;; diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 1d883caf8bb..f5a714b4c75 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -52,6 +52,13 @@ # demand-deep-inspection marker, for human inspection # only - never an automatic interrupt, signal, or restart # of the worker or its tool process. +# stale: (unread firstmate instruction: ...) +# the steering-inbox ladder spent its delivery-attempt +# budget on an idle pane without an acknowledgement +# stale: (steering-inbox ladder bookkeeping unwritable: ...) +# an unhandled record's ladder cannot advance; quiet +# successful attempts never wake firstmate +# (bin/fm-task-inbox-lib.sh owns the ladder policy) # check: