From 468f72369f082a8bac3fffbdd7f8c8035961ea1b Mon Sep 17 00:00:00 2001 From: Amplify Logic AI Date: Sun, 23 Aug 2026 10:20:02 +0200 Subject: [PATCH 1/4] feat(bin): make capability green mean first-try validation pass Teardown derives the capability outcome from the repo-scoped no-mistakes run records for the task branch: exactly one completed attempt is green, a pass after earlier attempts is fixed with its fix-round count, a never-completed history is failed, and anything underivable is unknown rather than guessed. fm-send records confirmed supervisor sends per task so teardown can log the steer count. Trailing counts are written only when derivable, and older six-field lines keep parsing side by side with the new format. --- bin/fm-capability-lib.sh | 113 +++++++++++++++++---- bin/fm-send.sh | 12 +++ bin/fm-teardown.sh | 54 +++++++++- docs/configuration.md | 17 +++- docs/scripts.md | 2 +- tests/fm-capability.test.sh | 102 +++++++++++++++++-- tests/fm-gotmp.test.sh | 2 + tests/fm-send-steer-count.test.sh | 122 +++++++++++++++++++++++ tests/fm-teardown.test.sh | 160 ++++++++++++++++++++++++++++++ 9 files changed, 548 insertions(+), 36 deletions(-) create mode 100755 tests/fm-send-steer-count.test.sh diff --git a/bin/fm-capability-lib.sh b/bin/fm-capability-lib.sh index a80c5ec76b..5195b66be1 100755 --- a/bin/fm-capability-lib.sh +++ b/bin/fm-capability-lib.sh @@ -6,9 +6,24 @@ # This header owns the wire format and selection contracts: # - Log path: $FM_HOME/data/capability-outcomes.log (override: FM_CAPABILITY_LOG). # - One append-only line per finished ship/scout teardown: -# ||||| +# |||||[||] # Fields never contain '|' or newlines; invalid fields refuse the append. -# - Outcomes: green (normal landed teardown) or discarded (--force). +# The trailing counts are written only when derivable, each as one +# non-negative integer: fix-rounds is the number of earlier recorded +# pipeline attempts for the task's branch before its final attempt, and +# steers is the confirmed supervisor send count from state/.steers. +# Absent counts are omitted from the line, never guessed; older six-field +# lines without them stay valid forever. +# - Outcomes (green means exactly what it claims): +# green validation passed on the first recorded pipeline attempt +# (fix-rounds 0) +# fixed validation passed only after earlier recorded attempts +# failed validation ran but its newest recorded attempt never +# completed +# unknown no validation result was derivable at teardown (scout +# reports, direct-PR/local-only delivery, or unavailable run +# records) +# discarded work was discarded by an approved --force teardown # - Secondmate teardowns are not recorded (not a worker capability sample). # - task-type is a free-form slug from meta task_type= when present, else kind # (ship|scout). Firstmate should pass a stable slug at spawn for finer bins. @@ -17,9 +32,10 @@ # cost-filtered profile set; this lib never invents a harness outside it and # never bypasses third-party-model / crew-dispatch guards. # - select=capability-recent ranks allowed profiles by green density -# (green / (green+discarded)) in the window; a sampled profile outranks an -# earlier unsampled one only when density > 0; all-zero or absent evidence -# keeps input (configured) order; no samples for a task-type keep the first. +# (first-try greens / all samples) in the window; a sampled profile +# outranks an earlier unsampled one only when density > 0; all-zero or +# absent evidence keeps input (configured) order; no samples for a +# task-type keep the first. # - Scout tax (~10%): advisory CAPABILITY_SCOUT_TAX stderr suggestion of a # different allowed profile; never changes the selected stdout profile. # FM_CAPABILITY_SCOUT_TAX=0 disables; =1 forces; otherwise a roll @@ -68,45 +84,102 @@ fm_capability_field_ok() { } # Append one outcome line. Args: task-type harness model effort outcome +# [fix-rounds] [steers] +# Each count is optional: a non-empty value must be a non-negative integer and +# is appended as the next trailing field; empty means absent (field omitted). # Best-effort: creates data/ as needed; returns non-zero on invalid fields or # write failure but never blocks teardown callers that ignore the status. fm_capability_log_append() { local task_type=$1 harness=$2 model=$3 effort=$4 outcome=$5 - local log_path ts dir + local fix_rounds=${6:-} steers=${7:-} + local log_path ts dir line case "$outcome" in - green|discarded) ;; + green|fixed|failed|unknown|discarded) ;; *) return 1 ;; esac fm_capability_field_ok "$task_type" || return 1 fm_capability_field_ok "$harness" || return 1 fm_capability_field_ok "$model" || return 1 fm_capability_field_ok "$effort" || return 1 + case "$fix_rounds" in + '') ;; + *[!0-9]*) return 1 ;; + esac + case "$steers" in + '') ;; + *[!0-9]*) return 1 ;; + esac log_path=$(fm_capability_log_path) dir=$(dirname "$log_path") mkdir -p "$dir" || return 1 ts=$(fm_capability_now) fm_capability_field_ok "$ts" || return 1 - printf '%s|%s|%s|%s|%s|%s\n' "$ts" "$task_type" "$harness" "$model" "$effort" "$outcome" >> "$log_path" + line="$ts|$task_type|$harness|$model|$effort|$outcome" + case "$fix_rounds" in '') ;; *) line="$line|$fix_rounds" ;; esac + case "$steers" in '') ;; *) line="$line|$steers" ;; esac + printf '%s\n' "$line" >> "$log_path" +} + +# Derive the teardown capability outcome from captured `no-mistakes runs` text. +# Args: branch runs-output (empty when unavailable). Rows are newest-first, +# whitespace-separated: