diff --git a/.cargo/audit.toml b/.cargo/audit.toml index fac178b3..978b151a 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -8,4 +8,11 @@ ignore = [ # rustyclawd-tools → octocrab → jsonwebtoken → rsa. # Tracked upstream: https://github.com/RustCrypto/RSA/issues/19 "RUSTSEC-2023-0071", + # RUSTSEC-2026-0204: invalid pointer dereference in the `fmt::Display` impl + # for `Atomic`/`Shared` in `crossbeam-epoch`. Issued 2026-07-06 with no + # fixed upstream release. Transitive dep via + # simard → rustyclawd-tools → moka → crossbeam-epoch. Only reachable by + # `Display`-formatting a null crossbeam `Atomic`/`Shared`, which Simard + # never does. Tracked upstream: https://rustsec.org/advisories/RUSTSEC-2026-0204 + "RUSTSEC-2026-0204", ] diff --git a/deny.toml b/deny.toml index 7f74fa8f..586fe377 100644 --- a/deny.toml +++ b/deny.toml @@ -49,6 +49,13 @@ ignore = [ # exemption in .cargo/audit.toml. Tracked: # https://github.com/RustCrypto/RSA/issues/19 { id = "RUSTSEC-2023-0071", reason = "rsa Marvin timing side-channel; no upstream fix; transitive JWT-verify only (rustyclawd-tools->octocrab->jsonwebtoken). Tracked: https://github.com/RustCrypto/RSA/issues/19" }, + # crossbeam-epoch — invalid pointer dereference in the `fmt::Display` impl + # for `Atomic`/`Shared` when the pointer is null. Issued 2026-07-06 with NO + # fixed upstream release, so it fails by default and must be exempted here. + # Transitive: simard -> rustyclawd-tools -> moka -> crossbeam-epoch. Only + # reachable by `Display`-formatting a null crossbeam `Atomic`/`Shared`, which + # Simard never does, so the dereference is unreachable in our usage. + { id = "RUSTSEC-2026-0204", reason = "crossbeam-epoch fmt::Display null-pointer deref; no upstream fix (issued 2026-07-06); transitive via simard->rustyclawd-tools->moka; unreachable (Simard never Display-formats crossbeam Atomic/Shared). Tracked: https://rustsec.org/advisories/RUSTSEC-2026-0204" }, ] # ── Licenses ───────────────────────────────────────────────────────────────── diff --git a/docs/reference/ci-health-sweep.md b/docs/reference/ci-health-sweep.md new file mode 100644 index 00000000..046dbd5b --- /dev/null +++ b/docs/reference/ci-health-sweep.md @@ -0,0 +1,128 @@ +--- +title: CI-Health Sweep — Governed-Fleet Reference +description: Reference for simard::ci_health and the `simard ci-health` subcommand — the codified, reproducible governed-fleet CI-health sweep that classifies each default-branch workflow as green, actionable_failure, or ignored(reason). +last_updated: 2026-07-06 +review_schedule: as-needed +owner: simard +doc_type: reference +status: implemented +related: + - ./stewardship-api.md + - ../concepts/stewardship-mode.md + - ./cross-repo-merge-authority.md + - ../../src/ci_health/mod.rs + - ../../src/operator_cli/ci_health.rs +--- + +# CI-Health Sweep — Governed-Fleet Reference + +Module: `simard::ci_health` +Source: `src/ci_health/` +CLI: `simard ci-health` + +The CI-health sweep is a precise, reproducible check of whether every **active** +default-branch workflow across the amplihack ecosystem (Simard + its governed +sibling repos) is green. It exists so the standing *CI-health stewardship* goal +produces verifiable, re-runnable evidence instead of hand-rolled `gh run list` +claims. + +## Why a codified sweep + +A naive `gh run list --branch main` sweep cannot distinguish an **actionable +active-CI failure** from a **non-actionable non-green signal**. Two real cases +made that ambiguity concrete: + +- **azlin** has seven agentic scheduled workflows (Code Quality Tracker, CI/CD + Workflow Health Monitor, …) whose *latest* default-branch run is `failure` — + but every one is `disabled_manually`. A disabled workflow cannot run again, so + its stale failure is not active CI. +- **agent-kgpacks** "Build Knowledge Pack" latest run is `cancelled`/`skipped` — + a non-failure conclusion, not a broken build. + +A sweep that reads only run conclusions either (a) mislabels those as failures, +or (b) glosses over them and claims "every workflow is `success`" — which is +literally false for those latest runs. Carrying the workflow *enablement state* +alongside the run conclusion is what lets the sweep be both correct and honest. + +## Classification + +For each workflow, the sweep reads its enablement state and the latest run on +the repo's default branch, then classifies it: + +| Verdict | Condition | +|---|---| +| `green` | workflow **active** and latest run concluded `success` | +| `actionable_failure` | workflow **active** and latest run concluded `failure`, `timed_out`, or `startup_failure` | +| `ignored` | any of: workflow disabled (`workflow_disabled`); non-failure conclusion such as cancelled/skipped/neutral/action_required/stale (`non_failure_conclusion:`); no default-branch run (`no_default_branch_run`); run not completed (`run_in_progress`) | + +The **fleet is green iff it contains zero actionable failures.** Disabled +workflows, cancelled/skipped runs, and in-progress runs never fail the fleet; +an active workflow whose latest run genuinely failed always does. + +Classification ([`classify::build_report`]) is a total, pure function over the +[`FleetSnapshot`] — no I/O, no `gh`, no clock — so the verdict is deterministic +and exhaustively unit-tested. + +## Module layout + +``` +src/ci_health/ +├── mod.rs public entrypoint, GOVERNED_REPOS, sweep_live/sweep_fixture/report_to_json +├── types.rs WorkflowState, RunConclusion, WorkflowRun/Snapshot, RepoSnapshot, FleetSnapshot +├── classify.rs WorkflowVerdict, IgnoreReason, build_report, FleetReport (serializable DTOs) +├── gh.rs GhWorkflowClient trait, RealGhWorkflowClient, pure parse/join helpers, fixture loader +├── report.rs render_human +└── tests.rs unit tests +``` + +## `simard ci-health` + +``` +simard ci-health [--json] [--from-json ] + + --json Emit the FleetReport as JSON (default: human table). + --from-json Classify an offline snapshot fixture instead of calling + `gh` (the fixture shape mirrors the live snapshot). +``` + +- Without `--from-json`, the sweep reads live GitHub state via `gh` for every + slug in [`ci_health::GOVERNED_REPOS`]: the repo's default branch + (`gh repo view`), workflow states + ids (`gh workflow list --json name,state,id`), and + the latest default-branch run per workflow + (`gh run list --branch --json workflowName,workflowDatabaseId,status,conclusion,event,createdAt,databaseId`). + Runs are matched to workflows by the unique `workflowDatabaseId`, not the + (non-unique) display name, so two workflow files sharing a `name:` never + collapse onto one run. + Because that branch-wide run list is windowed, any **active** workflow with no + run inside the window is queried directly (`gh run list --workflow --limit 1`) + so a stale failing run of an infrequently-triggered workflow can never be + silently dropped and reported as green. +- **Exit code** follows the verdict: `0` when the fleet is green, non-zero when + any actionable failure exists (mirrors `simard self-health`). + +The human report leads with a greppable banner (`CI-HEALTH: GREEN` / +`CI-HEALTH: FAILING`) and a per-repo breakdown; each actionable failure is +hoisted to the top with a direct run URL. The `--json` report is the same data +as a stable `FleetReport` object. + +### Governed fleet + +`GOVERNED_REPOS` is the source of truth in code for the swept slugs; it mirrors +the ecosystem table in `prompt_assets/simard/engineer_system.md` (note +`amplihack` → `amplihack-rs` on GitHub). + +## Reproducing a captured sweep + +The offline path makes any sweep reproducible without network access: capture a +snapshot fixture in the shape under `tests/gadugi/fixtures/ci-health-*.json`, +then `simard ci-health --from-json `. The gadugi scenario +`tests/gadugi/ci-health-sweep.yaml` uses two committed fixtures to assert that +disabled/cancelled/in-progress signals stay green while a genuine active-CI +failure turns the fleet red. + +## Related + +- Concept: [Goal Stewardship Mode](../concepts/stewardship-mode.md) +- [Stewardship API](./stewardship-api.md) — orchestrator-failure → issue routing +- [Cross-Repo Merge Authority](./cross-repo-merge-authority.md) +- Source: `src/ci_health/`, `src/operator_cli/ci_health.rs` diff --git a/docs/reference/dependency-trust-policy.md b/docs/reference/dependency-trust-policy.md index c20960c5..cbbb8f47 100644 --- a/docs/reference/dependency-trust-policy.md +++ b/docs/reference/dependency-trust-policy.md @@ -1,7 +1,7 @@ --- title: Dependency trust policy description: "Reference for cargo-vet transitive-dependency trust certification, the supply-chain/ baseline, trusted-crate and exemption criteria, and the advisory-resolution workflow." -last_updated: 2026-06-28 +last_updated: 2026-07-06 review_schedule: as-needed owner: simard doc_type: reference @@ -216,7 +216,7 @@ flowchart TD ### The `rsa` exemption (RUSTSEC-2023-0071) -The one standing exemption is `rsa` / **RUSTSEC-2023-0071** (the "Marvin" +One standing exemption is `rsa` / **RUSTSEC-2023-0071** (the "Marvin" timing side-channel): - **No fixed release exists** upstream. @@ -231,6 +231,26 @@ It is exempted **once per tool**, with identical justification, in `.cargo/audit.toml` (existing) and `deny.toml` (added for #2260), and is re-checked whenever a fixed `rsa` release ships. +### The `crossbeam-epoch` exemption (RUSTSEC-2026-0204) + +The second standing exemption is `crossbeam-epoch` / **RUSTSEC-2026-0204** +(invalid pointer dereference in the `fmt::Display` impl for `Atomic`/`Shared`): + +- **No fixed release exists** upstream (issued 2026-07-06 with empty + *Patched*/*Unaffected* fields), so it fails `cargo deny check advisories` + and `cargo audit` by default and must be exempted explicitly. +- It reaches the graph **transitively**: + `rustyclawd-tools → moka → crossbeam-epoch`. +- The dereference only occurs when `Display`-formatting a **null** crossbeam + `Atomic`/`Shared`; Simard never formats crossbeam pointers, so the faulty + path is unreachable in Simard's usage. +- Tracked upstream: . + +Like `rsa`, it is exempted **once per tool** with identical justification in +`.cargo/audit.toml` and `deny.toml`, and is re-checked whenever a fixed +`crossbeam-epoch` release (or a `moka`/`rustyclawd-tools` bump that drops it) +ships. + ### Transitive unmaintained / unsound advisories These advisories are *not* vulnerabilities and reach the graph only @@ -259,9 +279,9 @@ start failing — a *new* unmaintained advisory landing on a *direct* dependency or one of these being re-classified as a vulnerability or pulled in directly — the `workspace` scope forces an explicit decision: carry a *temporary* justified `ignore` (ID + "via ``, no upgrade yet" + tracking link) in `deny.toml` -and `.cargo/audit.toml` until the bump lands. The only **permanent** `ignore` in -the policy remains `rsa` (RUSTSEC-2023-0071), the one advisory with no upstream -fix. +and `.cargo/audit.toml` until the bump lands. The **permanent** `ignore`s in +the policy are `rsa` (RUSTSEC-2023-0071) and `crossbeam-epoch` +(RUSTSEC-2026-0204), the two advisories with no upstream fix. ## Workflow: vetting a crate or resolving an advisory diff --git a/mkdocs.yml b/mkdocs.yml index 242d6dbf..85021de1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -277,6 +277,7 @@ nav: - Goal Coverage Allocation: reference/goal-coverage-allocation.md - spawn_agent_for_goal API: reference/spawn-agent-for-goal.md - Stewardship API: reference/stewardship-api.md + - CI-Health Sweep: reference/ci-health-sweep.md - Simard Whisperer API: reference/simard-whisperer-api.md - Maximum Safe Parallelism: reference/maximum-safe-parallelism.md - Concurrent Engineer Dispatch: reference/concurrent-engineer-dispatch.md diff --git a/src/ci_health/classify.rs b/src/ci_health/classify.rs new file mode 100644 index 00000000..d6bbb6ea --- /dev/null +++ b/src/ci_health/classify.rs @@ -0,0 +1,197 @@ +//! Pure CI-health classification — the root-cause fix. +//! +//! Given a [`FleetSnapshot`], decide, per workflow, whether its latest +//! default-branch run is a **genuine actionable failure**, a **green** result, +//! or a non-actionable signal to **ignore** (with a reason). The fleet is +//! healthy iff no workflow is an actionable failure. +//! +//! This module is deliberately free of `gh`, I/O, and time: it is a total +//! function over the snapshot, which makes the sweep's verdict reproducible +//! and exhaustively unit-testable. + +use serde::Serialize; + +use super::types::{FleetSnapshot, RunConclusion, WorkflowSnapshot}; + +/// Why a workflow's latest run is not counted as an actionable failure. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum IgnoreReason { + /// The workflow is turned off (`disabled_manually`/`disabled_inactivity`); + /// a stale last-run failure cannot recur and is not active CI. + WorkflowDisabled, + /// The latest run completed with a non-failure conclusion such as + /// `cancelled`, `skipped`, `neutral`, `action_required`, or `stale`. + NonFailureConclusion(RunConclusion), + /// The workflow has never run on the default branch. + NoRun, + /// The latest run has not completed yet. + InProgress, +} + +/// The verdict for a single workflow's latest default-branch run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum WorkflowVerdict { + /// Active workflow whose latest run concluded `success`. + Green, + /// Active workflow whose latest run genuinely failed — the only verdict + /// that makes the fleet non-green. + ActionableFailure { conclusion: RunConclusion }, + /// A non-actionable signal; see [`IgnoreReason`]. + Ignored { reason: IgnoreReason }, +} + +/// Classify one workflow. Disabled state is checked **first** so an active- +/// looking stale failure on a turned-off workflow is correctly ignored. +pub fn classify_workflow(wf: &WorkflowSnapshot) -> WorkflowVerdict { + if wf.state.is_disabled() { + return WorkflowVerdict::Ignored { + reason: IgnoreReason::WorkflowDisabled, + }; + } + let Some(run) = &wf.latest_run else { + return WorkflowVerdict::Ignored { + reason: IgnoreReason::NoRun, + }; + }; + if run.status != "completed" { + return WorkflowVerdict::Ignored { + reason: IgnoreReason::InProgress, + }; + } + match &run.conclusion { + Some(c) if c.is_actionable_failure() => WorkflowVerdict::ActionableFailure { + conclusion: c.clone(), + }, + Some(RunConclusion::Success) => WorkflowVerdict::Green, + Some(other) => WorkflowVerdict::Ignored { + reason: IgnoreReason::NonFailureConclusion(other.clone()), + }, + // Completed with a null conclusion is indeterminate, not a failure. + None => WorkflowVerdict::Ignored { + reason: IgnoreReason::InProgress, + }, + } +} + +// ── Serializable report DTOs ──────────────────────────────────────────────── + +/// One actionable failure, hoisted to the top of the report for triage. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ActionableFailure { + pub repo: String, + pub default_branch: String, + pub workflow: String, + pub conclusion: String, + pub run_id: Option, + pub run_url: Option, +} + +/// Per-workflow verdict, flattened to stable strings for JSON/consumers. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct WorkflowReport { + pub name: String, + /// One of `green`, `actionable_failure`, `ignored`. + pub verdict: String, + /// GitHub conclusion string for `actionable_failure`; `None` otherwise. + pub conclusion: Option, + /// Ignore reason for `ignored`; `None` otherwise. + pub reason: Option, + pub run_id: Option, +} + +/// Per-repo report. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RepoReport { + pub slug: String, + pub default_branch: String, + pub workflows: Vec, +} + +/// The whole-fleet report; `green` is the gate. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct FleetReport { + pub green: bool, + pub repos_checked: usize, + pub workflows_checked: usize, + pub actionable_failures: Vec, + pub repos: Vec, +} + +fn run_url(slug: &str, run_id: u64) -> String { + format!("https://github.com/{slug}/actions/runs/{run_id}") +} + +fn ignore_reason_str(reason: &IgnoreReason) -> String { + match reason { + IgnoreReason::WorkflowDisabled => "workflow_disabled".to_string(), + IgnoreReason::NonFailureConclusion(c) => { + format!("non_failure_conclusion:{}", c.as_gh_str()) + } + IgnoreReason::NoRun => "no_default_branch_run".to_string(), + IgnoreReason::InProgress => "run_in_progress".to_string(), + } +} + +/// Classify a whole fleet snapshot into a serializable report. The fleet is +/// `green` iff it contains zero actionable failures. +pub fn build_report(snapshot: &FleetSnapshot) -> FleetReport { + let mut actionable_failures = Vec::new(); + let mut repos = Vec::with_capacity(snapshot.repos.len()); + let mut workflows_checked = 0usize; + + for repo in &snapshot.repos { + let mut wf_reports = Vec::with_capacity(repo.workflows.len()); + for wf in &repo.workflows { + workflows_checked += 1; + let run_id = wf.latest_run.as_ref().map(|r| r.database_id); + let verdict = classify_workflow(wf); + let report = match &verdict { + WorkflowVerdict::Green => WorkflowReport { + name: wf.name.clone(), + verdict: "green".to_string(), + conclusion: None, + reason: None, + run_id, + }, + WorkflowVerdict::ActionableFailure { conclusion } => { + actionable_failures.push(ActionableFailure { + repo: repo.slug.clone(), + default_branch: repo.default_branch.clone(), + workflow: wf.name.clone(), + conclusion: conclusion.as_gh_str(), + run_id, + run_url: run_id.map(|id| run_url(&repo.slug, id)), + }); + WorkflowReport { + name: wf.name.clone(), + verdict: "actionable_failure".to_string(), + conclusion: Some(conclusion.as_gh_str()), + reason: None, + run_id, + } + } + WorkflowVerdict::Ignored { reason } => WorkflowReport { + name: wf.name.clone(), + verdict: "ignored".to_string(), + conclusion: None, + reason: Some(ignore_reason_str(reason)), + run_id, + }, + }; + wf_reports.push(report); + } + repos.push(RepoReport { + slug: repo.slug.clone(), + default_branch: repo.default_branch.clone(), + workflows: wf_reports, + }); + } + + FleetReport { + green: actionable_failures.is_empty(), + repos_checked: snapshot.repos.len(), + workflows_checked, + actionable_failures, + repos, + } +} diff --git a/src/ci_health/gh.rs b/src/ci_health/gh.rs new file mode 100644 index 00000000..8351274c --- /dev/null +++ b/src/ci_health/gh.rs @@ -0,0 +1,345 @@ +//! `gh`-backed collection for the CI-health sweep, plus the pure parse/join +//! helpers that turn `gh` JSON (or an offline fixture) into a +//! [`FleetSnapshot`]. +//! +//! The network-touching [`RealGhWorkflowClient`] is intentionally thin; all of +//! the interesting logic — parsing, picking the newest run per workflow, and +//! joining workflows to their latest run — lives in pure functions so it can be +//! unit-tested without a network or `gh`. + +use std::collections::HashMap; + +use serde::Deserialize; + +use crate::error::{SimardError, SimardResult}; + +use super::types::{ + FleetSnapshot, RepoSnapshot, RunConclusion, WorkflowRun, WorkflowSnapshot, WorkflowState, +}; + +/// One row of `gh workflow list -R --all --json name,state,id`. +#[derive(Clone, Debug, Deserialize)] +pub struct RawWorkflowRow { + pub name: String, + pub state: String, + /// Workflow id, used to fetch a specific workflow's latest run directly + /// (`gh run list --workflow `). + pub id: u64, +} + +/// One row of `gh run list -R --branch --json +/// workflowName,workflowDatabaseId,status,conclusion,event,createdAt,databaseId`. +#[derive(Clone, Debug, Deserialize)] +pub struct RawRunRow { + #[serde(rename = "workflowName")] + pub workflow_name: String, + /// The run's parent workflow id. Runs are keyed to workflows by this + /// unique id rather than the (non-unique) display name. + #[serde(rename = "workflowDatabaseId")] + pub workflow_database_id: u64, + pub status: String, + /// GitHub emits `""` for runs that have not completed. + #[serde(default)] + pub conclusion: String, + pub event: String, + #[serde(rename = "createdAt")] + pub created_at: String, + #[serde(rename = "databaseId")] + pub database_id: u64, +} + +// ── Offline fixture shape (mirrors FleetSnapshot with string enums) ────────── + +#[derive(Clone, Debug, Deserialize)] +struct RawFixtureRun { + status: String, + #[serde(default)] + conclusion: Option, + #[serde(default)] + event: String, + #[serde(default)] + created_at: String, + #[serde(default)] + database_id: u64, +} + +#[derive(Clone, Debug, Deserialize)] +struct RawFixtureWorkflow { + name: String, + state: String, + #[serde(default)] + latest_run: Option, +} + +#[derive(Clone, Debug, Deserialize)] +struct RawFixtureRepo { + slug: String, + default_branch: String, + #[serde(default)] + workflows: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +struct RawFixtureFleet { + repos: Vec, +} + +/// Parse the JSON array from `gh workflow list ... --json name,state,id`. +pub fn parse_workflow_rows(json: &[u8]) -> SimardResult> { + serde_json::from_slice(json).map_err(|e| SimardError::CiHealthGhCommandFailed { + reason: format!("failed to parse `gh workflow list` JSON: {e}"), + }) +} + +/// Parse the JSON array from `gh run list ... --json workflowName,...`. +pub fn parse_run_rows(json: &[u8]) -> SimardResult> { + serde_json::from_slice(json).map_err(|e| SimardError::CiHealthGhCommandFailed { + reason: format!("failed to parse `gh run list` JSON: {e}"), + }) +} + +fn conclusion_from_gh(s: &str) -> Option { + if s.is_empty() { + None + } else { + Some(RunConclusion::parse(s)) + } +} + +fn run_from_row(row: &RawRunRow) -> WorkflowRun { + WorkflowRun { + status: row.status.clone(), + conclusion: conclusion_from_gh(&row.conclusion), + event: row.event.clone(), + created_at: row.created_at.clone(), + database_id: row.database_id, + } +} + +/// Reduce a run list to the newest run per workflow, keyed by the workflow's +/// unique id (`workflowDatabaseId`) rather than its non-unique display name. +/// ISO-8601 UTC timestamps sort chronologically as strings, so a lexicographic +/// max picks the latest run. +pub fn latest_run_by_workflow(rows: &[RawRunRow]) -> HashMap { + let mut latest: HashMap = HashMap::new(); + for row in rows { + match latest.get(&row.workflow_database_id) { + Some(existing) if existing.created_at >= row.created_at => {} + _ => { + latest.insert(row.workflow_database_id, row.clone()); + } + } + } + latest +} + +/// Join workflow rows to their latest run into a [`RepoSnapshot`]. Pure — this +/// is the core of the collector and is unit-tested directly. Runs are matched +/// to workflows by id, so two workflows sharing a display name never collapse. +pub fn build_repo_snapshot( + slug: &str, + default_branch: &str, + workflows: &[RawWorkflowRow], + runs: &[RawRunRow], +) -> RepoSnapshot { + let latest = latest_run_by_workflow(runs); + let workflows = workflows + .iter() + .map(|wf| WorkflowSnapshot { + name: wf.name.clone(), + state: WorkflowState::parse(&wf.state), + latest_run: latest.get(&wf.id).map(run_from_row), + }) + .collect(); + RepoSnapshot { + slug: slug.to_string(), + default_branch: default_branch.to_string(), + workflows, + } +} + +/// Load a [`FleetSnapshot`] from an offline fixture (`simard ci-health +/// --from-json `). The fixture shape mirrors [`FleetSnapshot`] using the +/// same GitHub state/conclusion strings the live collector reads. +pub fn snapshot_from_fixture(json: &[u8]) -> SimardResult { + let raw: RawFixtureFleet = + serde_json::from_slice(json).map_err(|e| SimardError::CiHealthGhCommandFailed { + reason: format!("failed to parse CI-health fixture JSON: {e}"), + })?; + let repos = raw + .repos + .into_iter() + .map(|r| RepoSnapshot { + slug: r.slug, + default_branch: r.default_branch, + workflows: r + .workflows + .into_iter() + .map(|w| WorkflowSnapshot { + name: w.name, + state: WorkflowState::parse(&w.state), + latest_run: w.latest_run.map(|run| WorkflowRun { + status: run.status, + conclusion: run.conclusion.as_deref().and_then(conclusion_from_gh), + event: run.event, + created_at: run.created_at, + database_id: run.database_id, + }), + }) + .collect(), + }) + .collect(); + Ok(FleetSnapshot { repos }) +} + +/// Abstract the `gh` reads the sweep needs, so [`collect_fleet`] is testable +/// with a fake client. +pub trait GhWorkflowClient { + fn default_branch(&self, repo: &str) -> SimardResult; + fn list_workflows(&self, repo: &str) -> SimardResult>; + fn list_runs(&self, repo: &str, branch: &str) -> SimardResult>; + /// The single latest run of one workflow (by id) on `branch`, or `None` + /// when it has never run there. Used as a fallback for workflows whose + /// latest run fell outside the branch-wide window fetched by [`list_runs`]. + fn latest_run( + &self, + repo: &str, + branch: &str, + workflow_id: u64, + ) -> SimardResult>; +} + +/// Collect a live snapshot of every governed repo. Fail-loud: any `gh` error +/// aborts the sweep rather than silently reporting a partial fleet as green. +/// +/// `list_runs` fetches a branch-wide window (the N most-recent runs across all +/// workflows). If a workflow appears in that window at all, the newest row for +/// it is necessarily its true latest run. The only gap is an **active** +/// workflow with *zero* rows in the window (its runs are all older than the +/// window) — that would otherwise look like `NoRun` and be silently ignored, +/// hiding a stale failing run. For exactly those workflows we query the latest +/// run directly so a truncated window can never be reported as green. +pub fn collect_fleet(gh: &dyn GhWorkflowClient, repos: &[&str]) -> SimardResult { + let mut out = Vec::with_capacity(repos.len()); + for &repo in repos { + let branch = gh.default_branch(repo)?; + let workflows = gh.list_workflows(repo)?; + let runs = gh.list_runs(repo, &branch)?; + let mut snapshot = build_repo_snapshot(repo, &branch, &workflows, &runs); + for (row, wf) in workflows.iter().zip(snapshot.workflows.iter_mut()) { + let disabled = WorkflowState::parse(&row.state).is_disabled(); + if !disabled + && wf.latest_run.is_none() + && let Some(run) = gh.latest_run(repo, &branch, row.id)? + { + wf.latest_run = Some(run_from_row(&run)); + } + } + out.push(snapshot); + } + Ok(FleetSnapshot { repos: out }) +} + +/// Production [`GhWorkflowClient`] that shells out to the `gh` binary. +#[derive(Default)] +pub struct RealGhWorkflowClient; + +impl RealGhWorkflowClient { + pub fn new() -> Self { + Self + } + + fn run_gh(args: &[&str]) -> SimardResult> { + let output = std::process::Command::new("gh") + .args(args) + .output() + .map_err(|e| SimardError::CiHealthGhCommandFailed { + reason: format!("failed to spawn `gh {}`: {e}", args.join(" ")), + })?; + if !output.status.success() { + return Err(SimardError::CiHealthGhCommandFailed { + reason: format!( + "`gh {}` exited {}: {}", + args.join(" "), + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ), + }); + } + Ok(output.stdout) + } +} + +impl GhWorkflowClient for RealGhWorkflowClient { + fn default_branch(&self, repo: &str) -> SimardResult { + let out = Self::run_gh(&[ + "repo", + "view", + repo, + "--json", + "defaultBranchRef", + "-q", + ".defaultBranchRef.name", + ])?; + let branch = String::from_utf8_lossy(&out).trim().to_string(); + if branch.is_empty() { + return Err(SimardError::CiHealthGhCommandFailed { + reason: format!("`gh repo view {repo}` returned an empty default branch"), + }); + } + Ok(branch) + } + + fn list_workflows(&self, repo: &str) -> SimardResult> { + let out = Self::run_gh(&[ + "workflow", + "list", + "-R", + repo, + "--all", + "--json", + "name,state,id", + ])?; + parse_workflow_rows(&out) + } + + fn list_runs(&self, repo: &str, branch: &str) -> SimardResult> { + let out = Self::run_gh(&[ + "run", + "list", + "-R", + repo, + "--branch", + branch, + "--limit", + "200", + "--json", + "workflowName,workflowDatabaseId,status,conclusion,event,createdAt,databaseId", + ])?; + parse_run_rows(&out) + } + + fn latest_run( + &self, + repo: &str, + branch: &str, + workflow_id: u64, + ) -> SimardResult> { + let id = workflow_id.to_string(); + let out = Self::run_gh(&[ + "run", + "list", + "-R", + repo, + "--branch", + branch, + "--workflow", + &id, + "--limit", + "1", + "--json", + "workflowName,workflowDatabaseId,status,conclusion,event,createdAt,databaseId", + ])?; + Ok(parse_run_rows(&out)?.into_iter().next()) + } +} diff --git a/src/ci_health/mod.rs b/src/ci_health/mod.rs new file mode 100644 index 00000000..c380d31c --- /dev/null +++ b/src/ci_health/mod.rs @@ -0,0 +1,79 @@ +//! Governed-fleet CI-health sweep — a precise, reproducible check of whether +//! every **active** default-branch workflow across the amplihack ecosystem is +//! green. +//! +//! ## Why this exists +//! The CI-health stewardship goal was previously executed by hand-rolling +//! `gh run list` each cycle, which cannot distinguish an *actionable active-CI +//! failure* from a *disabled/cancelled non-failure*. That ambiguity produced +//! imprecise, un-evidenced claims (e.g. "every workflow's latest run is +//! `success`" when a governed repo's latest runs were actually `failure` — on +//! workflows that had been manually **disabled**). This module codifies the +//! sweep so its verdict is reproducible and its evidence is explicit. +//! +//! ## Pipeline +//! 1. [`gh::collect_fleet`] reads, per repo: default branch, workflow states, +//! and the latest default-branch run per workflow. +//! 2. [`classify::build_report`] classifies each workflow into +//! green / actionable-failure / ignored(reason). +//! 3. The fleet is green iff there are zero actionable failures. +//! +//! See `docs/reference/ci-health-sweep.md`. + +pub mod classify; +pub mod gh; +pub mod report; +pub mod types; + +#[cfg(test)] +mod tests; + +pub use classify::{ + ActionableFailure, FleetReport, WorkflowVerdict, build_report, classify_workflow, +}; +pub use gh::{ + GhWorkflowClient, RealGhWorkflowClient, build_repo_snapshot, collect_fleet, + snapshot_from_fixture, +}; +pub use report::render_human; +pub use types::{ + FleetSnapshot, RepoSnapshot, RunConclusion, WorkflowRun, WorkflowSnapshot, WorkflowState, +}; + +use crate::error::{SimardError, SimardResult}; + +/// The amplihack ecosystem fleet: Simard plus its governed sibling repos, by +/// GitHub `owner/repo` slug. Source of truth: the ecosystem table in +/// `prompt_assets/simard/engineer_system.md` (note `amplihack` → `amplihack-rs` +/// on GitHub). +pub const GOVERNED_REPOS: &[&str] = &[ + "rysweet/Simard", + "rysweet/RustyClawd", + "rysweet/amplihack-rs", + "rysweet/azlin", + "rysweet/amplihack-memory-lib", + "rysweet/amplihack-agent-eval", + "rysweet/agent-kgpacks", + "rysweet/amplihack-recipe-runner", + "rysweet/amplihack-xpia-defender", + "rysweet/gadugi-agentic-test", +]; + +/// Run a live sweep of [`GOVERNED_REPOS`] and classify it into a report. +pub fn sweep_live(gh: &dyn GhWorkflowClient) -> SimardResult { + let snapshot = collect_fleet(gh, GOVERNED_REPOS)?; + Ok(build_report(&snapshot)) +} + +/// Classify an offline fixture snapshot into a report (`--from-json`). +pub fn sweep_fixture(json: &[u8]) -> SimardResult { + let snapshot = snapshot_from_fixture(json)?; + Ok(build_report(&snapshot)) +} + +/// Serialize a report as pretty JSON. +pub fn report_to_json(report: &FleetReport) -> SimardResult { + serde_json::to_string_pretty(report).map_err(|e| SimardError::CiHealthGhCommandFailed { + reason: format!("failed to serialize CI-health report: {e}"), + }) +} diff --git a/src/ci_health/report.rs b/src/ci_health/report.rs new file mode 100644 index 00000000..9f6940f5 --- /dev/null +++ b/src/ci_health/report.rs @@ -0,0 +1,65 @@ +//! Human-readable rendering of a [`FleetReport`]. The `--json` path serializes +//! the report directly; this module renders the operator-facing table. + +use super::classify::FleetReport; + +/// Render a fleet report as a plain-text table. The first line is a stable, +/// greppable verdict banner (`CI-HEALTH: GREEN` / `CI-HEALTH: FAILING`). +pub fn render_human(report: &FleetReport) -> String { + let mut out = String::new(); + let banner = if report.green { + "CI-HEALTH: GREEN" + } else { + "CI-HEALTH: FAILING" + }; + out.push_str(banner); + out.push('\n'); + out.push_str(&format!( + "repos checked: {} workflows checked: {} actionable failures: {}\n", + report.repos_checked, + report.workflows_checked, + report.actionable_failures.len() + )); + + if !report.actionable_failures.is_empty() { + out.push_str( + "\nActionable failures (active workflow, latest default-branch run failed):\n", + ); + for af in &report.actionable_failures { + out.push_str(&format!( + " ! {repo} [{branch}] {wf} -> {concl}{url}\n", + repo = af.repo, + branch = af.default_branch, + wf = af.workflow, + concl = af.conclusion, + url = af + .run_url + .as_ref() + .map(|u| format!(" {u}")) + .unwrap_or_default(), + )); + } + } + + out.push_str("\nPer-repo detail:\n"); + for repo in &report.repos { + out.push_str(&format!(" {} [{}]\n", repo.slug, repo.default_branch)); + for wf in &repo.workflows { + let marker = match wf.verdict.as_str() { + "green" => "ok ", + "actionable_failure" => "FAIL", + _ => "skip", + }; + let detail = match wf.verdict.as_str() { + "actionable_failure" => { + format!(" ({})", wf.conclusion.clone().unwrap_or_default()) + } + "ignored" => format!(" ({})", wf.reason.clone().unwrap_or_default()), + _ => String::new(), + }; + out.push_str(&format!(" [{marker}] {}{detail}\n", wf.name)); + } + } + + out +} diff --git a/src/ci_health/tests.rs b/src/ci_health/tests.rs new file mode 100644 index 00000000..9c35f0b8 --- /dev/null +++ b/src/ci_health/tests.rs @@ -0,0 +1,581 @@ +//! Unit tests for the CI-health sweep. These pin the exact distinction that +//! motivated the module: a `disabled` workflow's stale `failure` is **ignored**, +//! while an `active` workflow's `failure` is an **actionable** failure. + +use super::classify::{IgnoreReason, WorkflowVerdict, build_report, classify_workflow}; +use super::gh::{ + GhWorkflowClient, RawRunRow, RawWorkflowRow, build_repo_snapshot, collect_fleet, + latest_run_by_workflow, parse_run_rows, snapshot_from_fixture, +}; +use super::types::{ + FleetSnapshot, RepoSnapshot, RunConclusion, WorkflowRun, WorkflowSnapshot, WorkflowState, +}; +use super::{report_to_json, sweep_fixture}; +use crate::error::SimardResult; + +fn run(status: &str, conclusion: Option<&str>) -> WorkflowRun { + WorkflowRun { + status: status.to_string(), + conclusion: conclusion.map(RunConclusion::parse), + event: "push".to_string(), + created_at: "2026-07-06T00:00:00Z".to_string(), + database_id: 42, + } +} + +fn wf(name: &str, state: WorkflowState, latest: Option) -> WorkflowSnapshot { + WorkflowSnapshot { + name: name.to_string(), + state, + latest_run: latest, + } +} + +// ── enum parsing ──────────────────────────────────────────────────────────── + +#[test] +fn workflow_state_parses_and_flags_disabled() { + assert_eq!(WorkflowState::parse("active"), WorkflowState::Active); + assert!(!WorkflowState::parse("active").is_disabled()); + assert!(WorkflowState::parse("disabled_manually").is_disabled()); + assert!(WorkflowState::parse("disabled_inactivity").is_disabled()); + assert_eq!( + WorkflowState::parse("weird_new_state"), + WorkflowState::Unknown("weird_new_state".to_string()) + ); + assert!(!WorkflowState::parse("weird_new_state").is_disabled()); +} + +#[test] +fn run_conclusion_actionable_set_is_exactly_failures() { + for s in ["failure", "timed_out", "startup_failure"] { + assert!( + RunConclusion::parse(s).is_actionable_failure(), + "{s} must be actionable" + ); + } + for s in [ + "success", + "cancelled", + "skipped", + "neutral", + "action_required", + "stale", + ] { + assert!( + !RunConclusion::parse(s).is_actionable_failure(), + "{s} must NOT be actionable" + ); + } + // Unknown conclusions are never treated as actionable failures. + assert!(!RunConclusion::parse("brand_new").is_actionable_failure()); +} + +// ── classify_workflow ─────────────────────────────────────────────────────── + +#[test] +fn active_success_is_green() { + let v = classify_workflow(&wf( + "CI", + WorkflowState::Active, + Some(run("completed", Some("success"))), + )); + assert_eq!(v, WorkflowVerdict::Green); +} + +#[test] +fn active_failure_is_actionable() { + let v = classify_workflow(&wf( + "CI", + WorkflowState::Active, + Some(run("completed", Some("failure"))), + )); + assert_eq!( + v, + WorkflowVerdict::ActionableFailure { + conclusion: RunConclusion::Failure + } + ); +} + +#[test] +fn active_timed_out_is_actionable() { + let v = classify_workflow(&wf( + "CI", + WorkflowState::Active, + Some(run("completed", Some("timed_out"))), + )); + assert!(matches!(v, WorkflowVerdict::ActionableFailure { .. })); +} + +#[test] +fn disabled_failure_is_ignored_not_actionable() { + // The azlin case: a disabled workflow whose last run failed months ago. + for state in [ + WorkflowState::DisabledManually, + WorkflowState::DisabledInactivity, + ] { + let v = classify_workflow(&wf( + "Code Quality Tracker", + state, + Some(run("completed", Some("failure"))), + )); + assert_eq!( + v, + WorkflowVerdict::Ignored { + reason: IgnoreReason::WorkflowDisabled + } + ); + } +} + +#[test] +fn active_cancelled_is_ignored_non_failure() { + // The agent-kgpacks "Build Knowledge Pack" case. + let v = classify_workflow(&wf( + "Build Knowledge Pack", + WorkflowState::Active, + Some(run("completed", Some("cancelled"))), + )); + assert_eq!( + v, + WorkflowVerdict::Ignored { + reason: IgnoreReason::NonFailureConclusion(RunConclusion::Cancelled) + } + ); +} + +#[test] +fn active_skipped_is_ignored_non_failure() { + let v = classify_workflow(&wf( + "Deploy", + WorkflowState::Active, + Some(run("completed", Some("skipped"))), + )); + assert!(matches!( + v, + WorkflowVerdict::Ignored { + reason: IgnoreReason::NonFailureConclusion(_) + } + )); +} + +#[test] +fn active_no_run_is_ignored_no_run() { + let v = classify_workflow(&wf("Never Ran", WorkflowState::Active, None)); + assert_eq!( + v, + WorkflowVerdict::Ignored { + reason: IgnoreReason::NoRun + } + ); +} + +#[test] +fn in_progress_run_is_ignored() { + let v = classify_workflow(&wf( + "CI", + WorkflowState::Active, + Some(run("in_progress", None)), + )); + assert_eq!( + v, + WorkflowVerdict::Ignored { + reason: IgnoreReason::InProgress + } + ); +} + +#[test] +fn completed_null_conclusion_is_ignored_not_failure() { + let v = classify_workflow(&wf( + "CI", + WorkflowState::Active, + Some(run("completed", None)), + )); + assert_eq!( + v, + WorkflowVerdict::Ignored { + reason: IgnoreReason::InProgress + } + ); +} + +// ── build_report ──────────────────────────────────────────────────────────── + +fn mixed_fleet() -> FleetSnapshot { + FleetSnapshot { + repos: vec![ + RepoSnapshot { + slug: "rysweet/azlin".to_string(), + default_branch: "main".to_string(), + workflows: vec![ + wf( + "CI", + WorkflowState::Active, + Some(run("completed", Some("success"))), + ), + wf( + "Code Quality Tracker", + WorkflowState::DisabledManually, + Some(run("completed", Some("failure"))), + ), + ], + }, + RepoSnapshot { + slug: "rysweet/agent-kgpacks".to_string(), + default_branch: "main".to_string(), + workflows: vec![wf( + "Build Knowledge Pack", + WorkflowState::Active, + Some(run("completed", Some("cancelled"))), + )], + }, + ], + } +} + +#[test] +fn report_is_green_when_only_disabled_and_cancelled_non_green() { + let report = build_report(&mixed_fleet()); + assert!(report.green, "disabled+cancelled must not fail the fleet"); + assert_eq!(report.repos_checked, 2); + assert_eq!(report.workflows_checked, 3); + assert!(report.actionable_failures.is_empty()); +} + +#[test] +fn report_flags_active_failure_with_run_url() { + let mut fleet = mixed_fleet(); + fleet.repos.push(RepoSnapshot { + slug: "rysweet/RustyClawd".to_string(), + default_branch: "main".to_string(), + workflows: vec![WorkflowSnapshot { + name: "CI".to_string(), + state: WorkflowState::Active, + latest_run: Some(WorkflowRun { + status: "completed".to_string(), + conclusion: Some(RunConclusion::Failure), + event: "push".to_string(), + created_at: "2026-07-06T00:00:00Z".to_string(), + database_id: 999, + }), + }], + }); + let report = build_report(&fleet); + assert!(!report.green); + assert_eq!(report.actionable_failures.len(), 1); + let af = &report.actionable_failures[0]; + assert_eq!(af.repo, "rysweet/RustyClawd"); + assert_eq!(af.workflow, "CI"); + assert_eq!(af.conclusion, "failure"); + assert_eq!( + af.run_url.as_deref(), + Some("https://github.com/rysweet/RustyClawd/actions/runs/999") + ); +} + +#[test] +fn report_serializes_to_json_with_stable_keys() { + let json = report_to_json(&build_report(&mixed_fleet())).unwrap(); + assert!(json.contains("\"green\": true")); + assert!(json.contains("\"workflow_disabled\"")); + assert!(json.contains("non_failure_conclusion:cancelled")); +} + +// ── gh join helpers (pure) ────────────────────────────────────────────────── + +fn run_row( + wfname: &str, + wf_id: u64, + created: &str, + status: &str, + conclusion: &str, + id: u64, +) -> RawRunRow { + RawRunRow { + workflow_name: wfname.to_string(), + workflow_database_id: wf_id, + status: status.to_string(), + conclusion: conclusion.to_string(), + event: "push".to_string(), + created_at: created.to_string(), + database_id: id, + } +} + +#[test] +fn latest_run_by_workflow_picks_newest() { + let rows = vec![ + run_row("CI", 1, "2026-06-01T00:00:00Z", "completed", "failure", 1), + run_row("CI", 1, "2026-07-01T00:00:00Z", "completed", "success", 2), + run_row("Docs", 2, "2026-05-01T00:00:00Z", "completed", "success", 3), + ]; + let latest = latest_run_by_workflow(&rows); + assert_eq!(latest.get(&1).unwrap().database_id, 2); + assert_eq!(latest.get(&2).unwrap().database_id, 3); +} + +#[test] +fn build_repo_snapshot_keys_by_id_so_same_named_workflows_dont_collapse() { + // Two DISTINCT workflows share the display name "CI" (allowed by GitHub). + let workflows = vec![ + RawWorkflowRow { + name: "CI".to_string(), + state: "active".to_string(), + id: 100, + }, + RawWorkflowRow { + name: "CI".to_string(), + state: "active".to_string(), + id: 200, + }, + ]; + // id 100's latest is a newer success; id 200's latest is an older failure. + let runs = vec![ + run_row("CI", 100, "2026-07-06T00:00:00Z", "completed", "success", 1), + run_row("CI", 200, "2026-07-05T00:00:00Z", "completed", "failure", 2), + ]; + let snap = build_repo_snapshot("rysweet/foo", "main", &workflows, &runs); + // Keying by id must NOT collapse them onto the newer success. + assert_eq!( + classify_workflow(&snap.workflows[0]), + WorkflowVerdict::Green + ); + assert!(matches!( + classify_workflow(&snap.workflows[1]), + WorkflowVerdict::ActionableFailure { .. } + )); + let report = build_report(&FleetSnapshot { repos: vec![snap] }); + assert!(!report.green); + assert_eq!(report.actionable_failures.len(), 1); +} + +#[test] +fn build_repo_snapshot_joins_and_marks_missing_runs() { + let workflows = vec![ + RawWorkflowRow { + name: "CI".to_string(), + state: "active".to_string(), + id: 100, + }, + RawWorkflowRow { + name: "Nightly".to_string(), + state: "active".to_string(), + id: 200, + }, + ]; + let runs = vec![run_row( + "CI", + 100, + "2026-07-01T00:00:00Z", + "completed", + "success", + 7, + )]; + let snap = build_repo_snapshot("rysweet/foo", "main", &workflows, &runs); + assert_eq!(snap.workflows.len(), 2); + let ci = snap.workflows.iter().find(|w| w.name == "CI").unwrap(); + assert_eq!(ci.latest_run.as_ref().unwrap().database_id, 7); + let nightly = snap.workflows.iter().find(|w| w.name == "Nightly").unwrap(); + assert!(nightly.latest_run.is_none()); +} + +#[test] +fn parse_run_rows_maps_empty_conclusion_to_none() { + let json = br#"[{"workflowName":"CI","workflowDatabaseId":100,"status":"in_progress","conclusion":"","event":"push","createdAt":"2026-07-06T00:00:00Z","databaseId":5}]"#; + let rows = parse_run_rows(json).unwrap(); + let snap = build_repo_snapshot( + "rysweet/foo", + "main", + &[RawWorkflowRow { + name: "CI".to_string(), + state: "active".to_string(), + id: 100, + }], + &rows, + ); + let ci = &snap.workflows[0]; + assert!(ci.latest_run.as_ref().unwrap().conclusion.is_none()); + assert_eq!(classify_workflow(ci), { + WorkflowVerdict::Ignored { + reason: IgnoreReason::InProgress, + } + }); +} + +// ── collect_fleet against a fake client ───────────────────────────────────── + +struct FakeGh; + +impl GhWorkflowClient for FakeGh { + fn default_branch(&self, _repo: &str) -> SimardResult { + Ok("main".to_string()) + } + fn list_workflows(&self, _repo: &str) -> SimardResult> { + Ok(vec![ + RawWorkflowRow { + name: "CI".to_string(), + state: "active".to_string(), + id: 100, + }, + RawWorkflowRow { + name: "Old".to_string(), + state: "disabled_manually".to_string(), + id: 200, + }, + ]) + } + fn list_runs(&self, _repo: &str, _branch: &str) -> SimardResult> { + Ok(vec![ + run_row( + "CI", + 100, + "2026-07-01T00:00:00Z", + "completed", + "success", + 10, + ), + run_row( + "Old", + 200, + "2026-01-01T00:00:00Z", + "completed", + "failure", + 11, + ), + ]) + } + fn latest_run( + &self, + _repo: &str, + _branch: &str, + _workflow_id: u64, + ) -> SimardResult> { + Ok(None) + } +} + +#[test] +fn collect_fleet_builds_green_snapshot_from_fake() { + let snap = collect_fleet(&FakeGh, &["rysweet/a", "rysweet/b"]).unwrap(); + assert_eq!(snap.repos.len(), 2); + let report = build_report(&snap); + assert!(report.green); + assert_eq!(report.workflows_checked, 4); +} + +// ── collect_fleet fallback: a workflow whose latest run fell outside the +// branch-wide window must not be silently reported as NoRun/green ───────── + +struct WindowGapGh { + queried: std::cell::RefCell>, +} + +impl GhWorkflowClient for WindowGapGh { + fn default_branch(&self, _repo: &str) -> SimardResult { + Ok("main".to_string()) + } + fn list_workflows(&self, _repo: &str) -> SimardResult> { + Ok(vec![ + RawWorkflowRow { + name: "CI".to_string(), + state: "active".to_string(), + id: 100, + }, + // An infrequently-triggered active workflow whose last (failing) + // run was pushed out of the branch-wide window by CI's volume. + RawWorkflowRow { + name: "Nightly".to_string(), + state: "active".to_string(), + id: 200, + }, + // A disabled workflow also absent from the window; must NOT be + // queried and must stay ignored. + RawWorkflowRow { + name: "OldDisabled".to_string(), + state: "disabled_manually".to_string(), + id: 300, + }, + ]) + } + fn list_runs(&self, _repo: &str, _branch: &str) -> SimardResult> { + // Only CI appears in the window; Nightly and OldDisabled are absent. + Ok(vec![run_row( + "CI", + 100, + "2026-07-06T00:00:00Z", + "completed", + "success", + 10, + )]) + } + fn latest_run( + &self, + _repo: &str, + _branch: &str, + workflow_id: u64, + ) -> SimardResult> { + self.queried.borrow_mut().push(workflow_id); + Ok(if workflow_id == 200 { + Some(run_row( + "Nightly", + 200, + "2026-06-01T00:00:00Z", + "completed", + "failure", + 20, + )) + } else { + None + }) + } +} + +#[test] +fn collect_fleet_falls_back_for_active_workflow_missing_from_window() { + let gh = WindowGapGh { + queried: std::cell::RefCell::new(Vec::new()), + }; + let snap = collect_fleet(&gh, &["rysweet/x"]).unwrap(); + let report = build_report(&snap); + + // The out-of-window active failure must surface as actionable, not be + // silently ignored as NoRun (which would falsely report the fleet green). + assert!(!report.green); + assert_eq!(report.actionable_failures.len(), 1); + assert_eq!(report.actionable_failures[0].workflow, "Nightly"); + + let queried = gh.queried.borrow(); + // Nightly (absent from window) was queried directly... + assert!(queried.contains(&200)); + // ...CI (present in window) was not re-queried... + assert!(!queried.contains(&100)); + // ...and the disabled workflow was skipped entirely. + assert!(!queried.contains(&300)); +} + +// ── fixture path (drives the gadugi scenario) ─────────────────────────────── + +#[test] +fn fixture_round_trips_real_scenario() { + let fixture = br#"{ + "repos": [ + {"slug":"rysweet/azlin","default_branch":"main","workflows":[ + {"name":"CI","state":"active","latest_run":{"status":"completed","conclusion":"success","event":"push","created_at":"2026-07-04T00:00:00Z","database_id":1}}, + {"name":"Code Quality Tracker","state":"disabled_manually","latest_run":{"status":"completed","conclusion":"failure","event":"schedule","created_at":"2026-06-15T00:00:00Z","database_id":2}} + ]}, + {"slug":"rysweet/agent-kgpacks","default_branch":"main","workflows":[ + {"name":"Build Knowledge Pack","state":"active","latest_run":{"status":"completed","conclusion":"cancelled","event":"issues","created_at":"2026-03-06T00:00:00Z","database_id":3}} + ]} + ] + }"#; + let report = sweep_fixture(fixture).unwrap(); + assert!(report.green); + assert!(report.actionable_failures.is_empty()); + // And a genuine active failure in the same fixture shape flips it red. + let failing = br#"{"repos":[{"slug":"rysweet/x","default_branch":"main","workflows":[ + {"name":"CI","state":"active","latest_run":{"status":"completed","conclusion":"failure","event":"push","created_at":"2026-07-06T00:00:00Z","database_id":9}}]}]}"#; + let snap: FleetSnapshot = snapshot_from_fixture(failing).unwrap(); + assert!(!build_report(&snap).green); +} diff --git a/src/ci_health/types.rs b/src/ci_health/types.rs new file mode 100644 index 00000000..4a37201e --- /dev/null +++ b/src/ci_health/types.rs @@ -0,0 +1,148 @@ +//! Data types for the governed-fleet CI-health sweep. +//! +//! These model exactly the two `gh` surfaces the sweep reads — workflow +//! *enablement state* (`gh workflow list --json name,state`) and the *latest +//! run* of each workflow on the default branch (`gh run list --json +//! workflowName,status,conclusion,event,createdAt,databaseId`). Keeping the +//! enablement state alongside the run conclusion is the whole point: a +//! `disabled_manually` workflow whose last run happened to be a `failure` is +//! **not** an actionable CI failure, and only a type that carries both facts +//! can express that. + +/// GitHub Actions workflow enablement state, from `gh workflow list --json state`. +/// +/// A disabled workflow will never run again, so a stale `failure` on its last +/// run is not an active-CI signal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum WorkflowState { + Active, + DisabledManually, + DisabledInactivity, + /// A state string GitHub returned that we do not model explicitly. + Unknown(String), +} + +impl WorkflowState { + /// Parse the `state` field GitHub emits. Never fails — unmodeled values + /// fall through to [`WorkflowState::Unknown`] so a new GitHub state can + /// never silently look "active". + pub fn parse(s: &str) -> Self { + match s { + "active" => Self::Active, + "disabled_manually" => Self::DisabledManually, + "disabled_inactivity" => Self::DisabledInactivity, + other => Self::Unknown(other.to_string()), + } + } + + /// True when the workflow is turned off and cannot produce new runs. + pub fn is_disabled(&self) -> bool { + matches!(self, Self::DisabledManually | Self::DisabledInactivity) + } + + /// Canonical GitHub string for reporting. + pub fn as_gh_str(&self) -> String { + match self { + Self::Active => "active".to_string(), + Self::DisabledManually => "disabled_manually".to_string(), + Self::DisabledInactivity => "disabled_inactivity".to_string(), + Self::Unknown(s) => s.clone(), + } + } +} + +/// GitHub Actions run conclusion, from `gh run list --json conclusion`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RunConclusion { + Success, + Failure, + Cancelled, + Skipped, + Neutral, + TimedOut, + ActionRequired, + Stale, + StartupFailure, + /// A conclusion string GitHub returned that we do not model explicitly. + Unknown(String), +} + +impl RunConclusion { + /// Parse the `conclusion` field GitHub emits. Never fails. + pub fn parse(s: &str) -> Self { + match s { + "success" => Self::Success, + "failure" => Self::Failure, + "cancelled" => Self::Cancelled, + "skipped" => Self::Skipped, + "neutral" => Self::Neutral, + "timed_out" => Self::TimedOut, + "action_required" => Self::ActionRequired, + "stale" => Self::Stale, + "startup_failure" => Self::StartupFailure, + other => Self::Unknown(other.to_string()), + } + } + + /// True for conclusions that represent a genuine failing run of an + /// **active** workflow — the only conclusions the sweep treats as + /// actionable. `cancelled`/`skipped`/`neutral`/`action_required`/`stale` + /// are deliberately excluded: they are not failures. + pub fn is_actionable_failure(&self) -> bool { + matches!(self, Self::Failure | Self::TimedOut | Self::StartupFailure) + } + + /// Canonical GitHub string for reporting. + pub fn as_gh_str(&self) -> String { + match self { + Self::Success => "success".to_string(), + Self::Failure => "failure".to_string(), + Self::Cancelled => "cancelled".to_string(), + Self::Skipped => "skipped".to_string(), + Self::Neutral => "neutral".to_string(), + Self::TimedOut => "timed_out".to_string(), + Self::ActionRequired => "action_required".to_string(), + Self::Stale => "stale".to_string(), + Self::StartupFailure => "startup_failure".to_string(), + Self::Unknown(s) => s.clone(), + } + } +} + +/// The latest run of a single workflow on a repo's default branch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkflowRun { + /// Run status, e.g. `completed`, `in_progress`, `queued`. + pub status: String, + /// Conclusion; `None` until the run has completed. + pub conclusion: Option, + /// Triggering event, e.g. `push`, `pull_request`, `schedule`, `issues`. + pub event: String, + /// ISO-8601 creation timestamp, used only to pick the newest run. + pub created_at: String, + /// GitHub run database id, echoed into reports so a human can open the run. + pub database_id: u64, +} + +/// A workflow plus its enablement state and latest default-branch run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WorkflowSnapshot { + pub name: String, + pub state: WorkflowState, + /// `None` when the workflow has never run on the default branch. + pub latest_run: Option, +} + +/// Every workflow of one governed repo, resolved against its default branch. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSnapshot { + pub slug: String, + pub default_branch: String, + pub workflows: Vec, +} + +/// A full sweep of the governed fleet: the sole input to classification. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FleetSnapshot { + pub repos: Vec, +} diff --git a/src/error/display.rs b/src/error/display.rs index 139f7e06..10bcbdd1 100644 --- a/src/error/display.rs +++ b/src/error/display.rs @@ -299,6 +299,9 @@ impl Display for SimardError { "stewardship: orchestrator run summary missing required field '{field}'" ) } + Self::CiHealthGhCommandFailed { reason } => { + write!(f, "ci-health: gh command failed: {reason}") + } Self::MergeAuthorityGhCommandFailed { reason } => { write!(f, "merge-authority: gh command failed: {reason}") } diff --git a/src/error/mod.rs b/src/error/mod.rs index c5b72a2e..7d0d612f 100644 --- a/src/error/mod.rs +++ b/src/error/mod.rs @@ -263,6 +263,11 @@ pub enum SimardError { StewardshipInvalidRunSummary { field: &'static str, }, + /// CI-health sweep: a `gh` subprocess invocation failed (non-zero exit, + /// missing binary, malformed JSON) or a report could not be serialized. + CiHealthGhCommandFailed { + reason: String, + }, /// Merge authority: a `gh pr` subprocess invocation failed. MergeAuthorityGhCommandFailed { reason: String, diff --git a/src/error/tests_variants_extra.rs b/src/error/tests_variants_extra.rs index 65acee40..95655a39 100644 --- a/src/error/tests_variants_extra.rs +++ b/src/error/tests_variants_extra.rs @@ -166,6 +166,19 @@ fn display_stewardship_invalid_run_summary() { assert!(msg.contains("run_id"), "{msg}"); } +// --- Display: CiHealthGhCommandFailed --- + +#[test] +fn display_ci_health_gh_command_failed() { + let err = SimardError::CiHealthGhCommandFailed { + reason: "gh run list exited 1".to_string(), + }; + let msg = err.to_string(); + assert!(msg.contains("ci-health"), "{msg}"); + assert!(msg.contains("gh"), "{msg}"); + assert!(msg.contains("gh run list exited 1"), "{msg}"); +} + // --- Display: MergeAuthorityGhCommandFailed --- #[test] diff --git a/src/lib.rs b/src/lib.rs index 74579e7a..b95d7857 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod base_types; pub mod bootstrap; pub mod build_lock; pub mod cargo_jobs; +pub mod ci_health; pub mod cmd_cleanup; pub mod cmd_ensure_deps; pub mod cmd_install; diff --git a/src/operator_cli/ci_health.rs b/src/operator_cli/ci_health.rs new file mode 100644 index 00000000..ed75fb8c --- /dev/null +++ b/src/operator_cli/ci_health.rs @@ -0,0 +1,101 @@ +//! `simard ci-health` — run the governed-fleet CI-health sweep and print a +//! report. Exit code 0 when the fleet is green (zero actionable failures), +//! non-zero when an active workflow's latest default-branch run failed. +//! +//! By default the sweep reads live GitHub state via `gh`. `--from-json ` +//! classifies an offline snapshot fixture instead (used by tests and for +//! reproducing a captured sweep). +//! +//! See `docs/reference/ci-health-sweep.md`. + +use crate::ci_health::{ + RealGhWorkflowClient, render_human, report_to_json, sweep_fixture, sweep_live, +}; + +pub(super) const CI_HEALTH_HELP: &str = "\ +Simard ci-health subcommand + +Usage: simard ci-health [--json] [--from-json ] + + --json Emit the FleetReport as JSON (default: human table). + --from-json Classify an offline snapshot fixture instead of calling + `gh` (the fixture shape mirrors the live snapshot). + +Sweeps every active default-branch workflow across the amplihack ecosystem +(Simard + governed sibling repos). A workflow is reported as an actionable +failure only when it is ENABLED and its latest default-branch run concluded +failure / timed_out / startup_failure. Disabled workflows and non-failure +conclusions (cancelled, skipped, neutral, action_required, stale) are ignored +with a reason. + +Exit code: 0 when the fleet is green; non-zero when any actionable failure +exists. +"; + +/// Parsed `ci-health` flags. +struct Flags { + json: bool, + from_json: Option, +} + +fn parse_flags(args: impl Iterator) -> Result> { + let mut json = false; + let mut from_json = None; + let mut args = args.peekable(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--json" => json = true, + "--from-json" => { + let path = args + .next() + .ok_or_else(|| "flag `--from-json` requires a path argument".to_string())?; + from_json = Some(path); + } + other => { + if let Some(path) = other.strip_prefix("--from-json=") { + from_json = Some(path.to_string()); + } else { + return Err(format!( + "unexpected argument '{other}' (see `simard ci-health --help`)" + ) + .into()); + } + } + } + } + Ok(Flags { json, from_json }) +} + +/// Dispatch `simard ci-health`. Returns `Ok(())` when the fleet is green and an +/// `Err` (non-zero exit) when any actionable failure exists, matching the +/// `self-health` convention. +pub(super) fn dispatch_ci_health_command( + args: impl Iterator, +) -> Result<(), Box> { + let flags = parse_flags(args)?; + + let report = match &flags.from_json { + Some(path) => { + let bytes = + std::fs::read(path).map_err(|e| format!("failed to read fixture '{path}': {e}"))?; + sweep_fixture(&bytes)? + } + None => sweep_live(&RealGhWorkflowClient::new())?, + }; + + if flags.json { + println!("{}", report_to_json(&report)?); + } else { + print!("{}", render_human(&report)); + } + + if report.green { + Ok(()) + } else { + Err(format!( + "ci-health: {} actionable failure(s) across the governed fleet", + report.actionable_failures.len() + ) + .into()) + } +} diff --git a/src/operator_cli/mod.rs b/src/operator_cli/mod.rs index 8c3e34e1..0a2ba40e 100644 --- a/src/operator_cli/mod.rs +++ b/src/operator_cli/mod.rs @@ -1,4 +1,5 @@ mod args; +mod ci_health; mod curation; mod dashboard; mod decisions; @@ -129,6 +130,7 @@ Product modes: update self-test self-health — post-deploy probes (version/memory/board/brains/quarantine) + ci-health [--json] — sweep active default-branch CI across the governed fleet self-deploy [--check] — close the merged-but-not-running gap (operator-only) safe-update — drain → snapshot → pre-test → swap → exec rollback — restore the latest backup over the install path @@ -287,6 +289,14 @@ where } self_health::dispatch_self_health_command(args) } + "ci-health" => { + let mut args = args.peekable(); + if let Some(help) = check_help_flag(&mut args, ci_health::CI_HEALTH_HELP) { + print!("{help}"); + return Ok(()); + } + ci_health::dispatch_ci_health_command(args) + } "self-deploy" => { let mut args = args.peekable(); if let Some(help) = check_help_flag(&mut args, self_deploy::SELF_DEPLOY_HELP) { diff --git a/tests/gadugi/ci-health-sweep.sh b/tests/gadugi/ci-health-sweep.sh new file mode 100755 index 00000000..25ce12dc --- /dev/null +++ b/tests/gadugi/ci-health-sweep.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Outside-in coverage for `simard ci-health`: the codified governed-fleet +# CI-health sweep. Drives the CLI with committed offline fixtures (no network) +# that encode the real-world signal classes we observed: +# - active workflow, latest run success -> green +# - disabled workflow, stale last-run failure -> ignored (workflow_disabled) +# - active workflow, cancelled last run -> ignored (non-failure) +# - active workflow, in-progress last run -> ignored (in progress) +# - active workflow, latest run failure -> ACTIONABLE failure (red) +# +# Asserts that disabled/cancelled/in-progress signals never fail the fleet, +# that a genuine active-CI failure does, and that the exit code follows the +# verdict. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" + +GREEN_FIXTURE="tests/gadugi/fixtures/ci-health-green.json" +FAILING_FIXTURE="tests/gadugi/fixtures/ci-health-failing.json" + +run_ci_health() { + cargo run --quiet --bin simard -- ci-health "$@" 2>/dev/null +} + +# ── 1. Green fixture: only disabled/cancelled/in-progress non-green signals ── +GREEN_OUT="$(run_ci_health --from-json "$GREEN_FIXTURE")" +printf '%s\n' "$GREEN_OUT" + +printf '%s\n' "$GREEN_OUT" | grep -F "CI-HEALTH: GREEN" >/dev/null +printf '%s\n' "$GREEN_OUT" | grep -F "actionable failures: 0" >/dev/null +# The disabled azlin monitors must be ignored with the disabled reason, not +# counted as failures. +printf '%s\n' "$GREEN_OUT" | grep -F "Code Quality Tracker (workflow_disabled)" >/dev/null +printf '%s\n' "$GREEN_OUT" | grep -F "CI/CD Workflow Health Monitor (workflow_disabled)" >/dev/null +# The cancelled Build Knowledge Pack must be an ignored non-failure. +printf '%s\n' "$GREEN_OUT" | grep -F "Build Knowledge Pack (non_failure_conclusion:cancelled)" >/dev/null +# The in-progress Simard verify run must be ignored, not treated as a failure. +printf '%s\n' "$GREEN_OUT" | grep -F "verify (run_in_progress)" >/dev/null + +# Exit code must be 0 for a green fleet. +if ! run_ci_health --from-json "$GREEN_FIXTURE" >/dev/null; then + echo "FAIL: green fixture returned non-zero exit code" >&2 + exit 1 +fi + +# ── 2. JSON output shape ──────────────────────────────────────────────────── +GREEN_JSON="$(run_ci_health --json --from-json "$GREEN_FIXTURE")" +printf '%s\n' "$GREEN_JSON" | grep -F '"green": true' >/dev/null +printf '%s\n' "$GREEN_JSON" | grep -F '"actionable_failures": []' >/dev/null + +# ── 3. Failing fixture: one ACTIVE workflow failed ────────────────────────── +set +e +FAIL_OUT="$(run_ci_health --from-json "$FAILING_FIXTURE")" +FAIL_CODE=$? +set -e +printf '%s\n' "$FAIL_OUT" + +printf '%s\n' "$FAIL_OUT" | grep -F "CI-HEALTH: FAILING" >/dev/null +# The active CI failure is flagged with its repo and conclusion... +printf '%s\n' "$FAIL_OUT" | grep -F "rysweet/azlin" >/dev/null +printf '%s\n' "$FAIL_OUT" | grep -F -- "-> failure" >/dev/null +# ...while the disabled workflow in the SAME repo stays ignored. +printf '%s\n' "$FAIL_OUT" | grep -F "Code Quality Tracker (workflow_disabled)" >/dev/null + +if [ "$FAIL_CODE" -eq 0 ]; then + echo "FAIL: failing fixture returned zero exit code" >&2 + exit 1 +fi + +echo "ci-health-sweep: PASS" diff --git a/tests/gadugi/ci-health-sweep.yaml b/tests/gadugi/ci-health-sweep.yaml new file mode 100644 index 00000000..89149aa4 --- /dev/null +++ b/tests/gadugi/ci-health-sweep.yaml @@ -0,0 +1,52 @@ +name: "CI-health governed-fleet sweep" +description: "End-to-end operator coverage for `simard ci-health`: the codified governed-fleet CI-health sweep. Verifies the sweep classifies workflow signals precisely — disabled workflows' stale failures, cancelled runs, and in-progress runs are ignored (never fail the fleet), while a genuine active-workflow failure IS flagged as actionable and drives a non-zero exit. Uses committed offline fixtures (no network) so the verdict is deterministic. Motivated by prior hand-rolled `gh run list` sweeps whose 'every workflow is success' claims were imprecise/reviewer-rejected because they could not distinguish a disabled-and-failing workflow from an active-and-failing one." +version: "1.0.0" + +config: + timeout: 300000 + retries: 1 + parallel: false + +environment: + requires: [] + +agents: + - name: "cli-agent" + type: "cli" + config: + workingDirectory: "." + defaultTimeout: 300000 + executionMode: "spawn" + captureOutput: true + ioConfig: + encoding: "utf8" + handleInteractivePrompts: false + logConfig: + logCommands: true + logOutput: true + logLevel: "info" + +steps: + - name: "ci-health classifies disabled/cancelled/in-progress as ignored and active failure as actionable" + agent: "cli-agent" + action: "execute_command" + params: + command: "tests/gadugi/ci-health-sweep.sh" + timeout: 300000 + +assertions: [] + +cleanup: + - name: "CLI agent cleanup" + agent: "cli-agent" + action: "cleanup" + +metadata: + tags: + - "cli" + - "ci-health" + - "stewardship" + - "outside-in" + priority: "high" + author: "simard-engineer" + test_type: "outside-in" diff --git a/tests/gadugi/fixtures/ci-health-failing.json b/tests/gadugi/fixtures/ci-health-failing.json new file mode 100644 index 00000000..874c1eaf --- /dev/null +++ b/tests/gadugi/fixtures/ci-health-failing.json @@ -0,0 +1,12 @@ +{ + "repos": [ + { + "slug": "rysweet/azlin", + "default_branch": "main", + "workflows": [ + {"name": "CI", "state": "active", "latest_run": {"status": "completed", "conclusion": "failure", "event": "push", "created_at": "2026-07-06T00:00:00Z", "database_id": 28718588578}}, + {"name": "Code Quality Tracker", "state": "disabled_manually", "latest_run": {"status": "completed", "conclusion": "failure", "event": "schedule", "created_at": "2026-06-15T11:08:17Z", "database_id": 27542084782}} + ] + } + ] +} diff --git a/tests/gadugi/fixtures/ci-health-green.json b/tests/gadugi/fixtures/ci-health-green.json new file mode 100644 index 00000000..5c4bb2d0 --- /dev/null +++ b/tests/gadugi/fixtures/ci-health-green.json @@ -0,0 +1,30 @@ +{ + "repos": [ + { + "slug": "rysweet/azlin", + "default_branch": "main", + "workflows": [ + {"name": "CI", "state": "active", "latest_run": {"status": "completed", "conclusion": "success", "event": "push", "created_at": "2026-07-04T20:24:06Z", "database_id": 28718588578}}, + {"name": "Rust CI", "state": "active", "latest_run": {"status": "completed", "conclusion": "success", "event": "push", "created_at": "2026-07-04T20:24:06Z", "database_id": 28718588579}}, + {"name": "Code Quality Tracker", "state": "disabled_manually", "latest_run": {"status": "completed", "conclusion": "failure", "event": "schedule", "created_at": "2026-06-15T11:08:17Z", "database_id": 27542084782}}, + {"name": "CI/CD Workflow Health Monitor", "state": "disabled_manually", "latest_run": {"status": "completed", "conclusion": "failure", "event": "schedule", "created_at": "2026-06-15T10:04:55Z", "database_id": 27538870063}} + ] + }, + { + "slug": "rysweet/agent-kgpacks", + "default_branch": "main", + "workflows": [ + {"name": "CI", "state": "active", "latest_run": {"status": "completed", "conclusion": "success", "event": "push", "created_at": "2026-06-30T00:10:53Z", "database_id": 28411188790}}, + {"name": "Build Knowledge Pack", "state": "active", "latest_run": {"status": "completed", "conclusion": "cancelled", "event": "issues", "created_at": "2026-03-06T21:56:54Z", "database_id": 22783611001}} + ] + }, + { + "slug": "rysweet/Simard", + "default_branch": "main", + "workflows": [ + {"name": "verify", "state": "active", "latest_run": {"status": "in_progress", "conclusion": null, "event": "push", "created_at": "2026-07-06T20:06:02Z", "database_id": 28819793406}}, + {"name": "docs", "state": "active", "latest_run": {"status": "completed", "conclusion": "success", "event": "push", "created_at": "2026-07-06T20:06:02Z", "database_id": 28819793431}} + ] + } + ] +}