From 5df02b7b4af603bfc0a4f2725b6465412399c546 Mon Sep 17 00:00:00 2001 From: Nikola Katsarov Date: Sat, 18 Jul 2026 10:18:03 +0300 Subject: [PATCH 1/2] =?UTF-8?q?refactor(cli):=20remove=20HTTP=20serving=20?= =?UTF-8?q?layer=20=E2=80=94=20local=20engine=20+=20CLI=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes everything gated by the `serve` cargo feature of boruna-cli: the coordinator HTTP server, distributed worker, workflow dashboard, evidence web viewer, and approval console. - Delete coordinator.rs, dashboard.rs, evidence_serve.rs, serve.rs, worker.rs and the serve-only integration tests (cli_coordinator_mtls, cli_coordinator_worker, cli_dashboard). - Strip all `#[cfg(feature = "serve")]` sites from main.rs: the mod decls, the Dashboard/Coordinator/Worker command variants + their enums, the FrameworkCommand::Serve and EvidenceCommand::Serve variants, dispatch arms, and run_coordinator/run_worker_cmd/ run_dashboard. The `workflow run/approve/reject/trigger --coordinator` paths now return a "no longer supported" error. - Drop the `serve` feature and its exclusive deps (axum, reqwest, uuid, rustls, rustls-pemfile, tokio-rustls, hyper, hyper-util, tower, tower-service) plus the serve-only dev-deps. `tokio` kept (telemetry). - Remove `--features serve` CI steps; keep default/http/clippy/fmt. - README: drop distributed-execution bullet + coord doc links. CHANGELOG: add Unreleased > Removed entry. Gates green: build, test (1388 passed / 0 failed), clippy -D warnings, fmt --check, plus boruna-vm/http build + test (243 passed). Claude-Session: https://claude.ai/code/session_01BEk2VHyMr1MrSPitnhox5o --- .github/workflows/ci.yml | 9 - CHANGELOG.md | 6 + Cargo.lock | 178 - README.md | 8 +- crates/llmvm-cli/Cargo.toml | 44 +- crates/llmvm-cli/src/coordinator.rs | 4617 ----------------- crates/llmvm-cli/src/dashboard.rs | 1174 ----- crates/llmvm-cli/src/doctor.rs | 3 - crates/llmvm-cli/src/evidence_serve.rs | 711 --- crates/llmvm-cli/src/main.rs | 530 +- crates/llmvm-cli/src/serve.rs | 506 -- crates/llmvm-cli/src/worker.rs | 722 --- .../llmvm-cli/tests/cli_coordinator_mtls.rs | 440 -- .../llmvm-cli/tests/cli_coordinator_worker.rs | 1718 ------ crates/llmvm-cli/tests/cli_dashboard.rs | 297 -- 15 files changed, 32 insertions(+), 10931 deletions(-) delete mode 100644 crates/llmvm-cli/src/coordinator.rs delete mode 100644 crates/llmvm-cli/src/dashboard.rs delete mode 100644 crates/llmvm-cli/src/evidence_serve.rs delete mode 100644 crates/llmvm-cli/src/serve.rs delete mode 100644 crates/llmvm-cli/src/worker.rs delete mode 100644 crates/llmvm-cli/tests/cli_coordinator_mtls.rs delete mode 100644 crates/llmvm-cli/tests/cli_coordinator_worker.rs delete mode 100644 crates/llmvm-cli/tests/cli_dashboard.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31fa3e8..a732284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,12 +53,6 @@ jobs: - name: Run tests with http feature run: cargo test -p boruna-vm --features http - - name: Build with serve feature - run: cargo build -p boruna-cli --features serve - - - name: Run tests with serve feature - run: cargo test -p boruna-cli --features serve - - name: Validate example workflows run: | for dir in examples/workflows/*/; do @@ -160,9 +154,6 @@ jobs: - name: Run Clippy with http feature run: cargo clippy --workspace --features boruna-vm/http --all-targets -- -D warnings - - name: Run Clippy with serve feature - run: cargo clippy -p boruna-cli --features serve --all-targets -- -D warnings - fmt: name: Format runs-on: self-hosted diff --git a/CHANGELOG.md b/CHANGELOG.md index c36b2cb..8aa5fdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ Versioning follows [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Removed + +- Removed the HTTP serving layer — coordinator, distributed workers, workflow + dashboard, evidence web viewer, and approval console. Boruna is now + local-engine + CLI only. + ## [2.0.0] — 2026-07-17 First major release. A security-hardening + language-completeness sprint that diff --git a/Cargo.lock b/Cargo.lock index ed6736a..c8fc5ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,80 +171,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "aws-lc-rs" -version = "1.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.40.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", - "bytes", - "form_urlencoded", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower 0.5.3", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "base64" version = "0.21.7" @@ -388,7 +314,6 @@ dependencies = [ name = "boruna-cli" version = "2.0.0" dependencies = [ - "axum", "boruna-bytecode", "boruna-compiler", "boruna-framework", @@ -396,22 +321,12 @@ dependencies = [ "boruna-tooling", "boruna-vm", "clap", - "hyper", - "hyper-util", "notify", - "rcgen", - "reqwest", - "rustls", - "rustls-pemfile", "serde", "serde_json", "sha2", "tempfile", "tokio", - "tokio-rustls", - "tower 0.5.3", - "tower-service", - "uuid", ] [[package]] @@ -596,8 +511,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -704,15 +617,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -978,12 +882,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.20" @@ -1160,12 +1058,6 @@ dependencies = [ "num", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1837,16 +1729,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" version = "0.3.95" @@ -2025,12 +1907,6 @@ dependencies = [ "url", ] -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - [[package]] name = "md-5" version = "0.10.6" @@ -2047,12 +1923,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2399,16 +2269,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5a797f0e07bdf071d15742978fc3128ec6c22891c31a3a931513263904c982a" -[[package]] -name = "pem" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" -dependencies = [ - "base64 0.22.1", - "serde_core", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2717,19 +2577,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rcgen" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" -dependencies = [ - "pem", - "ring", - "rustls-pki-types", - "time", - "yasna", -] - [[package]] name = "redox_syscall" version = "0.3.5" @@ -2961,7 +2808,6 @@ version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ - "aws-lc-rs", "log", "once_cell", "ring", @@ -3008,7 +2854,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3171,17 +3016,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", -] - [[package]] name = "serde_repr" version = "0.1.20" @@ -3709,7 +3543,6 @@ dependencies = [ "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -3782,7 +3615,6 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3945,7 +3777,6 @@ version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ - "getrandom 0.4.2", "js-sys", "wasm-bindgen", ] @@ -4516,15 +4347,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "yasna" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" -dependencies = [ - "time", -] - [[package]] name = "yoke" version = "0.8.2" diff --git a/README.md b/README.md index 1442ee8..298c1ce 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ This makes Boruna suited for teams building AI workflows that touch regulated da - **Capability enforcement** — every side effect (LLM calls, HTTP, database, filesystem) is declared and policy-gated at the VM level - **Evidence bundles** — hash-chained tamper-evident logs, written automatically with `--record`. Optional **AES-256-GCM envelope encryption** for compliance-sensitive deployments. `evidence inspect` shows step output content for plaintext bundles. - **Deterministic replay** — re-execute any recorded workflow with identical outputs, verified by the VM -- **Distributed execution** — coord+workers HTTP cluster with active-active **HA**, worker URL failover, capability-tagged placement, and optional **mTLS** with per-worker client certs - **Approval gates** — pause workflow execution for human review or external triggers before continuing - **Diagnostics, auto-repair, and migration** — `boruna lang check`, `boruna lang repair`, `boruna migrate` for `.ax` files and bundle/workflow upgrades - **`boruna new`** — interactive scaffold for new workflows from templates @@ -129,7 +128,7 @@ Boruna is a Rust workspace with 10 production crates plus a `benches/` member: | `boruna-tooling` | Diagnostics, repair, trace-to-tests, templates | | `boruna-pkg` | Package registry, resolver, lockfiles | -1175+ tests across 11 workspace members. `cargo test --workspace --features boruna-cli/serve` — all pass. +1175+ tests across 11 workspace members. `cargo test --workspace` — all pass. ## Documentation @@ -140,9 +139,6 @@ Boruna is a Rust workspace with 10 production crates plus a `benches/` member: | [Concepts: Capabilities](docs/concepts/capabilities.md) | Side effect declaration and policy gating | | [Concepts: Evidence Bundles](docs/concepts/evidence-bundles.md) | Hash-chained audit logs and replay | | [Guide: First Workflow](docs/guides/first-workflow.md) | Build a workflow from scratch | -| [Guide: Coord HA](docs/guides/coord-ha.md) | Multi-coord deployment topologies | -| [Guide: Coord mTLS](docs/guides/coord-mtls.md) | X.509 client certs + cert generation | -| [Guide: Worker Capability Tagging](docs/guides/worker-capability-tagging.md) | Heterogeneous fleet placement | | [Guide: Migration](docs/guides/migration.md) | Upgrade legacy bundles and workflow files | | [Spec: `.ax` Language 1.0](docs/spec/ax-language-1.0.md) | Formal language specification | | [Spec: Workflow DAG 1.0](docs/spec/workflow-dag-1.0.md) | `workflow.json` schema | @@ -158,7 +154,7 @@ Boruna is a Rust workspace with 10 production crates plus a `benches/` member: ## Status -Boruna is at **v2.0.0** — the first major release. 2.0 is a security-hardening and language-completeness milestone that remediates a whole-codebase research audit: SSRF/XSS fixes, coordinator claim-ownership and approval-gate enforcement, tamper-evident evidence bundles (external anchor + ed25519 signing), and real language semantics (enum construction with per-variant match tags, higher-order calls, `for` loops, arity checking, and warn-only type-consistency diagnostics). It ships **deliberate breaking changes** — integer overflow is now a runtime error, and several coordinator/framework defaults fail closed — so review the 2.0.0 entry in [`CHANGELOG.md`](CHANGELOG.md), each of which has a documented override or migration. The core execution engine, distributed-execution stack, evidence bundles, and four formal versioned specifications (`.ax` language, bytecode, workflow DAG, evidence bundle) remain feature-complete; the 1.x LTS line continues per [`docs/lts.md`](docs/lts.md). +Boruna is at **v2.0.0** — the first major release. 2.0 is a security-hardening and language-completeness milestone that remediates a whole-codebase research audit: SSRF/XSS fixes, coordinator claim-ownership and approval-gate enforcement, tamper-evident evidence bundles (external anchor + ed25519 signing), and real language semantics (enum construction with per-variant match tags, higher-order calls, `for` loops, arity checking, and warn-only type-consistency diagnostics). It ships **deliberate breaking changes** — integer overflow is now a runtime error, and several coordinator/framework defaults fail closed — so review the 2.0.0 entry in [`CHANGELOG.md`](CHANGELOG.md), each of which has a documented override or migration. The core execution engine, evidence bundles, and four formal versioned specifications (`.ax` language, bytecode, workflow DAG, evidence bundle) remain feature-complete; the 1.x LTS line continues per [`docs/lts.md`](docs/lts.md). The project is suited for evaluation, internal tooling, and audit-sensitive AI pipelines. **Operator action**: validate the [`docs/PERFORMANCE.md`](docs/PERFORMANCE.md) budget against your workload, and review [`docs/limitations.md`](docs/limitations.md) for known constraints. External security audit booking is the Q4 2026 commitment in `lts.md`. diff --git a/crates/llmvm-cli/Cargo.toml b/crates/llmvm-cli/Cargo.toml index 30ad229..05f3bb4 100644 --- a/crates/llmvm-cli/Cargo.toml +++ b/crates/llmvm-cli/Cargo.toml @@ -10,19 +10,6 @@ path = "src/main.rs" [features] default = ["persist-sqlite"] -serve = [ - "dep:axum", - "dep:tokio", - "dep:reqwest", - "dep:uuid", - "dep:rustls", - "dep:rustls-pemfile", - "dep:tokio-rustls", - "dep:hyper", - "dep:hyper-util", - "dep:tower", - "dep:tower-service", -] http = ["boruna-vm/http"] telemetry = ["boruna-vm/telemetry", "dep:tokio"] # Forwards to boruna-orchestrator's persist-sqlite feature so the CLI's @@ -55,37 +42,10 @@ clap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tempfile = "3" +# `tokio` is required by the `telemetry` feature (OTel batch exporter +# runtime). Optional so non-telemetry builds don't pull it in. tokio = { workspace = true, optional = true } -axum = { workspace = true, optional = true } -# `reqwest` for the worker's HTTP client. `default-features = false` -# drops `openssl` so we don't pull a system dep; `rustls-tls` gives -# us TLS via a Rust impl, `json` is the body shape. -reqwest = { version = "0.12", optional = true, default-features = false, features = ["json", "rustls-tls"] } -uuid = { version = "1", optional = true, features = ["v4"] } sha2 = "0.10" -# mTLS stack (sprint W6-A). All optional; pulled in by `serve`. -rustls = { workspace = true, optional = true } -rustls-pemfile = { workspace = true, optional = true } -tokio-rustls = { workspace = true, optional = true } -# Direct hyper/hyper-util/tower deps for the mTLS connection -# loop (W6-A). These are already in the dep graph via axum so -# pulling them in directly does not increase the binary size. -hyper = { version = "1", optional = true, features = ["server", "http1", "http2"] } -hyper-util = { version = "0.1", optional = true, features = ["server-auto", "tokio"] } -tower = { version = "0.5", optional = true } -tower-service = { version = "0.3", optional = true } # `boruna run --watch` (post1-T-1.4) — filesystem-watch loop that # re-executes a `.ax` file on change. notify = { workspace = true } - -[dev-dependencies] -tower = { version = "0.5", features = ["util"] } -rcgen = { workspace = true } -# Dev-only: blocking reqwest for the mTLS integration test. -# The runtime worker uses async reqwest under tokio. -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "blocking"] } -# Dev-only: rustls + rustls-pemfile so tests can synthesize a -# bare TLS handshake (no client cert) and assert the server -# rejects it before any HTTP is exchanged. -rustls = { workspace = true } -rustls-pemfile = { workspace = true } diff --git a/crates/llmvm-cli/src/coordinator.rs b/crates/llmvm-cli/src/coordinator.rs deleted file mode 100644 index 01c7601..0000000 --- a/crates/llmvm-cli/src/coordinator.rs +++ /dev/null @@ -1,4617 +0,0 @@ -//! Distributed-execution coordinator HTTP server (sprint -//! `0.5-S2b`). Wraps the persistence-layer claim/lease state -//! machine from `0.5-S2a` in an HTTP protocol so remote workers -//! can claim work over the wire. -//! -//! See `docs/design-coordinator-worker-http.md` and -//! `docs/architecture-coordinator-worker-http.md` for the -//! design rationale and wire format. -//! -//! ## Security posture -//! -//! - Loopback (`127.0.0.1`) by default. `--bind 0.0.0.0` emits a -//! loud stderr warning and includes `bind_warning` in any -//! future dashboard banner. -//! - **No authentication.** Operators exposing the coordinator -//! to a network MUST front it with an auth-enforcing reverse -//! proxy. Mutations are possible — this is a stronger -//! warning than the dashboard's read-only one. -//! - Output payload size is capped at 8 MiB per ADR 002. -//! - Workers must match the coordinator's `capability_set_hash` -//! (atomic-upgrade rule from ADR 002). -//! -//! ## Protocol -//! -//! Every response carries `protocol_version: 1`. Failure -//! responses also carry `error_kind: ""` from -//! the locked `coord.*` taxonomy. - -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::net::IpAddr; -use std::path::PathBuf; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use axum::extract::{DefaultBodyLimit, Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; -use axum::middleware::{self, Next}; -use axum::response::{IntoResponse, Json, Response}; -use axum::routing::{get, post}; -use axum::Router; -// mTLS surface (sprint W6-A). All TLS types live behind an Option -// in the coordinator config — when no mTLS flags are set the -// coordinator's listener path is identical to the pre-W6 plain TCP -// behavior. -use boruna_bytecode::compute_capability_set_hash; -use boruna_orchestrator::persistence::{ - BlobStoreError, ClaimOutcome, ExtendOutcome, RunCheckpointStore, RunStatus, StepStatus, - TerminalOutcome, -}; -use rustls::pki_types::CertificateDer; -use rustls::server::WebPkiClientVerifier; -use rustls::RootCertStore; -use serde::{Deserialize, Serialize}; - -const PROTOCOL_VERSION: u32 = 1; -const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; - -#[derive(Clone)] -pub struct CoordinatorState { - store: Arc>, - workers: Arc>>, - /// Reserved for 0.5-S2c: maps run_id → workflow_dir on disk - /// so the coordinator can resolve `.ax` source paths from the - /// run metadata. Today the MVP gets sources inline via - /// `metadata_json.step_sources`. - #[allow(dead_code)] - workflow_dirs: Arc>>, - capability_set_hash: String, - config: CoordinatorConfig, - /// Process start time, captured once at `run_serve` entry. Used - /// by the `/api/health` endpoint to report uptime so operators - /// can detect coord restarts during a multi-coord rollout - /// (sprint W2). - start_time_ms: i64, -} - -#[derive(Clone)] -struct WorkerSession { - session_token: String, - last_heartbeat_ms: i64, - /// Captured at registration; reserved for future - /// rolling-upgrade detection (per ADR 002 open question 5). - #[allow(dead_code)] - capability_set_hash: String, - /// Sprint `W3-A` — placement filter ONLY (operational state, - /// per project §15). When `Some(map)`, this worker only - /// receives steps whose policy-required capabilities each - /// resolve to a `(name, version)` the worker has advertised. - /// When `None`, the worker is treated as a full-fleet worker - /// (matches every step). The capability gateway in `boruna-vm` - /// remains the security boundary; a worker that lies about its - /// advertised set is still denied by policy at execution time. - /// - /// Post-1.0 (T-1.3): the value is a map `name -> version`. - /// Coord normalizes legacy bare-string entries to the coord's - /// current `Capability::version()` for that name at parse time. - advertised_capabilities: Option>, -} - -#[derive(Clone)] -pub struct CoordinatorConfig { - pub max_lease_ttl_ms: u64, - pub poll_timeout_ms: u64, - /// Forwarded to the merged dashboard's HTML banner so the - /// red WARNING block appears on coordinator-served pages - /// when bound to a non-loopback address (sprint 0.5-S2d). - pub bind_warning: Option, - /// Shared-secret bearer token for HTTP authentication - /// (sprint `0.5-S3`). When `Some`, every coord HTTP route - /// requires `Authorization: Bearer ` header; mismatched - /// or missing headers return `401 + coord.unauthorized`. When - /// `None`, no auth is enforced (the pre-0.5-S3 behavior is - /// preserved for backwards-compatibility on loopback-only - /// deployments). - /// - /// Operators generate a secret via `openssl rand -hex 32` and - /// pass it via `--shared-secret ` flag or - /// `BORUNA_COORD_SECRET` env var. The same value MUST be set - /// on every worker via the analogous flag or env var. - pub shared_secret: Option, - /// Whether mTLS is enabled on this coord (sprint `W6-A`). - /// When `true`, the `auth_middleware` requires every request - /// to carry a [`ClientIdentity`] extension (extracted from - /// the TLS handshake's client cert). Defense-in-depth: if a - /// request reaches the middleware without an identity (e.g. - /// because of a bug in the TLS plumbing) the request is - /// rejected with 401 `coord.unauthorized`. - pub mtls_required: bool, -} - -/// Minimum sweep interval. Lower values would cause the -/// background task to busy-loop. 100 ms is fast enough for -/// integration tests; production operators pick something -/// larger via `--sweep-interval-ms`. -const MIN_SWEEP_INTERVAL_MS: u64 = 100; - -/// File-path bundle for the coord's mTLS server config (sprint -/// `W6-A`). All three paths are required together — passing -/// fewer than three is a typed startup error so misconfigurations -/// surface at parse time rather than as a silent fallback to -/// plaintext. -#[derive(Debug, Clone)] -pub struct ServerTlsPaths { - pub cert: PathBuf, - pub key: PathBuf, - pub client_ca: PathBuf, -} - -impl ServerTlsPaths { - /// Validate the three optional paths and produce either - /// `None` (no TLS — pre-W6 behavior) or a fully-populated - /// triple. Mixing-and-matching (e.g. `--tls-cert` without - /// `--tls-key`) is a startup error per project §1. - pub fn from_optional( - cert: Option, - key: Option, - client_ca: Option, - ) -> Result, Box> { - match (cert, key, client_ca) { - (None, None, None) => Ok(None), - (Some(cert), Some(key), Some(client_ca)) => Ok(Some(Self { - cert, - key, - client_ca, - })), - _ => Err("--tls-cert, --tls-key, --tls-client-ca must all be provided together".into()), - } - } -} - -/// Compiled rustls server config + the path-shaped originals -/// (kept so the run_serve path can log them at startup). Wrapped -/// in `Arc` so [`CoordinatorState`] stays cheap-clone. -#[derive(Clone)] -struct CompiledServerTls { - config: Arc, -} - -/// Per-connection identity extracted from a presented client -/// certificate. The CN drives worker identity per the W6-A -/// design: handlers compare incoming `worker_id` body fields -/// (case-insensitively) against this CN; mismatch returns -/// `coord.identity_mismatch`. -#[derive(Clone, Debug)] -pub struct ClientIdentity { - pub common_name: String, -} - -#[tokio::main] -#[allow(clippy::too_many_arguments)] -pub async fn run_serve( - data_dir: PathBuf, - port: u16, - bind: IpAddr, - max_lease_ttl_ms: u64, - poll_timeout_ms: u64, - sweep_interval_ms: u64, - shared_secret: Option, - tls_paths: Option, -) -> Result<(), Box> { - let db_path = data_dir.join("runs.db"); - if !db_path.exists() { - return Err(format!( - "no runs.db at {} — run a workflow first or pass a different --data-dir", - db_path.display() - ) - .into()); - } - - let store = RunCheckpointStore::open(&db_path) - .map_err(|e| format!("failed to open {}: {e}", db_path.display()))?; - - // On startup, eagerly sweep expired leases. The persistence - // layer's `expire_leases_and_requeue(threshold)` only voids - // leases whose `lease_expires_at < threshold` (CAS update on - // a strict comparison) — so this is HA-safe under concurrent - // coords: peer coords with healthy in-flight leases (those - // with `lease_expires_at >= now_ms`) are unaffected. - // - // Note (sprint W2 audit): an earlier comment claimed this - // sweep voids "any row in Running status." That was misleading - // — the SQL filter on `lease_expires_at < ?1` always preserved - // healthy leases. The ADR 002 phrase "coordinator restart = - // all leases void" was the design intent before the lease - // mechanism stabilized; the actual implementation is the - // safer threshold-based variant we keep here. - let now_ms = now_unix_ms(); - let n = store - .expire_leases_and_requeue(now_ms + 1) - .map_err(|e| format!("startup lease sweep failed: {e}"))?; - if n > 0 { - eprintln!("coordinator startup: requeued {n} expired-lease step(s)"); - } - let start_time_ms = now_ms; - - let bind_warning = if bind.is_loopback() { - None - } else { - let msg = format!("{bind}:{port}"); - eprintln!( - "[WARNING] coordinator bound to non-loopback {msg}; \ - anyone with network access can SUBMIT and CONTROL distributed work; \ - the coordinator ships no auth — front it with an auth-enforcing reverse proxy" - ); - Some(msg) - }; - - let capability_set_hash = compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - - // Compile TLS config first so we know whether mTLS is on - // before we record the auth state. - let compiled_tls = match tls_paths.as_ref() { - Some(paths) => Some(build_server_tls(paths)?), - None => None, - }; - let mtls_required = compiled_tls.is_some(); - - let auth_state = match (shared_secret.as_deref(), mtls_required) { - (Some(_), true) => "enabled (mTLS + shared-secret bearer)", - (Some(_), false) => "enabled (shared-secret bearer)", - (None, true) => "enabled (mTLS only)", - (None, false) if bind.is_loopback() => "disabled (loopback bind only)", - (None, false) => "DISABLED (non-loopback bind without --shared-secret or mTLS)", - }; - eprintln!(" auth: {auth_state}"); - if shared_secret.is_none() && !mtls_required && !bind.is_loopback() { - // Fail CLOSED: a non-loopback bind with no auth exposes SUBMIT/APPROVE/work - // control to any network peer. Previously this only warned and served - // anyway; now it refuses to start unless the operator explicitly - // acknowledges the risk via BORUNA_COORD_ALLOW_INSECURE=1 (intended for a - // trusted reverse-proxy front-end that supplies auth). - let allow_insecure = std::env::var("BORUNA_COORD_ALLOW_INSECURE") - .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) - .unwrap_or(false); - if allow_insecure { - eprintln!( - "[WARNING] coordinator bound to a non-loopback address with NO auth \ - (--shared-secret / mTLS) — running anyway because \ - BORUNA_COORD_ALLOW_INSECURE is set. Ensure a trusted reverse proxy \ - supplies authentication; otherwise any network peer can submit, \ - approve, and control distributed work." - ); - } else { - return Err(format!( - "refusing to start: coordinator is bound to a non-loopback address ({bind}) \ - with NO --shared-secret (or BORUNA_COORD_SECRET) and NO mTLS \ - (--tls-cert/--tls-key/--tls-client-ca). Anyone with network access could \ - submit, approve, and control distributed work. Enable auth, bind to loopback, \ - or set BORUNA_COORD_ALLOW_INSECURE=1 to override (only behind a trusted \ - auth-terminating proxy)." - ) - .into()); - } - } - - let state = CoordinatorState { - store: Arc::new(Mutex::new(store)), - workers: Arc::new(Mutex::new(HashMap::new())), - workflow_dirs: Arc::new(Mutex::new(HashMap::new())), - capability_set_hash, - config: CoordinatorConfig { - max_lease_ttl_ms, - poll_timeout_ms, - bind_warning, - shared_secret, - mtls_required, - }, - start_time_ms, - }; - - // Background lease-expiry sweep (sprint 0.5-S2c). Wakes - // up every `sweep_interval_ms`, calls - // `expire_leases_and_requeue`. Logs only when a non-zero - // number of leases were requeued. - // - // Without this loop, the coordinator's startup sweep is - // the ONLY recovery from a worker crash — operators - // would have to restart the coordinator process to - // unstick a stranded step. - let effective_sweep_ms = sweep_interval_ms.max(MIN_SWEEP_INTERVAL_MS); - if sweep_interval_ms < MIN_SWEEP_INTERVAL_MS { - eprintln!( - "[WARNING] --sweep-interval-ms {sweep_interval_ms} below minimum \ - {MIN_SWEEP_INTERVAL_MS}; using {effective_sweep_ms} ms" - ); - } - let sweep_state = state.clone(); - let sweep_task = tokio::spawn(background_sweep_loop(sweep_state, effective_sweep_ms)); - - let app = build_router(state); - - let addr = std::net::SocketAddr::new(bind, port); - let scheme = if compiled_tls.is_some() { - "https" - } else { - "http" - }; - eprintln!("coordinator serving on {scheme}://{addr}"); - eprintln!(" data-dir: {}", data_dir.display()); - eprintln!(" max_lease_ttl_ms: {max_lease_ttl_ms}"); - eprintln!(" poll_timeout_ms: {poll_timeout_ms}"); - eprintln!(" sweep_interval_ms: {effective_sweep_ms}"); - - let listener = tokio::net::TcpListener::bind(addr).await?; - let result = match compiled_tls { - Some(tls) => serve_with_tls(listener, app, tls).await, - None => axum::serve(listener, app).await, - }; - sweep_task.abort(); - result?; - Ok(()) -} - -/// Build a rustls `ServerConfig` that REQUIRES client cert -/// authentication. Loads cert + key for the server identity and -/// the client CA as a webpki trust root for verifying connecting -/// workers' certs. -fn build_server_tls( - paths: &ServerTlsPaths, -) -> Result> { - install_default_crypto_provider()?; - - let cert_chain = load_cert_chain(&paths.cert)?; - let key = load_private_key(&paths.key)?; - let client_ca = load_cert_chain(&paths.client_ca)?; - - let mut roots = RootCertStore::empty(); - for cert in client_ca { - roots - .add(cert) - .map_err(|e| format!("invalid client CA cert: {e}"))?; - } - - let verifier = WebPkiClientVerifier::builder(Arc::new(roots)) - .build() - .map_err(|e| format!("build client cert verifier: {e}"))?; - - let server_config = rustls::ServerConfig::builder() - .with_client_cert_verifier(verifier) - .with_single_cert(cert_chain, key) - .map_err(|e| format!("rustls ServerConfig: {e}"))?; - - Ok(CompiledServerTls { - config: Arc::new(server_config), - }) -} - -/// Install the rustls default crypto provider exactly once. Calls -/// after the first successfully-installed provider are idempotent; -/// rustls returns an error on second-call so we swallow it. -fn install_default_crypto_provider() -> Result<(), Box> { - // The provider may already be installed by another part of the - // process (e.g. reqwest in the same binary). `install_default` - // returns Err if a provider is already present — that's fine. - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - Ok(()) -} - -fn load_cert_chain( - path: &std::path::Path, -) -> Result>, Box> { - let pem = - std::fs::read(path).map_err(|e| format!("read certificate {}: {e}", path.display()))?; - let mut reader = std::io::Cursor::new(pem); - let chain: Result, _> = rustls_pemfile::certs(&mut reader).collect(); - let chain = chain.map_err(|e| format!("parse certificate {}: {e}", path.display()))?; - if chain.is_empty() { - return Err(format!("no PEM certificates found in {}", path.display()).into()); - } - Ok(chain) -} - -fn load_private_key( - path: &std::path::Path, -) -> Result, Box> { - let pem = std::fs::read(path).map_err(|e| format!("read key {}: {e}", path.display()))?; - let mut reader = std::io::Cursor::new(pem); - let key = rustls_pemfile::private_key(&mut reader) - .map_err(|e| format!("parse key {}: {e}", path.display()))? - .ok_or_else(|| format!("no private key found in {}", path.display()))?; - Ok(key) -} - -/// Per-connection TLS accept loop. Wraps each accepted TCP stream -/// in a `tokio_rustls::TlsAcceptor`, extracts the client cert -/// subject CN from the completed handshake, and stuffs a -/// `ClientIdentity` into the request extensions so handlers and -/// the auth middleware can see it. -/// -/// Failed handshakes (no cert, untrusted cert, bad cipher) are -/// logged at the connection layer and the connection is dropped — -/// no HTTP response is produced (the client never made it past -/// the TLS layer). Successful handshakes hand off to axum's -/// hyper service for normal HTTP processing. -async fn serve_with_tls( - listener: tokio::net::TcpListener, - router: Router, - tls: CompiledServerTls, -) -> std::io::Result<()> { - use hyper_util::rt::{TokioExecutor, TokioIo}; - use hyper_util::server::conn::auto::Builder; - use hyper_util::service::TowerToHyperService; - use tokio_rustls::TlsAcceptor; - use tower::ServiceBuilder; - use tower_service::Service; - - let acceptor = TlsAcceptor::from(tls.config); - let make_service = router.into_make_service(); - loop { - let (tcp, peer) = match listener.accept().await { - Ok(pair) => pair, - Err(e) => { - eprintln!("coordinator TLS accept: {e}"); - continue; - } - }; - let acceptor = acceptor.clone(); - let mut make_service = make_service.clone(); - tokio::spawn(async move { - let tls_stream = match acceptor.accept(tcp).await { - Ok(s) => s, - Err(e) => { - // No client cert / untrusted CA / cipher - // mismatch all surface here. Per W6-A this is - // expected for adversarial probes; log at - // debug-level only. - eprintln!("coordinator TLS handshake from {peer}: {e}"); - return; - } - }; - - // Pull the peer cert chain off the completed handshake - // and convert the leaf into a `ClientIdentity`. Without - // a peer cert (which `with_client_cert_verifier` - // requires) the connection wouldn't reach this point — - // but defense-in-depth: if the chain is empty we let - // the auth middleware reject the request cleanly. - let identity = client_identity_from_stream(&tls_stream); - - // Resolve the per-connection axum Service from the - // make_service. axum's IntoMakeService::call is - // infallible so we can unwrap. - let tower_service = match make_service.call(()).await { - Ok(svc) => svc, - Err(_infallible) => return, - }; - - // Wrap the axum Service with a layer that stamps the - // ClientIdentity into the request extensions. Using - // `tower::ServiceBuilder::map_request` keeps the - // identity attached for every request on this - // connection. Note: hyper hands axum requests over as - // `Request`, NOT - // `axum::extract::Request`, so we type the closure - // generically. - let svc = ServiceBuilder::new() - .map_request(move |mut req: axum::http::Request| { - if let Some(id) = identity.clone() { - req.extensions_mut().insert(id); - } - req - }) - .service(tower_service); - - let hyper_service = TowerToHyperService::new(svc); - let io = TokioIo::new(tls_stream); - if let Err(e) = Builder::new(TokioExecutor::new()) - .serve_connection_with_upgrades(io, hyper_service) - .await - { - eprintln!("coordinator TLS connection from {peer}: {e}"); - } - }); - } -} - -/// Extract a `ClientIdentity` from a completed TLS stream. Pulls -/// the subject CN out of the leaf certificate using the limited -/// DN parser in `cn_from_subject_der`; returns `None` if no peer -/// certs are present (shouldn't happen with a `WebPkiClientVerifier` -/// but defended against anyway). -fn client_identity_from_stream( - stream: &tokio_rustls::server::TlsStream, -) -> Option { - let (_, conn) = stream.get_ref(); - let chain = conn.peer_certificates()?; - let leaf = chain.first()?; - cn_from_cert_der(leaf.as_ref()).map(|cn| ClientIdentity { common_name: cn }) -} - -/// Pull the CN out of an X.509 certificate's Subject DN. -/// -/// Mini-parser that walks the TBSCertificate to the Subject -/// field and reads the CN attribute. The Subject in -/// `TBSCertificate` sits AFTER the issuer DN, so a naive scan -/// for OID 2.5.4.3 would return the issuer's CN — which is the -/// CA, not the worker. We walk explicitly: -/// -/// ```text -/// Certificate ::= SEQUENCE { tbsCertificate, sigAlg, sigValue } -/// TBSCertificate ::= SEQUENCE { -/// [0] version, serialNumber, signature, issuer, -/// validity, subject, subjectPublicKeyInfo, ... -/// } -/// ``` -/// -/// Returns `None` on malformed DER. Corrupt input is unexpected -/// in production — `WebPkiClientVerifier` rejects bad certs -/// before they reach here. -fn cn_from_cert_der(der: &[u8]) -> Option { - // Outer Certificate SEQUENCE — read the TLV and the - // RETURN VALUE is the contents (the three sub-SEQUENCEs). - let (cert_inner, _) = read_tag_value(der, 0x30)?; - // First inner SEQUENCE = tbsCertificate. - let (tbs, _) = read_tag_value(cert_inner, 0x30)?; - - let mut cursor = tbs; - // Optional [0] version — explicit tag 0xA0. - if let Some(rest) = skip_tag_if(cursor, 0xA0) { - cursor = rest; - } - // serialNumber INTEGER (0x02). - let (_, cursor) = read_tag_skip(cursor, 0x02)?; - // signature AlgorithmIdentifier SEQUENCE (0x30). - let (_, cursor) = read_tag_skip(cursor, 0x30)?; - // issuer Name SEQUENCE (0x30) — skip. - let (_, cursor) = read_tag_skip(cursor, 0x30)?; - // validity SEQUENCE (0x30) — skip. - let (_, cursor) = read_tag_skip(cursor, 0x30)?; - // subject Name SEQUENCE (0x30) — this is where CN lives. - let (subject, _) = read_tag_skip(cursor, 0x30)?; - cn_from_dn(subject) -} - -/// Read an ASN.1 DER tag-length-value triple; return -/// `(value_bytes, remainder_after_tlv)`. Supports definite -/// short-form (one length byte) AND definite long-form -/// (multi-byte length) lengths so 256+-byte issuer DNs from -/// large CAs don't trip the parser. -fn read_tag_value(input: &[u8], expected_tag: u8) -> Option<(&[u8], &[u8])> { - let &tag = input.first()?; - if tag != expected_tag { - return None; - } - let &len_byte = input.get(1)?; - let (len, header_len) = if len_byte & 0x80 == 0 { - (len_byte as usize, 2) - } else { - let n = (len_byte & 0x7f) as usize; - if n == 0 || n > 4 { - return None; - } - let len_bytes = input.get(2..2 + n)?; - let mut len = 0usize; - for b in len_bytes { - len = (len << 8) | (*b as usize); - } - (len, 2 + n) - }; - let value = input.get(header_len..header_len + len)?; - let rest = input.get(header_len + len..)?; - Some((value, rest)) -} - -/// Like [`read_tag_value`] but returns `(value, remainder)` and -/// renames the tuple convention to "skip past this TLV." -fn read_tag_skip(input: &[u8], expected_tag: u8) -> Option<(&[u8], &[u8])> { - read_tag_value(input, expected_tag) -} - -/// Skip a TLV if the tag matches; return `None` if it doesn't, -/// indicating the caller should NOT advance. -fn skip_tag_if(input: &[u8], expected_tag: u8) -> Option<&[u8]> { - if input.first() == Some(&expected_tag) { - let (_, rest) = read_tag_value(input, expected_tag)?; - Some(rest) - } else { - None - } -} - -/// Walk a Distinguished Name SEQUENCE OF SET OF AttributeTypeAndValue -/// looking for OID 2.5.4.3 (commonName). -fn cn_from_dn(dn_seq: &[u8]) -> Option { - // OID 2.5.4.3 = CN. Encoded as 06 03 55 04 03. - const CN_OID: &[u8] = &[0x06, 0x03, 0x55, 0x04, 0x03]; - let mut cursor = dn_seq; - while !cursor.is_empty() { - // RelativeDistinguishedName ::= SET OF (0x31) - let (rdn, rest) = read_tag_value(cursor, 0x31)?; - cursor = rest; - let mut atv_cursor = rdn; - while !atv_cursor.is_empty() { - // AttributeTypeAndValue ::= SEQUENCE { type OID, value ANY } - let (atv, after_atv) = read_tag_value(atv_cursor, 0x30)?; - atv_cursor = after_atv; - if atv.starts_with(CN_OID) { - let after_oid = &atv[CN_OID.len()..]; - // Value is one of: PrintableString (0x13), - // UTF8String (0x0c), IA5String (0x16), - // TeletexString (0x14). Accept any and decode - // as UTF-8 (a CN with non-ASCII is unusual but - // we don't reject it here). - let &tag = after_oid.first()?; - let (value, _) = read_tag_value(after_oid, tag)?; - return std::str::from_utf8(value).ok().map(str::to_owned); - } - } - } - None -} - -/// Background lease-expiry sweep task. Runs for the lifetime -/// of the coordinator process; aborted when `axum::serve` -/// exits. -/// -/// Failure semantics: best-effort. Errors log + continue to -/// the next tick. The HTTP server keeps running even if the -/// sweep panics — operators monitor stderr to notice -/// unrecovered failures. -async fn background_sweep_loop(state: CoordinatorState, interval_ms: u64) { - let mut tick = tokio::time::interval(Duration::from_millis(interval_ms)); - // First tick fires immediately; skip it (the startup - // sweep already ran). - tick.tick().await; - // Track poison-mutex state so we log once instead of - // silently skipping forever (adversarial-review F2). - let mut poison_logged = false; - loop { - tick.tick().await; - // `now_ms + 1` matches the startup sweep's threshold - // (line ~109) so the boundary `lease_expires_at == now_ms` - // is treated as expired by both code paths - // (adversarial-review F1). - let now_ms = now_unix_ms(); - let result = { - let store = match state.store.lock() { - Ok(g) => g, - Err(_) => { - if !poison_logged { - eprintln!( - "coordinator sweep: store mutex poisoned; \ - background sweep is now silently skipping ticks. \ - A handler panicked while holding the lock — \ - investigate stderr for the original panic." - ); - poison_logged = true; - } - continue; - } - }; - store.expire_leases_and_requeue(now_ms + 1) - }; - match result { - Ok(0) => {} // no-op tick; quiet - Ok(n) => { - eprintln!("coordinator sweep: requeued {n} expired-lease step(s)") - } - Err(e) => { - eprintln!("coordinator sweep: error {e} — retrying next tick") - } - } - } -} - -/// Constant-time byte-slice equality. Avoids the early-exit pattern of `==` -/// that would leak per-byte timing information about a bearer token's -/// content to a network-adjacent attacker. -/// -/// **Length-leakage:** the early-return on length-mismatch leaks the -/// expected secret length. Acceptable for our use case — operators -/// generate secrets via `openssl rand -hex 32` (a known length) and the -/// length is not what an attacker is trying to brute-force. -fn constant_time_bytes_eq(a: &[u8], b: &[u8]) -> bool { - if a.len() != b.len() { - return false; - } - let mut diff: u8 = 0; - for (x, y) in a.iter().zip(b.iter()) { - diff |= x ^ y; - } - diff == 0 -} - -/// Build a JSON 401 response with the stable `coord.unauthorized` -/// `error_kind`. Matches the `ErrorBody` shape used elsewhere. -fn unauthorized_response() -> Response { - let body = ErrorBody::new( - "coord.unauthorized", - "missing or invalid Authorization: Bearer header", - ); - (StatusCode::UNAUTHORIZED, Json(body)).into_response() -} - -/// Axum middleware that validates BOTH gates the operator has -/// configured: -/// -/// - **mTLS** — when `mtls_required`, the request MUST carry a -/// [`ClientIdentity`] extension installed by the TLS listener -/// (sprint `W6-A`). Missing identity → 401 `coord.unauthorized` -/// (defense-in-depth: the listener guarantees a cert was -/// presented; this catches plumbing bugs). -/// - **Shared-secret bearer** — when `shared_secret` is `Some`, -/// the request MUST carry a matching `Authorization: Bearer …` -/// header (sprint `0.5-S3`). -/// -/// Both gates compose: an mTLS-only coord skips the bearer check; -/// a bearer-only coord skips the cert check; a coord with both -/// enabled requires both. A coord with neither (no flags, no -/// secret) is a pass-through — the pre-0.5-S3 behavior. -async fn auth_middleware( - State(state): State, - headers: HeaderMap, - request: axum::extract::Request, - next: Next, -) -> Response { - // Sprint W2: liveness/readiness probes bypass auth so external - // load balancers (and concerned operators with `curl`) can - // verify a coord is up without holding the shared secret. - // Health responses are non-sensitive — uptime, capability hash, - // and version — so the bypass does not leak secret state. - // The approval-console SHELL (`/console`) is exempt for the same reason as - // health: a browser navigation cannot carry an `Authorization: Bearer` - // header. The shell embeds no run data and no secret — all data reads and - // mutations happen via authed `fetch()` from its inline JS against the - // `/api/*` routes below, which stay behind this middleware — so serving the - // shell unauthenticated leaks nothing. - let path = request.uri().path(); - if path == "/api/health" || path == "/console" { - return next.run(request).await; - } - // Sprint W6-A: when mTLS is required, every other route MUST - // carry a verified ClientIdentity extension installed by the - // TLS listener layer. Defense-in-depth: even if some future - // misconfiguration bypasses the listener, the middleware - // refuses to pass without identity proof. - if state.config.mtls_required && request.extensions().get::().is_none() { - return unauthorized_response(); - } - if let Some(expected) = state.config.shared_secret.as_deref() { - let Some(header_val) = headers.get(axum::http::header::AUTHORIZATION) else { - return unauthorized_response(); - }; - let Ok(header_str) = header_val.to_str() else { - return unauthorized_response(); - }; - let Some(provided) = header_str.strip_prefix("Bearer ") else { - return unauthorized_response(); - }; - if !constant_time_bytes_eq(provided.as_bytes(), expected.as_bytes()) { - return unauthorized_response(); - } - } - next.run(request).await -} - -/// The operator approval console — a static, data-free HTML shell. -/// -/// Security model (audited): the shell embeds ZERO run state and ZERO secrets. -/// It is a token-entry form plus inline JS that calls the AUTHED `/api/runs`, -/// `/api/runs/{id}` and `/api/runs/{id}/approve` endpoints. The operator's -/// bearer token is held only in an in-memory input (cleared on unload) and sent -/// ONLY in the request `Authorization` header — never in a URL, never in -/// storage. All dynamic content is built with `textContent`/`createElement` -/// (never `innerHTML`), so a hostile `run_id`/`step_id` cannot inject markup or -/// script. Approvals additionally require the per-gate S9 token, typed by the -/// human. No external/CDN assets — fully self-contained for offline operation. -const CONSOLE_HTML: &str = r##" - - - - -Approval Console — Boruna - - - -

Boruna Approval Console

-

Approve or reject workflow runs paused at a human approval gate. This page holds no data of its own — it reads the coordinator's authenticated API and submits your decision. It cannot start, cancel, or edit runs.

- -
- 1 · Connect - - -
- - Kept in memory only; sent solely in the request Authorization header — never stored or placed in the URL. -
-
- -
-
- -
Read-only shell over the coordinator's authenticated API. Every action requires your bearer token plus the per-gate approval token issued when the gate paused. Keep the coordinator on a trusted network — it ships no built-in login.
- - - -"##; - -/// GET `/console` — serve the approval-console shell (see [`CONSOLE_HTML`]). -/// -/// EXEMPT from [`auth_middleware`] (like `/api/health`): a browser navigation -/// carries no `Authorization` header, so the shell must load without one. It -/// leaks nothing — no run data, no secret — and the `/api/*` endpoints its JS -/// calls stay behind the middleware. The response denies framing (clickjacking) -/// and forbids caching. -async fn handle_console() -> Response { - ( - [ - (axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8"), - (axum::http::header::CACHE_CONTROL, "no-store"), - (axum::http::header::X_FRAME_OPTIONS, "DENY"), - ( - axum::http::header::CONTENT_SECURITY_POLICY, - // Fully self-contained, same-origin: allow only inline - // style/script and same-origin fetch; deny everything else, - // framing, and injection (defense-in-depth beyond the - // X-Frame-Options above). - "default-src 'none'; style-src 'unsafe-inline'; \ - script-src 'unsafe-inline'; connect-src 'self'; \ - frame-ancestors 'none'; base-uri 'none'", - ), - ], - CONSOLE_HTML, - ) - .into_response() -} - -pub fn build_router(state: CoordinatorState) -> Router { - // Sprint 0.5-S2d: merge the dashboard's read-only routes - // (/, /runs/:id, /api/runs, /api/runs/:id) onto the - // coordinator's listener so operators get fleet visibility - // + distributed dispatch from a single port. - // - // Route paths don't overlap by design (per ADR 002): the - // coordinator owns /api/work/* and /api/workers/*; the - // dashboard owns /api/runs and /api/runs/:id. The HTML - // routes (/ and /runs/:id) are dashboard-only. - // - // The coordinator's bind_warning flows into the dashboard - // builder so the red HTML banner appears on coordinator- - // served pages too. - let dashboard_router = - crate::dashboard::dashboard_routes(state.store.clone(), state.config.bind_warning.clone()); - // Sprint 0.5-S3: auth middleware applies to BOTH coord routes - // (mutations + claims) AND the dashboard's read-only routes - // (since they expose run state including step_sources). Operators - // who specifically want a public read-only dashboard with auth- - // gated mutations should run a separate `boruna dashboard serve` - // process without the shared-secret. The merged listener is - // strictly all-or-nothing for auth. - let coord_router = Router::new() - .route("/api/workers/register", post(handle_register)) - .route("/api/workers/heartbeat", post(handle_heartbeat)) - .route("/api/work/claim", get(handle_claim)) - .route("/api/work/complete", post(handle_complete)) - .route("/api/work/fail", post(handle_fail)) - .route("/api/work/extend-lease", post(handle_extend_lease)) - // Sprint 0.5-S4: operator-facing routes for CI runners that - // do not share a data-dir with the cluster. Same auth - // middleware as worker routes. - .route("/api/runs/submit", post(handle_submit_run)) - .route("/api/runs/{run_id}/status", get(handle_run_status)) - // Sprint 0.5-S6: operator-facing routes for human-in-the-loop - // and webhook-driven gates. Same bearer-token auth as the - // submit / status routes. - .route("/api/runs/{run_id}/approve", post(handle_approve_run)) - .route("/api/runs/{run_id}/trigger", post(handle_trigger_run)) - // Sprint 0.5-S7: fetch a large step output stored in the - // coordinator's blob store. Run-scoped: the route only - // returns bytes if the requested hash is referenced by a - // checkpoint under this run, preventing the route from - // serving as a generic blob server. Same auth as the rest. - .route("/api/runs/{run_id}/blobs/{hash}", get(handle_get_blob)) - // Sprint W2: liveness/readiness probe for HA deployments. - // Returns 200 + a small JSON document when the coord is - // healthy; 503 when the SQLite store is unreachable. The - // health check probes the store via a lightweight query - // (PRAGMA quick_check would be too expensive; we just take - // and release the mutex guard). - .route("/api/health", get(handle_health)) - // Sprint W6-B: operator approval console (HTML shell). Exempted from - // auth in `auth_middleware` because a browser navigation carries no - // bearer header; it holds no data and drives the authed `/api/*` routes - // above via fetch(). Added to `coord_router` (not the dashboard router) - // so it is served wherever the coordinator runs. - .route("/console", get(handle_console)) - // The 8 MiB DefaultBodyLimit applies to coord routes - // ONLY (not dashboard routes) because Axum's per- - // router layer scoping means layers attached pre-merge - // stay bound to their own routes. Dashboard is - // GET-only today, so no body-limit need. If a future - // sprint adds a mutating dashboard route (e.g. "cancel - // run"), it must opt into a body limit explicitly OR - // be added to coord_router instead. - .layer(DefaultBodyLimit::max(MAX_BODY_BYTES)) - .with_state(state.clone()); - let merged = coord_router.merge(dashboard_router); - merged.layer(middleware::from_fn_with_state(state, auth_middleware)) -} - -// ── Wire shapes ── - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct RegisterRequest { - #[serde(default)] - pub worker_id: Option, - pub capability_set_hash: String, - /// Sprint `W3-A` — optional list of capability names this - /// worker advertises. When `None` (the default for - /// pre-W3-A workers), the coord treats the worker as a - /// full-fleet worker holding ALL capabilities. When - /// `Some(list)`, the coord only routes steps whose - /// policy-required capabilities are a subset of `list`. - /// Names are drawn from `boruna_bytecode::Capability::ALL` - /// (e.g. `"net.fetch"`, `"db.query"`); unknown names - /// reject registration with `coord.unknown_capability`. - /// - /// Post-1.0 (T-1.3): each entry can be either a bare string - /// (legacy / pre-1.x worker) or a `{name, version}` object - /// (version-aware worker). Coord normalizes legacy entries to - /// the coord's current `Capability::version()` for that name - /// on receipt; from there everything is `(name, version)`-keyed. - /// - /// **Operational state only** (project §15) — does NOT - /// participate in `capability_set_hash` and is purely a - /// placement filter. The VM's capability gateway remains - /// the security boundary. - #[serde(default)] - pub advertised_capabilities: Option>, -} - -/// Wire shape for one entry in `RegisterRequest.advertised_capabilities`. -/// -/// `untagged` so legacy bare-string entries from pre-1.1 workers -/// continue to deserialize. The coord normalizes both to -/// `(name, version)` immediately after parse. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -#[serde(untagged)] -pub enum CapabilityAdvertisement { - /// Legacy worker: just the capability name. The coord normalizes - /// to its own current `Capability::version()` for that name on - /// receipt. - Legacy(String), - /// Version-aware worker: explicit `(name, version)` pair. - Versioned { name: String, version: String }, -} - -impl CapabilityAdvertisement { - pub fn name(&self) -> &str { - match self { - CapabilityAdvertisement::Legacy(name) => name, - CapabilityAdvertisement::Versioned { name, .. } => name, - } - } -} - -impl From<&str> for CapabilityAdvertisement { - fn from(name: &str) -> Self { - CapabilityAdvertisement::Legacy(name.to_string()) - } -} - -impl From for CapabilityAdvertisement { - fn from(name: String) -> Self { - CapabilityAdvertisement::Legacy(name) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct RegisterResponse { - pub protocol_version: u32, - pub worker_id: String, - pub session_token: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct HeartbeatRequest { - pub worker_id: String, - pub session_token: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct OkResponse { - pub protocol_version: u32, - pub ok: bool, -} - -#[derive(Deserialize, Debug, Clone)] -pub struct ClaimQuery { - pub worker_id: String, - pub session_token: String, - pub lease_ttl_ms: u64, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct WorkItem { - pub protocol_version: u32, - pub run_id: String, - pub step_id: String, - pub claim_id: u64, - pub lease_expires_at_ms: i64, - pub source: String, - pub policy_json: String, - #[serde(default)] - pub inputs_json: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct CompleteRequest { - pub worker_id: String, - pub session_token: String, - pub run_id: String, - pub step_id: String, - pub claim_id: u64, - pub output_json: String, - pub output_hash: String, - pub attempt_count: u32, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FailRequest { - pub worker_id: String, - pub session_token: String, - pub run_id: String, - pub step_id: String, - pub claim_id: u64, - pub error_msg: String, - pub attempt_count: u32, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ExtendLeaseRequest { - pub worker_id: String, - pub session_token: String, - pub run_id: String, - pub step_id: String, - pub claim_id: u64, - pub extend_by_ms: u64, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ExtendLeaseResponse { - pub protocol_version: u32, - pub new_lease_expires_at_ms: i64, -} - -/// Sprint `0.5-S4` — operator-side `POST /api/runs/submit` payload. -/// Inlines the full workflow definition + every Source-kind step's -/// `.ax` body so the coordinator's data-dir is the single source of -/// truth (CI runner does not need shared filesystem access). -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct SubmitRunRequest { - pub workflow: boruna_orchestrator::workflow::definition::WorkflowDef, - #[serde(default)] - pub step_sources: BTreeMap, - #[serde(default)] - pub policy: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct SubmitRunResponse { - pub protocol_version: u32, - pub run_id: String, - pub workflow_hash: String, -} - -/// Sprint `0.5-S6` — `POST /api/runs/{run_id}/approve` body. Decision -/// is the canonical lowercase string (`"approved"` | `"rejected"`) -/// so the wire format matches the local CLI's argument shape. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ApproveRequest { - pub step_id: String, - pub decision: String, - #[serde(default)] - pub reason: Option, - /// Per-gate approval token stashed at pause-time (finding S9). Required: - /// without it any bearer/worker-cert holder could seize the gate. Mirrors - /// `TriggerRequest.token`. Defaults to empty so a token-less legacy body - /// deserializes — but an empty token never matches a stashed token, so it - /// is rejected (fail-closed). - #[serde(default)] - pub token: String, -} - -/// Sprint `0.5-S6` — `POST /api/runs/{run_id}/trigger` body. The -/// `token` field is the per-step trigger token stashed at gate-pause -/// time (NOT the bearer token for the auth middleware — that goes -/// in the `Authorization` header). Two separate secrets matches the -/// 0.3-S15 trigger model unchanged. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct TriggerRequest { - pub step_id: String, - pub token: String, - pub payload: String, -} - -/// Sprint `0.5-S4` — `GET /api/runs/{run_id}/status` response. -/// Per-step status map mirrors the format `coordinator wait` uses -/// for stdout transition lines so a future HTTP-mode `wait` can -/// reuse the same wire shape. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct RunStatusResponse { - pub protocol_version: u32, - pub run_id: String, - pub status: String, - pub step_statuses: BTreeMap, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_msg: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ErrorBody { - pub protocol_version: u32, - pub error_kind: String, - pub message: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub current_claim_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub current_status: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub expected_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub max_bytes: Option, -} - -impl ErrorBody { - fn new(error_kind: &str, message: impl Into) -> Self { - Self { - protocol_version: PROTOCOL_VERSION, - error_kind: error_kind.into(), - message: message.into(), - current_claim_id: None, - current_status: None, - expected_hash: None, - max_bytes: None, - } - } -} - -fn respond_err(status: StatusCode, body: ErrorBody) -> Response { - (status, Json(body)).into_response() -} - -/// Sprint `W3-A` (extended in post1-T-1.3) — validate and normalize -/// the `advertised_capabilities` list at the parse boundary. -/// Returns a `(name -> version)` map (or `None` if the field was -/// absent), or the first unknown name on failure. Names are matched -/// against `Capability::ALL` exact canonical names (e.g. -/// `"net.fetch"`, `"db.query"`); aliases from `Capability::from_name` -/// are NOT accepted here because the taxonomy needs to be -/// unambiguous on the wire. -/// -/// Legacy bare-string entries are normalized to the coord's current -/// `Capability::version()` for that name. Versioned entries keep -/// their explicit version (which may be older than the coord's, in -/// which case the worker won't be eligible for steps requiring the -/// newer version — see the claim filter). -fn validate_advertised_capabilities( - list: Option<&[CapabilityAdvertisement]>, -) -> Result>, String> { - let Some(items) = list else { - return Ok(None); - }; - let mut map = BTreeMap::new(); - for entry in items { - let name = entry.name(); - let cap = boruna_bytecode::Capability::from_name(name).ok_or_else(|| name.to_string())?; - let version = match entry { - CapabilityAdvertisement::Legacy(_) => cap.version().to_string(), - CapabilityAdvertisement::Versioned { version, .. } => version.clone(), - }; - map.insert(name.to_string(), version); - } - Ok(Some(map)) -} - -#[cfg(test)] -fn parse_major_minor(s: &str) -> Option<(u32, u32)> { - let (major, minor) = s.split_once('.')?; - Some((major.parse().ok()?, minor.parse().ok()?)) -} - -#[cfg(test)] -pub fn semver_gte(worker_ver: &str, required_ver: &str) -> bool { - match ( - parse_major_minor(worker_ver), - parse_major_minor(required_ver), - ) { - (Some(wv), Some(rv)) => wv >= rv, - _ => worker_ver == required_ver, - } -} - -#[cfg(test)] -pub fn version_compatible( - worker_versions: &BTreeMap, - required_versions: &BTreeMap, -) -> bool { - required_versions.iter().all(|(cap, required)| { - let worker_ver = worker_versions - .get(cap) - .map(|s| s.as_str()) - .unwrap_or("1.0"); - semver_gte(worker_ver, required) - }) -} - -/// Post-1.0 (T-1.3) — outcome of comparing a worker's advertised -/// `(name, version)` set against a step's required cap-names. -/// -/// `Covered` — worker can claim the step. -/// `MissingName` — worker doesn't advertise one of the required -/// names at all. This is the W3-A placement filter (silent skip). -/// `WrongVersion` — worker advertises a required name but at a -/// different version than the coord's current -/// `Capability::version()`. This surfaces -/// `coord.capability_version_mismatch` to the operator so they -/// can roll out matching workers. -enum CoverageOutcome { - Covered, - MissingName, - WrongVersion, -} - -fn worker_covers_required( - advertised: &BTreeMap, - required: &BTreeSet, -) -> CoverageOutcome { - let mut wrong_version = false; - for name in required { - let Some(adv_ver) = advertised.get(name) else { - return CoverageOutcome::MissingName; - }; - let Some(cap) = boruna_bytecode::Capability::from_name(name) else { - // Unknown cap on the required side — be conservative - // and treat as missing. - return CoverageOutcome::MissingName; - }; - let required_ver = cap.version(); - // Versions are `&str` compared by equality. The plan keeps - // versions as opaque tokens for 1.x; defining a `>=` - // ordering is a 2.0 concern. Equality means "the coord's - // current version" and is enough to surface drift. - if adv_ver != required_ver { - wrong_version = true; - } - } - if wrong_version { - CoverageOutcome::WrongVersion - } else { - CoverageOutcome::Covered - } -} - -/// Sprint `W3-A` — extract the set of capability names a step -/// requires from its workflow's serialized `Policy`. A capability -/// is "required" if (a) it appears in `rules` with `allow: true`, -/// or (b) `default_allow == true`, in which case ALL capabilities -/// are potentially required (a worker MUST advertise the full -/// fleet to claim such a step). Best-effort placement only — -/// malformed JSON falls back to "no requirements known", which -/// remains safe because the VM gateway re-checks at execution. -fn required_capabilities_from_policy(policy_json: &str) -> BTreeSet { - let mut required = BTreeSet::new(); - let Ok(v) = serde_json::from_str::(policy_json) else { - return required; - }; - let default_allow = v - .get("default_allow") - .and_then(|x| x.as_bool()) - .unwrap_or(false); - if default_allow { - // Wildcard policy — caller MUST advertise every capability. - for cap in boruna_bytecode::Capability::ALL.iter() { - required.insert(cap.name().to_string()); - } - return required; - } - if let Some(rules) = v.get("rules").and_then(|x| x.as_object()) { - for (name, rule) in rules { - let allow = rule.get("allow").and_then(|x| x.as_bool()).unwrap_or(false); - if allow { - required.insert(name.clone()); - } - } - } - required -} - -// ── Handlers ── - -async fn handle_register( - State(state): State, - identity: Option>, - Json(req): Json, -) -> Response { - if req.capability_set_hash != state.capability_set_hash { - let mut body = ErrorBody::new( - "coord.binary_mismatch", - format!( - "worker hash {:?} does not match coordinator's {:?}", - req.capability_set_hash, state.capability_set_hash - ), - ); - body.expected_hash = Some(state.capability_set_hash.clone()); - return respond_err(StatusCode::CONFLICT, body); - } - - // Sprint `W3-A` — validate and normalize advertised capability - // names at the parse boundary (project §1: reject unknown - // capability names with the stable taxonomy entry - // `coord.unknown_capability`). - let advertised_capabilities = - match validate_advertised_capabilities(req.advertised_capabilities.as_deref()) { - Ok(set) => set, - Err(unknown) => { - return respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new( - "coord.unknown_capability", - format!( - "advertised capability {unknown:?} is not a known capability name; \ - expected names from boruna_bytecode::Capability::ALL" - ), - ), - ); - } - }; - - // mTLS identity reconciliation (sprint W6-A). When the request - // arrived over a verified mTLS channel the listener stamps a - // `ClientIdentity` extension carrying the cert subject CN. If - // the body also includes a `worker_id` it MUST match - // (case-insensitive) — otherwise a worker holding a valid - // cert could impersonate any worker_id, defeating per-worker - // identity. Mismatch → 401 `coord.identity_mismatch`. - let cert_cn = identity.map(|axum::extract::Extension(id)| id.common_name); - if let (Some(cn), Some(body_id)) = (cert_cn.as_deref(), req.worker_id.as_deref()) { - if !cn.eq_ignore_ascii_case(body_id) { - return respond_err( - StatusCode::UNAUTHORIZED, - ErrorBody::new( - "coord.identity_mismatch", - format!("client cert CN '{cn}' does not match request worker_id '{body_id}'"), - ), - ); - } - } - - // CN drives identity when present; otherwise honor the body - // worker_id, otherwise auto-generate as before. - let worker_id = match (cert_cn, req.worker_id) { - (Some(cn), _) => cn, - (None, Some(id)) => id, - (None, None) => format!("wkr-{}", uuid::Uuid::new_v4().simple()), - }; - let session_token = format!("sess-{}", uuid::Uuid::new_v4().simple()); - - if state.config.mtls_required { - eprintln!("coordinator: registering worker '{worker_id}' via mTLS cert"); - } - - { - let mut workers = match state.workers.lock() { - Ok(g) => g, - Err(_) => return internal_error("workers lock poisoned"), - }; - workers.insert( - worker_id.clone(), - WorkerSession { - session_token: session_token.clone(), - last_heartbeat_ms: now_unix_ms(), - capability_set_hash: req.capability_set_hash, - advertised_capabilities, - }, - ); - } - - Json(RegisterResponse { - protocol_version: PROTOCOL_VERSION, - worker_id, - session_token, - }) - .into_response() -} - -async fn handle_heartbeat( - State(state): State, - Json(req): Json, -) -> Response { - let mut workers = match state.workers.lock() { - Ok(g) => g, - Err(_) => return internal_error("workers lock poisoned"), - }; - match workers.get_mut(&req.worker_id) { - Some(sess) if sess.session_token == req.session_token => { - sess.last_heartbeat_ms = now_unix_ms(); - Json(OkResponse { - protocol_version: PROTOCOL_VERSION, - ok: true, - }) - .into_response() - } - _ => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new( - "coord.unknown_worker", - format!("worker {} not registered; re-register", req.worker_id), - ), - ), - } -} - -async fn handle_claim( - State(state): State, - Query(q): Query, -) -> Response { - // Validate worker session and capture its advertised - // capability set (sprint `W3-A`). - let advertised = { - let workers = match state.workers.lock() { - Ok(g) => g, - Err(_) => return internal_error("workers lock poisoned"), - }; - match workers.get(&q.worker_id) { - Some(sess) if sess.session_token == q.session_token => { - sess.advertised_capabilities.clone() - } - _ => { - return respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new( - "coord.unknown_worker", - format!("worker {} not registered; re-register", q.worker_id), - ), - ); - } - } - }; - - // Cap lease TTL at coordinator config. - let lease_ttl_ms = q.lease_ttl_ms.min(state.config.max_lease_ttl_ms); - let poll_timeout = Duration::from_millis(state.config.poll_timeout_ms); - let poll_interval = Duration::from_millis(250); - let deadline = std::time::Instant::now() + poll_timeout; - - loop { - match try_claim_one(&state, &q.worker_id, lease_ttl_ms, advertised.as_ref()) { - Ok(Some(item)) => return Json(item).into_response(), - Ok(None) => { - if std::time::Instant::now() >= deadline { - return StatusCode::NO_CONTENT.into_response(); - } - tokio::time::sleep(poll_interval).await; - } - Err(resp) => return resp, - } - } -} - -/// Find one claimable Pending step in any run, atomically claim -/// it, and return the work item. Returns Ok(None) if nothing -/// claimable. -// `Response` is a fairly large enum; clippy's `result_large_err` -// fires here. Acceptable: this Err is the slow path and we -// don't want to box a heap allocation for every successful claim. -#[allow(clippy::result_large_err)] -fn try_claim_one( - state: &CoordinatorState, - worker_id: &str, - lease_ttl_ms: u64, - advertised: Option<&BTreeMap>, -) -> Result, Response> { - let now_ms = now_unix_ms(); - let lease_expires_at_ms = now_ms + lease_ttl_ms as i64; - - // Look up a Pending step. We do this in two phases to keep - // the lock-hold short: first find the (run_id, step_id), - // then call claim_step which has its own atomic CAS. - let candidate = { - let store = state - .store - .lock() - .map_err(|_| internal_error("store lock poisoned"))?; - find_one_pending_step(&store, advertised) - .map_err(|e| internal_error(&format!("scan pending: {e}")))? - }; - - let (run_id, step_id, source, policy_json) = match candidate { - PendingScanOutcome::Found(t) => t, - PendingScanOutcome::NoneAvailable => return Ok(None), - PendingScanOutcome::VersionMismatch => { - // post1-T-1.3 — pending steps exist but their required - // capability versions are not covered by this worker's - // advertisement. Surface a stable error_kind so the - // operator can scale up matching workers. - return Err(json_error_response( - axum::http::StatusCode::CONFLICT, - "coord.capability_version_mismatch", - "no advertised capability covers the version required by any pending step", - )); - } - }; - - let claim_id = { - let store = state - .store - .lock() - .map_err(|_| internal_error("store lock poisoned"))?; - match store - .claim_step(&run_id, &step_id, worker_id, lease_expires_at_ms, now_ms) - .map_err(|e| internal_error(&format!("claim_step: {e}")))? - { - ClaimOutcome::Claimed { claim_id } => claim_id, - ClaimOutcome::NotClaimable { .. } | ClaimOutcome::StepNotFound => { - // Race: someone else claimed between our SELECT and - // claim_step. The caller's loop will retry. - return Ok(None); - } - } - }; - - Ok(Some(WorkItem { - protocol_version: PROTOCOL_VERSION, - run_id, - step_id, - claim_id, - lease_expires_at_ms, - source, - policy_json, - inputs_json: None, - })) -} - -/// Scan runs.db for one Pending step. Returns -/// `(run_id, step_id, source, policy_json)`. The `source` is -/// resolved from the workflow_dir map populated at startup OR -/// inline in the run's metadata_json. For this MVP we use a -/// simple convention: metadata_json optionally carries -/// `step_sources: { "": "<.ax source>" }`. This keeps -/// the test surface simple while leaving room for the future -/// `boruna workflow run --coordinator` to populate it from -/// workflow_dir. -/// Tuple of `(run_id, step_id, source, policy_json)` returned -/// by [`find_one_pending_step`] when a claimable step exists. -type PendingStepDescriptor = (String, String, String, String); - -/// Outcome of scanning for a pending step that the calling worker -/// can claim. -/// -/// `Found` — a matching step exists; the worker should claim it. -/// `NoneAvailable` — no pending steps anywhere; the worker should -/// long-poll. `VersionMismatch` — pending steps exist but every -/// candidate requires a capability version this worker does not -/// advertise; the coord surfaces `coord.capability_version_mismatch` -/// in the claim response so the operator can scale up matching -/// workers (post1-T-1.3). -pub enum PendingScanOutcome { - Found(PendingStepDescriptor), - NoneAvailable, - VersionMismatch, -} - -fn find_one_pending_step( - store: &RunCheckpointStore, - advertised: Option<&BTreeMap>, -) -> Result> { - let runs = store.list_runs_by_status(RunStatus::Running)?; - let mut saw_pending_but_mismatched = false; - for run in runs { - let steps = store.list_step_checkpoints(&run.run_id)?; - for step in steps { - if step.status == StepStatus::Pending { - // Sprint `W3-A` (extended in post1-T-1.3) — placement - // filter. A worker that declared an advertised - // capability set sees only steps whose policy-required - // capabilities each resolve to a `(name, version)` the - // worker advertised at >= the coord's required version. - // Workers that did NOT advertise (i.e. `None`) match - // every step (the pre-W3-A behavior). - if let Some(adv) = advertised { - let required = required_capabilities_from_policy(&run.policy_json); - match worker_covers_required(adv, &required) { - CoverageOutcome::Covered => {} - CoverageOutcome::MissingName => { - // W3-A: silent skip; operator has - // intentionally restricted this worker - // to a subset of capabilities. - continue; - } - CoverageOutcome::WrongVersion => { - // post1-T-1.3: surface to the operator; - // the worker's binary is out of step - // with what the workflow needs. - saw_pending_but_mismatched = true; - continue; - } - } - } - let source = extract_step_source(&run.metadata_json, &step.step_id) - .ok_or_else(|| format!( - "step {} in run {} has no inline source; metadata_json.step_sources missing", - step.step_id, run.run_id - ))?; - return Ok(PendingScanOutcome::Found(( - run.run_id, - step.step_id, - source, - run.policy_json, - ))); - } - } - } - if saw_pending_but_mismatched { - Ok(PendingScanOutcome::VersionMismatch) - } else { - Ok(PendingScanOutcome::NoneAvailable) - } -} - -/// Pull the step's `.ax` source from the run's metadata JSON. -/// Convention for the MVP: `metadata_json` looks like -/// `{ "step_sources": { "extract": "fn main()..." } }`. -fn extract_step_source(metadata_json: &str, step_id: &str) -> Option { - let v: serde_json::Value = serde_json::from_str(metadata_json).ok()?; - v.get("step_sources")? - .get(step_id)? - .as_str() - .map(String::from) -} - -/// S6 (cross-worker claim ownership): reject a complete/fail/extend whose caller -/// is not the worker holding the step's claim. `claim_id` is a predictable -/// per-step counter, so it cannot prove ownership; the row's `worker_id` can. -/// Called under the store lock so the read + CAS are atomic within this process. -/// Returns `Some(rejection)` to short-circuit, `None` to proceed. -fn reject_if_not_claim_owner( - store: &RunCheckpointStore, - run_id: &str, - step_id: &str, - caller_worker_id: &str, -) -> Option { - match store.step_claimed_by(run_id, step_id) { - Ok(Some(holder)) if holder != caller_worker_id => Some(respond_err( - StatusCode::FORBIDDEN, - ErrorBody::new( - "coord.claim_not_owned", - format!( - "step {run_id}/{step_id} is claimed by another worker; caller \ - '{caller_worker_id}' does not own the claim" - ), - ), - )), - Ok(_) => None, - Err(e) => Some(internal_error(&format!("step_claimed_by: {e}"))), - } -} - -async fn handle_complete( - State(state): State, - Json(req): Json, -) -> Response { - if let Err(resp) = validate_session(&state, &req.worker_id, &req.session_token) { - return resp; - } - // Content-addressing integrity: the `output_hash` feeds the audit chain, so a - // worker must not be able to commit an output whose hash lies about its bytes. - // Recompute the worker's hash format (`sha256:` + lowercase hex) over the - // reported `output_json` and reject a mismatch before it reaches the store. - let expected_hash = worker_output_hash(&req.output_json); - if req.output_hash != expected_hash { - return respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new( - "coord.output_hash_mismatch", - format!( - "output_hash does not match SHA-256(output_json): claimed {}, computed {}", - req.output_hash, expected_hash - ), - ), - ); - } - let store = match state.store.lock() { - Ok(g) => g, - Err(_) => return internal_error("store lock poisoned"), - }; - if let Some(resp) = reject_if_not_claim_owner(&store, &req.run_id, &req.step_id, &req.worker_id) - { - return resp; - } - let now_ms = now_unix_ms(); - let outcome = match store.complete_step_cas( - &req.run_id, - &req.step_id, - req.claim_id, - &req.output_json, - &req.output_hash, - req.attempt_count, - now_ms, - ) { - Ok(o) => o, - Err(e) => return internal_error(&format!("complete_step_cas: {e}")), - }; - drop(store); - terminal_outcome_to_response(outcome) -} - -async fn handle_fail( - State(state): State, - Json(req): Json, -) -> Response { - if let Err(resp) = validate_session(&state, &req.worker_id, &req.session_token) { - return resp; - } - let store = match state.store.lock() { - Ok(g) => g, - Err(_) => return internal_error("store lock poisoned"), - }; - if let Some(resp) = reject_if_not_claim_owner(&store, &req.run_id, &req.step_id, &req.worker_id) - { - return resp; - } - let now_ms = now_unix_ms(); - let outcome = match store.fail_step_cas( - &req.run_id, - &req.step_id, - req.claim_id, - &req.error_msg, - req.attempt_count, - now_ms, - ) { - Ok(o) => o, - Err(e) => return internal_error(&format!("fail_step_cas: {e}")), - }; - drop(store); - terminal_outcome_to_response(outcome) -} - -async fn handle_extend_lease( - State(state): State, - Json(req): Json, -) -> Response { - if let Err(resp) = validate_session(&state, &req.worker_id, &req.session_token) { - return resp; - } - // Adversarial-review F4: enforce a floor so a worker that - // mistakenly passes `extend_by_ms: 0` doesn't get a - // 200-OK with a lease deadline that's already in the past. - // 1s is the minimum reasonable extension; below that the - // round-trip latency itself dominates. - const MIN_EXTEND_MS: u64 = 1_000; - if req.extend_by_ms < MIN_EXTEND_MS { - return respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new( - "coord.invalid_request", - format!( - "extend_by_ms {} below minimum {} ms", - req.extend_by_ms, MIN_EXTEND_MS - ), - ), - ); - } - let extend_by_ms = req.extend_by_ms.min(state.config.max_lease_ttl_ms); - let new_lease_expires_at_ms = now_unix_ms() + extend_by_ms as i64; - let store = match state.store.lock() { - Ok(g) => g, - Err(_) => return internal_error("store lock poisoned"), - }; - if let Some(resp) = reject_if_not_claim_owner(&store, &req.run_id, &req.step_id, &req.worker_id) - { - return resp; - } - let outcome = match store.extend_lease_cas( - &req.run_id, - &req.step_id, - req.claim_id, - new_lease_expires_at_ms, - ) { - Ok(o) => o, - Err(e) => return internal_error(&format!("extend_lease_cas: {e}")), - }; - drop(store); - match outcome { - ExtendOutcome::Extended { - new_lease_expires_at_ms, - } => Json(ExtendLeaseResponse { - protocol_version: PROTOCOL_VERSION, - new_lease_expires_at_ms, - }) - .into_response(), - ExtendOutcome::LeaseExpired { - current_claim_id, - current_status, - } => { - let mut body = ErrorBody::new( - "coord.lease_expired", - format!( - "lease for step has claim_id={current_claim_id} status={}", - current_status.as_str() - ), - ); - body.current_claim_id = Some(current_claim_id); - body.current_status = Some(current_status.as_str().to_string()); - respond_err(StatusCode::CONFLICT, body) - } - ExtendOutcome::StepNotFound => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new("coord.step_not_found", "step not found"), - ), - } -} - -fn terminal_outcome_to_response(outcome: TerminalOutcome) -> Response { - match outcome { - TerminalOutcome::Committed => Json(OkResponse { - protocol_version: PROTOCOL_VERSION, - ok: true, - }) - .into_response(), - TerminalOutcome::LeaseExpired { - current_claim_id, - current_status, - } => { - let mut body = ErrorBody::new( - "coord.lease_expired", - format!( - "step has claim_id={current_claim_id} status={}", - current_status.as_str() - ), - ); - body.current_claim_id = Some(current_claim_id); - body.current_status = Some(current_status.as_str().to_string()); - respond_err(StatusCode::CONFLICT, body) - } - TerminalOutcome::StepNotFound => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new("coord.step_not_found", "step not found"), - ), - } -} - -// Same `result_large_err` justification as `try_claim_one`. -#[allow(clippy::result_large_err)] -fn validate_session( - state: &CoordinatorState, - worker_id: &str, - session_token: &str, -) -> Result<(), Response> { - let workers = state - .workers - .lock() - .map_err(|_| internal_error("workers lock poisoned"))?; - match workers.get(worker_id) { - Some(sess) if sess.session_token == session_token => Ok(()), - _ => Err(respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new( - "coord.unknown_worker", - format!("worker {worker_id} not registered; re-register"), - ), - )), - } -} - -/// Recompute a worker's `output_hash` from its `output_json`, matching the exact -/// format the worker produces (`sha256:` + lowercase hex of `SHA-256(bytes)` — see -/// `worker::execute_step`). Used to reject content-addressing forgery at -/// `handle_complete`. -fn worker_output_hash(output_json: &str) -> String { - use sha2::{Digest, Sha256}; - let mut hasher = Sha256::new(); - hasher.update(output_json.as_bytes()); - let digest = hasher.finalize(); - let mut hex = String::with_capacity(7 + 64); - hex.push_str("sha256:"); - for b in digest { - hex.push_str(&format!("{b:02x}")); - } - hex -} - -fn internal_error(msg: &str) -> Response { - let body = ErrorBody::new("coord.invalid_request", msg); - (StatusCode::INTERNAL_SERVER_ERROR, Json(body)).into_response() -} - -fn json_error_response(status: StatusCode, error_kind: &str, msg: &str) -> Response { - let body = ErrorBody::new(error_kind, msg); - (status, Json(body)).into_response() -} - -fn now_unix_ms() -> i64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0) -} - -// ── operator-facing submit + status (sprint 0.5-S4) ── - -/// `POST /api/runs/submit` — register a workflow run from a CI -/// runner that does NOT share a `data-dir` with the cluster. Body -/// inlines the workflow def + every Source-kind step's `.ax` body; -/// the cluster persists everything into `metadata.audit_log`'s -/// surrounding metadata blob (same surface that `submit-only` mode -/// populates from disk). Auth: bearer-gated by the existing -/// `auth_middleware`. Failures map to the same `error_kind` -/// taxonomy used by other coordinator routes, plus three new kinds -/// scoped to this surface (`coord.submit.*`). -async fn handle_submit_run( - State(state): State, - Json(req): Json, -) -> Response { - use boruna_orchestrator::workflow::WorkflowRunner; - let policy = req.policy.unwrap_or_default(); - let store = state.store.clone(); - let store_guard = match store.lock() { - Ok(g) => g, - Err(e) => return internal_error(&format!("store mutex poisoned: {e}")), - }; - let result = WorkflowRunner::submit_with_inline_sources( - &req.workflow, - req.step_sources, - &policy, - &store_guard, - ); - drop(store_guard); - match result { - Ok(run_id) => { - let workflow_hash = WorkflowRunner::workflow_hash_from_def(&req.workflow); - ( - StatusCode::OK, - Json(SubmitRunResponse { - protocol_version: PROTOCOL_VERSION, - run_id, - workflow_hash, - }), - ) - .into_response() - } - Err(e) => match e { - boruna_orchestrator::workflow::WorkflowRunError::Validation(msg) => respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new("coord.submit.invalid_workflow", msg), - ), - boruna_orchestrator::workflow::WorkflowRunError::Internal(msg) => respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new("coord.submit.bad_payload", msg), - ), - other => internal_error(&format!("submit failed: {other}")), - }, - } -} - -/// `GET /api/runs/{run_id}/status` — return a compact status -/// snapshot for the named run. The shape (`status` string + -/// per-step status map + optional `error_msg`) matches what -/// `coordinator wait`'s stdout reflects, so a future HTTP-mode -/// `wait` can reuse the same wire format. 404 with stable -/// `coord.runs.not_found` when the run_id isn't in the store. -/// -/// The handler also advances the run one tick before reading -/// state. In the local-data-dir model `coordinator wait` was the -/// thing that drove `advance_run_one_tick`; in the remote-submit -/// model the operator is polling over HTTP and there is no -/// separate wait driver. Folding `advance` into the status read -/// makes the operator's poll the wait driver. Concurrent pollers -/// race-safely converge — the same property locked by the -/// `cli_coordinator_wait_two_concurrent_waits_converge` regression -/// test from the 0.5-S2f cleanup. -async fn handle_run_status( - State(state): State, - Path(run_id): Path, -) -> Response { - use boruna_orchestrator::workflow::{AdvanceRunStatus, WorkflowRunner}; - - let store = state.store.clone(); - let store_guard = match store.lock() { - Ok(g) => g, - Err(e) => return internal_error(&format!("store mutex poisoned: {e}")), - }; - - // Confirm the run exists before advancing — advance_run_one_tick - // returns an Internal error for unknown run ids; we want to - // produce the stable `coord.runs.not_found` taxonomy entry, so - // dispatch on existence first. - match store_guard.get_run(&run_id) { - Ok(Some(_)) => {} - Ok(None) => { - return respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new("coord.runs.not_found", format!("no run with id '{run_id}'")), - ); - } - Err(e) => { - return internal_error(&format!("read run: {e}")); - } - } - - // Drive the run forward. In the local-data-dir model - // `coordinator wait` was the thing that did this; in the - // remote-submit model the operator is polling over HTTP and - // there is no separate driver. Folding `advance` into the - // status read makes the operator's poll the wait driver. - // Concurrent pollers race-safely converge — same property - // locked by the `cli_coordinator_wait_two_concurrent_waits_converge` - // regression test from the 0.5-S2f cleanup. - let advance = match WorkflowRunner::advance_run_one_tick(&store_guard, &run_id) { - Ok(r) => r, - Err(e) => { - return internal_error(&format!("advance run: {e}")); - } - }; - let cps = match store_guard.list_step_checkpoints(&run_id) { - Ok(cs) => cs, - Err(e) => { - return internal_error(&format!("read checkpoints: {e}")); - } - }; - - let mut step_statuses = BTreeMap::new(); - let mut error_msg: Option = None; - for cp in &cps { - step_statuses.insert( - cp.step_id.clone(), - persist_status_str(cp.status).to_string(), - ); - if cp.status == StepStatus::Failed && error_msg.is_none() { - error_msg.clone_from(&cp.error_msg); - } - } - - // Surface the *computed* run status from advance_run_one_tick - // — the same value `coordinator wait` consults to exit. The - // run row's `status` column is operationally maintained by - // the in-process runner only; in distributed mode it doesn't - // transition. Mirroring the wait driver here keeps wire and - // local semantics aligned. - let status_str = match advance.run_status { - AdvanceRunStatus::Running => "running", - AdvanceRunStatus::Completed => "completed", - AdvanceRunStatus::Failed => "failed", - } - .to_string(); - - // Terminal: append closing WorkflowCompleted audit event - // idempotently — same posture as `coordinator wait`'s terminal - // exit paths. Without this, runs driven entirely through the - // remote API would have no closing audit chain entry. - if matches!( - advance.run_status, - AdvanceRunStatus::Completed | AdvanceRunStatus::Failed - ) { - if let Err(e) = WorkflowRunner::append_wait_terminal_audit_event(&store_guard, &run_id) { - eprintln!("warning: failed to append terminal audit event for '{run_id}': {e}"); - } - } - drop(store_guard); - - Json(RunStatusResponse { - protocol_version: PROTOCOL_VERSION, - run_id, - status: status_str, - step_statuses, - error_msg, - }) - .into_response() -} - -// ── operator-facing approve + trigger (sprint 0.5-S6) ── - -/// `POST /api/runs/{run_id}/approve` — record an approval-gate -/// decision (approved or rejected) for a paused step. Delegates to -/// `record_approval_decision_in_store`. Decision string is -/// lowercase `"approved"` / `"rejected"`. Auth: bearer-gated by -/// `auth_middleware` like all other operator routes. -async fn handle_approve_run( - State(state): State, - Path(run_id): Path, - Json(req): Json, -) -> Response { - use boruna_orchestrator::workflow::ApprovalKind; - - let kind = match req.decision.as_str() { - "approved" => ApprovalKind::Approved, - "rejected" => ApprovalKind::Rejected, - other => { - return respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new( - "coord.approve.bad_payload", - format!("decision must be \"approved\" or \"rejected\", got {other:?}"), - ), - ); - } - }; - let store = state.store.clone(); - let store_guard = match store.lock() { - Ok(g) => g, - Err(e) => return internal_error(&format!("store mutex poisoned: {e}")), - }; - // S9: require the per-gate token before recording a decision, so a holder of - // the bearer/worker credential cannot seize (or pre-empt) an approval gate. - // Constant-time compare against the token stashed at pause-time. When no token - // is stashed (unknown run, or the gate was never reached) fall through so - // record_approval_decision_in_store returns the precise 404/gate-state error - // rather than masking it as a token failure. - match boruna_orchestrator::workflow::approval_gate_token(&store_guard, &run_id, &req.step_id) { - Ok(Some(stashed)) => { - if !constant_time_bytes_eq(req.token.as_bytes(), stashed.as_bytes()) { - return respond_err( - StatusCode::FORBIDDEN, - ErrorBody::new( - "coord.approval_token_invalid", - "approval requires the per-gate token stashed at pause-time; \ - supplied token is missing or does not match" - .to_string(), - ), - ); - } - } - Ok(None) => {} - Err(e) => return internal_error(&format!("approval_gate_token: {e}")), - } - let result = boruna_orchestrator::workflow::record_approval_decision_in_store( - &store_guard, - &run_id, - &req.step_id, - kind, - req.reason.clone(), - ); - drop(store_guard); - match result { - Ok(()) => Json(OkResponse { - protocol_version: PROTOCOL_VERSION, - ok: true, - }) - .into_response(), - Err(e) => approve_error_response(e), - } -} - -/// `POST /api/runs/{run_id}/trigger` — record an external-trigger -/// payload for a paused step. Delegates to -/// `record_external_trigger_in_store`. Bearer-gated. -async fn handle_trigger_run( - State(state): State, - Path(run_id): Path, - Json(req): Json, -) -> Response { - let store = state.store.clone(); - let store_guard = match store.lock() { - Ok(g) => g, - Err(e) => return internal_error(&format!("store mutex poisoned: {e}")), - }; - let result = boruna_orchestrator::workflow::record_external_trigger_in_store( - &store_guard, - &run_id, - &req.step_id, - &req.token, - &req.payload, - ); - drop(store_guard); - match result { - Ok(()) => Json(OkResponse { - protocol_version: PROTOCOL_VERSION, - ok: true, - }) - .into_response(), - Err(e) => trigger_error_response(e), - } -} - -/// `GET /api/runs/{run_id}/blobs/{hash}` — return the bytes of a -/// large step output stored in the coordinator's blob store. Sprint -/// 0.5-S7. -/// -/// **Run-scoped:** the route only returns bytes if `hash` is referenced -/// by a step checkpoint under `run_id`. Even though hashes are -/// content-addressed and globally unique by collision resistance, this -/// scope makes the route's authorization story trivial — every run is -/// already gated by the bearer-token middleware, and access to -/// `run_id` implies access to its outputs. A future cross-run dedup -/// route would be a NEW endpoint with its own access-control story. -/// -/// Error_kind taxonomy (sprint 0.5-S7, locked): -/// - `coord.blobs.bad_hash` — 400 — `hash` is not 64 lowercase hex -/// characters. -/// - `coord.blobs.not_found` — 404 — no checkpoint under `run_id` -/// references this hash, OR the checkpoint references it but the -/// blob file is missing on disk. -/// - `coord.unauthorized` — 401 (handled upstream by `auth_middleware`). -async fn handle_get_blob( - State(state): State, - Path((run_id, hash)): Path<(String, String)>, -) -> Response { - // Validate hash format BEFORE any other check. A malformed hash is - // never a valid query — even on an unknown run — so 400 is the - // accurate signal for clients passing a bad URL. - if hash.len() != 64 - || !hash - .bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) - { - return respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new( - "coord.blobs.bad_hash", - "hash must be 64 lowercase hex characters", - ), - ); - } - - // Acquire store inside the async fn synchronously (the orchestrator's - // single-threaded SQLite connection lives inside Arc>). - let store_guard = match state.store.lock() { - Ok(g) => g, - Err(_) => return internal_error("store lock poisoned"), - }; - - // Run-scope check before any filesystem access. - let owned = match store_guard.run_owns_blob_ref(&run_id, &hash) { - Ok(b) => b, - Err(e) => return internal_error(&format!("run_owns_blob_ref: {e}")), - }; - if !owned { - // 404 covers both "no run", "no checkpoint", and "checkpoint - // does not reference this hash". Doesn't disambiguate to avoid - // exposing run existence to unauthorized-but-otherwise-valid- - // bearer callers. - return respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new( - "coord.blobs.not_found", - format!("no blob '{hash}' referenced by run '{run_id}'"), - ), - ); - } - - // Run owns the ref; resolve via blob store. - let blob_store = match store_guard.blob_store() { - Some(bs) => bs.clone(), - None => { - return internal_error("coordinator opened without a blob store (in-memory mode?)"); - } - }; - drop(store_guard); - - match blob_store.read_bytes(&hash) { - Ok(bytes) => ( - StatusCode::OK, - [(axum::http::header::CONTENT_TYPE, "application/octet-stream")], - bytes, - ) - .into_response(), - Err(BlobStoreError::BadHash) => respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new( - "coord.blobs.bad_hash", - "hash must be 64 lowercase hex characters", - ), - ), - Err(BlobStoreError::NotFound) => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new( - "coord.blobs.not_found", - format!( - "blob '{hash}' is referenced by a checkpoint under '{run_id}' \ - but is missing from the blob store on disk" - ), - ), - ), - Err(e) => internal_error(&format!("blob read: {e}")), - } -} - -/// Health/readiness probe response. Sprint W2. -/// -/// Non-sensitive content only — load balancers and external probes -/// receive this without bearer-auth (see `auth_middleware` bypass). -/// `boruna_version` is read from the workspace package version at -/// build time. `capability_set_hash` lets workers verify they're -/// addressing a coord with a compatible capability table before -/// committing to a registration. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct HealthResponse { - pub protocol_version: u32, - pub status: String, - pub boruna_version: &'static str, - pub capability_set_hash: String, - pub uptime_ms: i64, -} - -async fn handle_health(State(state): State) -> Response { - // Probe the store mutex to detect a poisoned lock — that's the - // only failure mode that wouldn't already surface as a TCP - // connect error. We don't run any SQL against the store: a - // crashed-process replacement coord still has a fresh - // connection, and forcing every probe through SQL would amplify - // load-balancer-driven query traffic. - if state.store.lock().is_err() { - return respond_err( - StatusCode::SERVICE_UNAVAILABLE, - ErrorBody::new( - "coord.unavailable", - "store mutex poisoned; coord is not ready", - ), - ); - } - let now_ms = now_unix_ms(); - let uptime_ms = now_ms.saturating_sub(state.start_time_ms); - let body = HealthResponse { - protocol_version: PROTOCOL_VERSION, - status: "ready".to_string(), - boruna_version: env!("CARGO_PKG_VERSION"), - capability_set_hash: state.capability_set_hash.clone(), - uptime_ms, - }; - (StatusCode::OK, Json(body)).into_response() -} - -/// Map a `WorkflowRunError` from the approve path to an HTTP -/// response with the locked `coord.approve.*` error_kind taxonomy. -/// Sprint 0.5-S6. -fn approve_error_response(e: boruna_orchestrator::workflow::WorkflowRunError) -> Response { - use boruna_orchestrator::workflow::WorkflowRunError; - match e { - WorkflowRunError::RunNotFound(id) => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new("coord.runs.not_found", format!("no run with id '{id}'")), - ), - WorkflowRunError::RunNotResumable { run_id, terminal_status } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.approve.invalid_state", - format!("run '{run_id}' is in terminal status '{terminal_status}'"), - ), - ), - WorkflowRunError::StepNotFound { run_id, step_id } => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new( - "coord.approve.invalid_state", - format!("step '{step_id}' not found in run '{run_id}'"), - ), - ), - WorkflowRunError::NotAnApprovalGateStep { run_id, step_id } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.approve.invalid_state", - format!("step '{step_id}' in run '{run_id}' is not an approval-gate step"), - ), - ), - WorkflowRunError::StepNotAtApprovalGate { run_id, step_id, current_status } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.approve.invalid_state", - format!( - "step '{step_id}' in run '{run_id}' is in '{current_status}', not 'awaiting_approval'" - ), - ), - ), - WorkflowRunError::StepAlreadyDecided { run_id, step_id, prior_decision } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.approve.invalid_state", - format!( - "step '{step_id}' in run '{run_id}' was already decided ({prior_decision})" - ), - ), - ), - other => internal_error(&format!("approve failed: {other}")), - } -} - -/// Map a `WorkflowRunError` from the trigger path to an HTTP -/// response with the locked `coord.trigger.*` error_kind taxonomy. -fn trigger_error_response(e: boruna_orchestrator::workflow::WorkflowRunError) -> Response { - use boruna_orchestrator::workflow::WorkflowRunError; - match e { - WorkflowRunError::RunNotFound(id) => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new("coord.runs.not_found", format!("no run with id '{id}'")), - ), - WorkflowRunError::RunNotResumable { run_id, terminal_status } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.trigger.invalid_state", - format!("run '{run_id}' is in terminal status '{terminal_status}'"), - ), - ), - WorkflowRunError::StepNotFound { run_id, step_id } => respond_err( - StatusCode::NOT_FOUND, - ErrorBody::new( - "coord.trigger.invalid_state", - format!("step '{step_id}' not found in run '{run_id}'"), - ), - ), - WorkflowRunError::NotAnExternalTriggerStep { run_id, step_id } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.trigger.invalid_state", - format!( - "step '{step_id}' in run '{run_id}' is not an external-trigger step" - ), - ), - ), - WorkflowRunError::StepNotAtExternalTriggerGate { run_id, step_id, current_status } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.trigger.invalid_state", - format!( - "step '{step_id}' in run '{run_id}' is in '{current_status}', not 'awaiting_external_event'" - ), - ), - ), - WorkflowRunError::InvalidTriggerToken { run_id, step_id } => respond_err( - StatusCode::UNAUTHORIZED, - ErrorBody::new( - "coord.trigger.bad_token", - format!( - "trigger token mismatch for step '{step_id}' in run '{run_id}'" - ), - ), - ), - WorkflowRunError::StepAlreadyTriggered { run_id, step_id, prior_triggered_at_ms } => respond_err( - StatusCode::CONFLICT, - ErrorBody::new( - "coord.trigger.invalid_state", - format!( - "step '{step_id}' in run '{run_id}' was already triggered at {prior_triggered_at_ms}" - ), - ), - ), - WorkflowRunError::Validation(msg) => respond_err( - StatusCode::BAD_REQUEST, - ErrorBody::new("coord.trigger.bad_payload", msg), - ), - other => internal_error(&format!("trigger failed: {other}")), - } -} - -// Helper accessors for tests / future dashboard merge. -#[allow(dead_code)] -impl CoordinatorState { - pub fn store_handle(&self) -> Arc> { - self.store.clone() - } - pub fn bind_warning(&self) -> Option<&str> { - self.config.bind_warning.as_deref() - } -} - -// ── workflow run --coordinator client (sprint 0.5-S4) ── - -/// Drive the operator-side flow for `boruna workflow run --coordinator`: -/// 1. Read `workflow_dir/workflow.json` + each Source-step's `.ax`. -/// 2. POST `/api/runs/submit` with the inlined payload. -/// 3. Poll `/api/runs/{run_id}/status` until terminal. -/// 4. Print step transitions to stdout (matching `coordinator wait`'s -/// line-per-transition format). -/// 5. Return an exit code: `0` Completed, `1` Failed, `2` Timeout. -/// -/// `coord_url` may end with or without a trailing slash; we normalize. -/// `coord_token` is sent as `Authorization: Bearer ` when -/// `Some`, omitted otherwise — operators running an unauthenticated -/// loopback coordinator can pass `None` (or omit the env var). -pub fn run_remote( - def: &boruna_orchestrator::workflow::definition::WorkflowDef, - workflow_dir: &std::path::Path, - policy: &boruna_vm::Policy, - coord_url: &str, - coord_token: Option<&str>, - poll_interval_ms: u64, - max_wait_secs: u64, -) -> Result> { - use boruna_orchestrator::workflow::definition::StepKind; - - // 1. Collect step sources from disk. - let mut step_sources: BTreeMap = BTreeMap::new(); - for (step_id, step_def) in &def.steps { - if let StepKind::Source { source } = &step_def.kind { - let path = workflow_dir.join(source); - let body = std::fs::read_to_string(&path).map_err(|e| { - format!( - "step '{step_id}' source '{}' read failed: {e}", - path.display() - ) - })?; - step_sources.insert(step_id.clone(), body); - } - } - - // 2. Build URLs. The submit URL is fixed; the status URL needs a - // placeholder for the run_id we don't yet know. - let base = coord_url.trim_end_matches('/').to_string(); - let submit_url = format!("{base}/api/runs/submit"); - - // 3. Synchronous Tokio runtime — `reqwest` here is the async - // client (the same one the worker uses) and we want a simple - // synchronous CLI surface. A short-lived current-thread - // runtime keeps memory + thread overhead minimal. - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| format!("failed to build tokio runtime: {e}"))?; - - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(60)) - .build() - .map_err(|e| format!("failed to build HTTP client: {e}"))?; - - let submit = SubmitRunRequest { - workflow: def.clone(), - step_sources, - policy: Some(policy.clone()), - }; - - eprintln!("workflow run --coordinator"); - eprintln!(" coordinator: {base}"); - eprintln!(" workflow: {}", def.name); - eprintln!(" poll-ms: {poll_interval_ms}"); - if max_wait_secs > 0 { - eprintln!(" max-wait-s: {max_wait_secs}"); - } - - // 4. Submit + poll under the runtime. - rt.block_on(async move { - let mut req = client.post(&submit_url).json(&submit); - if let Some(tok) = coord_token { - req = req.bearer_auth(tok); - } - let resp = req - .send() - .await - .map_err(|e| format!("submit failed: HTTP error: {e}"))?; - let status = resp.status(); - if !status.is_success() { - let body = resp.text().await.unwrap_or_default(); - return Err::>( - format!("submit failed: {status}: {body}").into(), - ); - } - let submit_resp: SubmitRunResponse = resp - .json() - .await - .map_err(|e| format!("submit response not parseable as JSON: {e}"))?; - let run_id = submit_resp.run_id; - eprintln!(" run_id: {run_id}"); - eprintln!(" workflow_hash: {}", submit_resp.workflow_hash); - - let status_url = format!("{base}/api/runs/{run_id}/status"); - let effective_poll_ms = poll_interval_ms.max(MIN_WAIT_POLL_INTERVAL_MS); - if poll_interval_ms < MIN_WAIT_POLL_INTERVAL_MS { - eprintln!( - "[WARNING] --coord-poll-interval-ms {poll_interval_ms} below minimum \ - {MIN_WAIT_POLL_INTERVAL_MS}; using {effective_poll_ms} ms" - ); - } - - let started = std::time::Instant::now(); - let mut prev: BTreeMap = BTreeMap::new(); - loop { - let mut req = client.get(&status_url); - if let Some(tok) = coord_token { - req = req.bearer_auth(tok); - } - let resp = req - .send() - .await - .map_err(|e| format!("status poll failed: {e}"))?; - if !resp.status().is_success() { - let s = resp.status(); - let body = resp.text().await.unwrap_or_default(); - return Err::>( - format!("status poll: {s}: {body}").into(), - ); - } - let snapshot: RunStatusResponse = resp - .json() - .await - .map_err(|e| format!("status response not parseable as JSON: {e}"))?; - - for (sid, sstatus) in &snapshot.step_statuses { - match prev.get(sid) { - Some(p) if p == sstatus => {} - _ => { - println!("step {sid}: {sstatus}"); - prev.insert(sid.clone(), sstatus.clone()); - } - } - } - - match snapshot.status.as_str() { - "completed" => { - println!("run {run_id}: completed"); - return Ok::>(0); - } - "failed" => { - if let Some(msg) = &snapshot.error_msg { - println!("run {run_id}: failed — {msg}"); - } else { - println!("run {run_id}: failed"); - } - return Ok(1); - } - _ => {} - } - - if max_wait_secs > 0 && started.elapsed().as_secs() >= max_wait_secs { - eprintln!( - "run {run_id}: exceeded --coord-max-wait-secs={max_wait_secs}; \ - remote run continues; CLI exiting with 2" - ); - return Ok(2); - } - - tokio::time::sleep(Duration::from_millis(effective_poll_ms)).await; - } - }) -} - -// ── workflow approve / reject / trigger client (sprint 0.5-S6) ── - -/// POST `/api/runs/{run_id}/approve` against a remote coordinator. -/// Used by `boruna workflow approve --coordinator ` and -/// `boruna workflow reject --coordinator `. Returns `Ok(())` on -/// success, an error with the coordinator's `error_kind` and -/// message verbatim on a non-2xx response. -#[allow(clippy::too_many_arguments)] -pub fn send_approve_remote( - coord_url: &str, - coord_token: Option<&str>, - run_id: &str, - step_id: &str, - decision: &str, - reason: Option<&str>, - token: &str, -) -> Result<(), Box> { - let base = coord_url.trim_end_matches('/'); - let url = format!("{base}/api/runs/{run_id}/approve"); - let body = ApproveRequest { - step_id: step_id.to_string(), - decision: decision.to_string(), - reason: reason.map(|s| s.to_string()), - token: token.to_string(), - }; - post_operator_command(&url, coord_token, &body) -} - -/// POST `/api/runs/{run_id}/trigger` against a remote coordinator. -/// Mirrors `send_approve_remote`'s shape; separate function only -/// because the body type differs. -pub fn send_trigger_remote( - coord_url: &str, - coord_token: Option<&str>, - run_id: &str, - step_id: &str, - trigger_token: &str, - payload: &str, -) -> Result<(), Box> { - let base = coord_url.trim_end_matches('/'); - let url = format!("{base}/api/runs/{run_id}/trigger"); - let body = TriggerRequest { - step_id: step_id.to_string(), - token: trigger_token.to_string(), - payload: payload.to_string(), - }; - post_operator_command(&url, coord_token, &body) -} - -/// Shared POST helper for the operator-side mutation routes. -/// Builds a tokio runtime, sends the request, surfaces non-2xx -/// responses with the coordinator's full error body so operators -/// get a clear `coord.*` error_kind without us re-parsing here. -fn post_operator_command( - url: &str, - coord_token: Option<&str>, - body: &T, -) -> Result<(), Box> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| format!("failed to build tokio runtime: {e}"))?; - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .build() - .map_err(|e| format!("failed to build HTTP client: {e}"))?; - rt.block_on(async move { - let mut req = client.post(url).json(body); - if let Some(tok) = coord_token { - req = req.bearer_auth(tok); - } - let resp = req.send().await.map_err(|e| format!("HTTP error: {e}"))?; - let status = resp.status(); - if status.is_success() { - return Ok::<(), Box>(()); - } - let body = resp.text().await.unwrap_or_default(); - Err::<(), Box>(format!("{status}: {body}").into()) - }) -} - -// ── coordinator wait (sprint 0.5-S2f) ── - -/// Minimum poll interval for the wait loop. Mirrors -/// `MIN_SWEEP_INTERVAL_MS` — sub-100 ms polling is allowed only as -/// a test/operator override, with a clamping warning. -const MIN_WAIT_POLL_INTERVAL_MS: u64 = 100; - -/// Drive a submit-only run to terminal status by computing -/// downstream-ready successors via -/// [`boruna_orchestrator::workflow::WorkflowRunner::advance_run_one_tick`] -/// every `poll_interval_ms`. Sprint `0.5-S2f`. -/// -/// Returns the intended process exit code: -/// - `0` — run reached `Completed`. -/// - `1` — run reached `Failed`. -/// - `2` — `--max-wait-secs` exceeded. -/// -/// This function is synchronous (no tokio runtime); the wait loop is -/// driven by `std::thread::sleep` between ticks. The advance call -/// itself is short-lived (a SQLite read + a few small writes). -pub fn run_wait( - data_dir: PathBuf, - run_id: String, - poll_interval_ms: u64, - max_wait_secs: u64, -) -> Result> { - use boruna_orchestrator::workflow::{AdvanceRunStatus, WorkflowRunner}; - - let db_path = data_dir.join("runs.db"); - if !db_path.exists() { - return Err(format!( - "no runs.db at {} — pass --data-dir matching the coordinator process", - db_path.display() - ) - .into()); - } - let store = RunCheckpointStore::open(&db_path) - .map_err(|e| format!("failed to open {}: {e}", db_path.display()))?; - - let effective_poll_ms = poll_interval_ms.max(MIN_WAIT_POLL_INTERVAL_MS); - if poll_interval_ms < MIN_WAIT_POLL_INTERVAL_MS { - eprintln!( - "[WARNING] --poll-interval-ms {poll_interval_ms} below minimum \ - {MIN_WAIT_POLL_INTERVAL_MS}; using {effective_poll_ms} ms" - ); - } - - eprintln!("coordinator wait run_id={run_id}"); - eprintln!(" data-dir: {}", data_dir.display()); - eprintln!(" poll-interval-ms: {effective_poll_ms}"); - if max_wait_secs > 0 { - eprintln!(" max-wait-secs: {max_wait_secs}"); - } - - // Track previous step statuses so we only print transitions, not - // the entire status map every tick. - let mut prev: BTreeMap = BTreeMap::new(); - let started = std::time::Instant::now(); - - loop { - let result = match WorkflowRunner::advance_run_one_tick(&store, &run_id) { - Ok(r) => r, - Err(e) => { - eprintln!("error: {e}"); - return Ok(2); - } - }; - // Print explicit "requeued" lines (sprint 0.5-S5) BEFORE the - // generic transition loop so operators see "step s1: requeued - // (retry)" instead of just "step s1: pending" when a Failed - // step transitions back to Pending via the retry policy. - for sid in &result.newly_requeued { - println!("step {sid}: requeued (retry)"); - prev.insert(sid.clone(), "pending".into()); - } - // Print step transitions in sorted order (BTreeMap iteration). - // newly_pending is always reflected in all_step_statuses (see - // advance_run_one_tick: status_map is updated alongside the - // newly_pending push), so this single loop covers all - // transitions. - for (sid, status) in &result.all_step_statuses { - let status_str = persist_status_str(*status).to_string(); - match prev.get(sid) { - Some(p) if p == &status_str => {} - _ => { - println!("step {sid}: {status_str}"); - prev.insert(sid.clone(), status_str); - } - } - } - match result.run_status { - AdvanceRunStatus::Completed => { - // Sprint follow-up to 0.5-S2f: emit a terminating - // WorkflowCompleted audit event so the chain has - // a closing entry to match its WorkflowStarted - // genesis. Idempotent on re-invocation. - if let Err(e) = WorkflowRunner::append_wait_terminal_audit_event(&store, &run_id) { - eprintln!( - "warning: failed to append WorkflowCompleted audit event for run \ - '{run_id}': {e}" - ); - } - println!("run {run_id}: completed"); - return Ok(0); - } - AdvanceRunStatus::Failed => { - if let Err(e) = WorkflowRunner::append_wait_terminal_audit_event(&store, &run_id) { - eprintln!( - "warning: failed to append WorkflowCompleted audit event for run \ - '{run_id}': {e}" - ); - } - println!("run {run_id}: failed"); - return Ok(1); - } - AdvanceRunStatus::Running => {} - } - if max_wait_secs > 0 && started.elapsed().as_secs() >= max_wait_secs { - eprintln!("error: --max-wait-secs {max_wait_secs} exceeded"); - return Ok(3); - } - std::thread::sleep(std::time::Duration::from_millis(effective_poll_ms)); - } -} - -fn persist_status_str(s: boruna_orchestrator::persistence::StepStatus) -> &'static str { - use boruna_orchestrator::persistence::StepStatus; - match s { - StepStatus::Pending => "pending", - StepStatus::Running => "running", - StepStatus::Completed => "completed", - StepStatus::Failed => "failed", - StepStatus::AwaitingApproval => "awaiting_approval", - StepStatus::AwaitingExternalEvent => "awaiting_external_event", - } -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::body::Body; - use axum::http::Request; - use boruna_orchestrator::persistence::{RunRow, StepCheckpoint}; - use tower::ServiceExt; - - fn fresh_state() -> CoordinatorState { - let store = RunCheckpointStore::open_in_memory().unwrap(); - let capability_set_hash = compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - CoordinatorState { - store: Arc::new(Mutex::new(store)), - workers: Arc::new(Mutex::new(HashMap::new())), - workflow_dirs: Arc::new(Mutex::new(HashMap::new())), - capability_set_hash, - config: CoordinatorConfig { - max_lease_ttl_ms: 600_000, - poll_timeout_ms: 200, - bind_warning: None, - shared_secret: None, - mtls_required: false, - }, - start_time_ms: 0, - } - } - - fn pending_step(state: &CoordinatorState, run_id: &str, step_id: &str, source: &str) { - let metadata_json = serde_json::json!({ - "step_sources": { step_id: source } - }) - .to_string(); - let store = state.store.lock().unwrap(); - let _ = store.insert_run(&RunRow { - run_id: run_id.into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Running, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: r#"{"default_allow":true}"#.into(), - metadata_json, - }); - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: run_id.into(), - step_id: step_id.into(), - status: StepStatus::Pending, - output_json: None, - output_hash: None, - started_at_ms: None, - ended_at_ms: None, - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); - } - - async fn post_json( - app: &Router, - path: &str, - body: &T, - ) -> (StatusCode, serde_json::Value) { - let req = Request::builder() - .method("POST") - .uri(path) - .header("content-type", "application/json") - .body(Body::from(serde_json::to_vec(body).unwrap())) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - let status = resp.status(); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let v: serde_json::Value = - serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); - (status, v) - } - - fn register_payload(hash: String) -> RegisterRequest { - RegisterRequest { - worker_id: None, - capability_set_hash: hash, - advertised_capabilities: None, - } - } - - #[tokio::test] - async fn register_allocates_worker_id() { - let state = fresh_state(); - let app = build_router(state.clone()); - let (status, v) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - assert_eq!(status, StatusCode::OK); - assert!(v["worker_id"].as_str().unwrap().starts_with("wkr-")); - assert!(v["session_token"].as_str().unwrap().starts_with("sess-")); - assert_eq!(v["protocol_version"], 1); - } - - #[tokio::test] - async fn register_accepts_caller_supplied_worker_id() { - let state = fresh_state(); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: Some("custom-host-7".into()), - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: None, - }; - let (status, v) = post_json(&app, "/api/workers/register", &req).await; - assert_eq!(status, StatusCode::OK); - assert_eq!(v["worker_id"], "custom-host-7"); - } - - #[tokio::test] - async fn register_rejects_binary_mismatch() { - let state = fresh_state(); - let app = build_router(state); - let (status, v) = post_json( - &app, - "/api/workers/register", - ®ister_payload("sha256:bogus".into()), - ) - .await; - assert_eq!(status, StatusCode::CONFLICT); - assert_eq!(v["error_kind"], "coord.binary_mismatch"); - assert!(v["expected_hash"].is_string()); - assert_eq!(v["protocol_version"], 1); - } - - /// Sprint `W3-A` — insert a Pending step with a specific - /// policy_json. Used by capability-tagging tests that need - /// to control the policy's rules block precisely. - fn pending_step_with_policy( - state: &CoordinatorState, - run_id: &str, - step_id: &str, - source: &str, - policy_json: &str, - ) { - let metadata_json = serde_json::json!({ - "step_sources": { step_id: source } - }) - .to_string(); - let store = state.store.lock().unwrap(); - let _ = store.insert_run(&RunRow { - run_id: run_id.into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Running, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: policy_json.into(), - metadata_json, - }); - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: run_id.into(), - step_id: step_id.into(), - status: StepStatus::Pending, - output_json: None, - output_hash: None, - started_at_ms: None, - ended_at_ms: None, - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); - } - - /// Build a JSON policy whose `rules` block lists the given - /// capability names with `allow: true`. `default_allow` is - /// `false` so the placement filter sees ONLY the listed caps. - fn policy_allowing(caps: &[&str]) -> String { - let rules: serde_json::Map = caps - .iter() - .map(|c| { - ( - c.to_string(), - serde_json::json!({ "allow": true, "budget": 0 }), - ) - }) - .collect(); - serde_json::json!({ - "schema_version": 1, - "default_allow": false, - "rules": serde_json::Value::Object(rules), - }) - .to_string() - } - - #[tokio::test] - async fn register_with_subset_advertised_caps_succeeds() { - let state = fresh_state(); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: Some(vec!["net.fetch".into(), "db.query".into()]), - }; - let (status, v) = post_json(&app, "/api/workers/register", &req).await; - assert_eq!(status, StatusCode::OK); - assert!(v["worker_id"].as_str().unwrap().starts_with("wkr-")); - } - - #[tokio::test] - async fn register_rejects_unknown_capability_name() { - let state = fresh_state(); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: Some(vec!["net.fetch".into(), "bogus.cap".into()]), - }; - let (status, v) = post_json(&app, "/api/workers/register", &req).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(v["error_kind"], "coord.unknown_capability"); - assert_eq!(v["protocol_version"], 1); - } - - #[tokio::test] - async fn claim_returns_only_steps_within_advertised_caps() { - let state = fresh_state(); - // Step requires only net.fetch; worker advertises net.fetch. - pending_step_with_policy( - &state, - "run-net", - "fetch-step", - "fn main() -> Int { 1 }\n", - &policy_allowing(&["net.fetch"]), - ); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: Some(vec!["net.fetch".into()]), - }; - let (_, reg) = post_json(&app, "/api/workers/register", &req).await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let claim_req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(claim_req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let item: WorkItem = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(item.step_id, "fetch-step"); - } - - #[tokio::test] - async fn claim_skips_step_requiring_unadvertised_capability() { - let state = fresh_state(); - // Step requires db.query; worker advertises only net.fetch. - pending_step_with_policy( - &state, - "run-db", - "db-step", - "fn main() -> Int { 1 }\n", - &policy_allowing(&["db.query"]), - ); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: Some(vec!["net.fetch".into()]), - }; - let (_, reg) = post_json(&app, "/api/workers/register", &req).await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let claim_req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(claim_req).await.unwrap(); - // Worker can't claim — long-poll times out → 204. - assert_eq!(resp.status(), StatusCode::NO_CONTENT); - } - - #[tokio::test] - async fn legacy_string_advertisement_normalizes_to_versioned() { - // post1-T-1.3 — pre-1.x worker sends Vec; coord - // normalizes each to Versioned { name, version: } at parse time. The - // worker is then eligible for any step whose required - // capabilities resolve to the same version. - let state = fresh_state(); - pending_step_with_policy( - &state, - "run-legacy", - "fetch-step", - "fn main() -> Int { 1 }\n", - &policy_allowing(&["net.fetch"]), - ); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: Some(vec![CapabilityAdvertisement::Legacy( - "net.fetch".into(), - )]), - }; - let (_, reg) = post_json(&app, "/api/workers/register", &req).await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let claim_req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(claim_req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let item: WorkItem = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(item.step_id, "fetch-step"); - } - - #[tokio::test] - async fn versioned_advertisement_at_correct_version_can_claim() { - // post1-T-1.3 — explicit Versioned advertisement at the - // coord's current version of `net.fetch` (always "1" in 1.x) - // routes correctly, demonstrating the wire shape works. - let state = fresh_state(); - pending_step_with_policy( - &state, - "run-versioned", - "fetch-step", - "fn main() -> Int { 1 }\n", - &policy_allowing(&["net.fetch"]), - ); - let app = build_router(state.clone()); - let coord_version = boruna_bytecode::Capability::NetFetch.version(); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: Some(vec![CapabilityAdvertisement::Versioned { - name: "net.fetch".into(), - version: coord_version.into(), - }]), - }; - let (_, reg) = post_json(&app, "/api/workers/register", &req).await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let claim_req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(claim_req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - } - - #[tokio::test] - async fn versioned_advertisement_at_wrong_version_returns_mismatch() { - // post1-T-1.3 — worker advertises net.fetch@OLD, coord's - // current version differs. Claim returns 409 + - // coord.capability_version_mismatch so the operator can - // ship a worker build with the matching version. - let state = fresh_state(); - pending_step_with_policy( - &state, - "run-mismatch", - "fetch-step", - "fn main() -> Int { 1 }\n", - &policy_allowing(&["net.fetch"]), - ); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: Some(vec![CapabilityAdvertisement::Versioned { - name: "net.fetch".into(), - version: "0".into(), // older than coord's "1" - }]), - }; - let (_, reg) = post_json(&app, "/api/workers/register", &req).await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let claim_req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(claim_req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::CONFLICT); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(body["error_kind"], "coord.capability_version_mismatch"); - } - - #[tokio::test] - async fn worker_without_advertised_caps_sees_all_steps() { - let state = fresh_state(); - // Step requires db.query; worker did NOT advertise (None) → - // backwards-compat full-fleet behavior: still claimable. - pending_step_with_policy( - &state, - "run-bc", - "db-step", - "fn main() -> Int { 1 }\n", - &policy_allowing(&["db.query"]), - ); - let app = build_router(state.clone()); - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let claim_req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(claim_req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let item: WorkItem = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(item.step_id, "db-step"); - } - - /// Sprint `W3-A` adversarial test (project §29): two pending - /// steps in the same fleet — one needing a cap the worker - /// DOESN'T advertise, one whose required caps are a subset of - /// what the worker advertises. The unadvertised step must be - /// skipped; the compatible sibling must still be claimable. - #[tokio::test] - async fn claim_skips_incompatible_step_but_claims_compatible_sibling() { - let state = fresh_state(); - // run-mix has TWO steps, one needing fs.write, one needing - // net.fetch. Same policy applies run-wide; we model the - // mix at the fleet level by inserting two runs. - pending_step_with_policy( - &state, - "run-fs", - "fs-step", - "fn main() -> Int { 1 }\n", - &policy_allowing(&["fs.write"]), - ); - pending_step_with_policy( - &state, - "run-net", - "net-step", - "fn main() -> Int { 2 }\n", - &policy_allowing(&["net.fetch"]), - ); - let app = build_router(state.clone()); - let req = RegisterRequest { - worker_id: None, - capability_set_hash: state.capability_set_hash.clone(), - // Worker advertises net.fetch only. - advertised_capabilities: Some(vec!["net.fetch".into()]), - }; - let (_, reg) = post_json(&app, "/api/workers/register", &req).await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let claim_req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(claim_req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let item: WorkItem = serde_json::from_slice(&bytes).unwrap(); - // The fs-step must NEVER be claimed by this worker. - assert_eq!(item.step_id, "net-step"); - } - - #[tokio::test] - async fn heartbeat_unknown_worker_returns_404() { - let state = fresh_state(); - let app = build_router(state); - let (status, v) = post_json( - &app, - "/api/workers/heartbeat", - &HeartbeatRequest { - worker_id: "ghost".into(), - session_token: "x".into(), - }, - ) - .await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(v["error_kind"], "coord.unknown_worker"); - } - - #[tokio::test] - async fn claim_returns_204_when_no_pending_work() { - let state = fresh_state(); - let app = build_router(state.clone()); - // Register first. - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::NO_CONTENT); - } - - #[tokio::test] - async fn claim_returns_work_item_when_pending_step_exists() { - let state = fresh_state(); - pending_step(&state, "run-1", "extract", "fn main() -> Int { 42 }\n"); - let app = build_router(state.clone()); - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let item: WorkItem = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(item.run_id, "run-1"); - assert_eq!(item.step_id, "extract"); - assert_eq!(item.claim_id, 1); - assert!(item.source.contains("42")); - } - - #[tokio::test] - async fn complete_with_stale_claim_returns_409() { - let state = fresh_state(); - pending_step(&state, "run-c", "s1", "fn main() -> Int { 1 }\n"); - let app = build_router(state.clone()); - // Register a worker up front so the SAME worker holds both claims — - // otherwise the S6 ownership guard (reject_if_not_claim_owner) fires - // first. Here the same worker's lease expires and it reclaims (new - // claim_id), then submits a stale complete → lease-expiry CAS → 409. - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let wid = reg["worker_id"].as_str().unwrap().to_string(); - { - let store = state.store.lock().unwrap(); - store - .claim_step("run-c", "s1", &wid, 5_000_000_000_000, 0) - .unwrap(); - store.expire_leases_and_requeue(5_000_000_000_001).unwrap(); - store - .claim_step("run-c", "s1", &wid, 5_000_000_000_002, 0) - .unwrap(); - } - // Late completion with stale claim_id=1 (current is 2). - let (status, v) = post_json( - &app, - "/api/work/complete", - &CompleteRequest { - worker_id: wid, - session_token: reg["session_token"].as_str().unwrap().into(), - run_id: "run-c".into(), - step_id: "s1".into(), - claim_id: 1, - output_json: r#""nope""#.into(), - output_hash: worker_output_hash(r#""nope""#), - attempt_count: 1, - }, - ) - .await; - assert_eq!(status, StatusCode::CONFLICT); - assert_eq!(v["error_kind"], "coord.lease_expired"); - assert_eq!(v["current_claim_id"], 2); - } - - #[tokio::test] - async fn complete_by_non_owner_returns_403() { - // S6: a registered worker that does NOT hold the claim cannot complete - // another worker's in-flight step, even with the correct claim_id. - let state = fresh_state(); - pending_step(&state, "run-own", "s1", "fn main() -> Int { 1 }\n"); - { - let store = state.store.lock().unwrap(); - store - .claim_step("run-own", "s1", "victim-worker", 5_000_000_000_000, 0) - .unwrap(); - } - let app = build_router(state.clone()); - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let (status, v) = post_json( - &app, - "/api/work/complete", - &CompleteRequest { - worker_id: reg["worker_id"].as_str().unwrap().into(), // NOT victim-worker - session_token: reg["session_token"].as_str().unwrap().into(), - run_id: "run-own".into(), - step_id: "s1".into(), - claim_id: 1, // correct, predictable claim_id - output_json: r#""forged""#.into(), - output_hash: worker_output_hash(r#""forged""#), - attempt_count: 1, - }, - ) - .await; - assert_eq!(status, StatusCode::FORBIDDEN); - assert_eq!(v["error_kind"], "coord.claim_not_owned"); - } - - #[tokio::test] - async fn complete_step_not_found_returns_404() { - let state = fresh_state(); - let app = build_router(state.clone()); - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let (status, v) = post_json( - &app, - "/api/work/complete", - &CompleteRequest { - worker_id: reg["worker_id"].as_str().unwrap().into(), - session_token: reg["session_token"].as_str().unwrap().into(), - run_id: "ghost".into(), - step_id: "ghost".into(), - claim_id: 1, - output_json: "0".into(), - output_hash: worker_output_hash("0"), - attempt_count: 1, - }, - ) - .await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(v["error_kind"], "coord.step_not_found"); - } - - #[tokio::test] - async fn complete_rejects_output_hash_mismatch() { - // Content-addressing forgery: a worker reporting an output_hash that does - // not match SHA-256(output_json) is rejected before touching the store. - let state = fresh_state(); - pending_step(&state, "run-h", "s1", "fn main() -> Int { 1 }\n"); - let app = build_router(state.clone()); - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let (status, v) = post_json( - &app, - "/api/work/complete", - &CompleteRequest { - worker_id: reg["worker_id"].as_str().unwrap().into(), - session_token: reg["session_token"].as_str().unwrap().into(), - run_id: "run-h".into(), - step_id: "s1".into(), - claim_id: 1, - output_json: r#""real-output""#.into(), - output_hash: "sha256:deadbeef".into(), // lies about the bytes - attempt_count: 1, - }, - ) - .await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(v["error_kind"], "coord.output_hash_mismatch"); - } - - #[tokio::test] - async fn extend_lease_rejects_below_minimum() { - // Adversarial-review F4 regression: extend_by_ms below - // the 1s floor returns 400 + coord.invalid_request. - let state = fresh_state(); - pending_step(&state, "run-e0", "s1", "fn main() -> Int { 1 }\n"); - let app = build_router(state.clone()); - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - // Claim first to set up a valid lease. - let req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let item: WorkItem = serde_json::from_slice(&bytes).unwrap(); - // Try to extend by 0 ms — should be rejected. - let (status, v) = post_json( - &app, - "/api/work/extend-lease", - &ExtendLeaseRequest { - worker_id: worker_id.into(), - session_token: token.into(), - run_id: item.run_id.clone(), - step_id: item.step_id.clone(), - claim_id: item.claim_id, - extend_by_ms: 0, - }, - ) - .await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(v["error_kind"], "coord.invalid_request"); - assert!(v["message"].as_str().unwrap().contains("minimum")); - } - - #[tokio::test] - async fn extend_lease_caps_at_max_lease_ttl_ms() { - let state = fresh_state(); - pending_step(&state, "run-e", "s1", "fn main() -> Int { 1 }\n"); - let app = build_router(state.clone()); - let (_, reg) = post_json( - &app, - "/api/workers/register", - ®ister_payload(state.capability_set_hash.clone()), - ) - .await; - // Claim via the http route to set up the lease. - let worker_id = reg["worker_id"].as_str().unwrap(); - let token = reg["session_token"].as_str().unwrap(); - let req = Request::builder() - .method("GET") - .uri(format!( - "/api/work/claim?worker_id={worker_id}&session_token={token}&lease_ttl_ms=10000" - )) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let item: WorkItem = serde_json::from_slice(&bytes).unwrap(); - - // Ask for more than max_lease_ttl_ms (600_000 in fresh_state). - let now_before = now_unix_ms(); - let (status, v) = post_json( - &app, - "/api/work/extend-lease", - &ExtendLeaseRequest { - worker_id: worker_id.into(), - session_token: token.into(), - run_id: item.run_id.clone(), - step_id: item.step_id.clone(), - claim_id: item.claim_id, - extend_by_ms: 9_999_999_999, - }, - ) - .await; - let now_after = now_unix_ms(); - assert_eq!(status, StatusCode::OK); - let new_deadline = v["new_lease_expires_at_ms"].as_i64().unwrap(); - // The handler caps `requested` at `now_handler + max_lease_ttl_ms`. - // `now_handler` falls between `now_before` and `now_after`, so the - // deadline must be in `[now_before + 600_000, now_after + 600_000]`. - // Using `now_after` as the upper bound eliminates the timing - // dependency that flaked under parallel CI load (sprint W10). - assert!( - new_deadline >= now_before + 600_000, - "deadline {new_deadline} below lower bound {}", - now_before + 600_000 - ); - assert!( - new_deadline <= now_after + 600_000, - "deadline {new_deadline} above upper bound {}", - now_after + 600_000 - ); - } - - // ── Sprint 0.5-S4 — submit + status handler tests ── - - fn make_submit_workflow() -> serde_json::Value { - // Minimal one-step Source workflow. Inline source compiles - // to `42`. The handler doesn't run the workflow, only - // registers it; compilability of the source is the - // worker's concern. - serde_json::json!({ - "schema_version": 1, - "name": "wf-s4", - "version": "1.0.0", - "steps": { - "s1": { - "kind": "source", - "source": "s1.ax" - } - }, - "edges": [] - }) - } - - #[tokio::test] - async fn submit_run_inserts_run_and_returns_run_id() { - let state = fresh_state(); - let app = build_router(state.clone()); - let body = serde_json::json!({ - "workflow": make_submit_workflow(), - "step_sources": { "s1": "fn main() -> Int { 42 }" } - }); - let (status, v) = post_json(&app, "/api/runs/submit", &body).await; - assert_eq!(status, StatusCode::OK, "body: {v}"); - let run_id = v["run_id"].as_str().expect("run_id present").to_string(); - assert_eq!(run_id.len(), 16, "run_id is 16-hex deterministic"); - assert_eq!(v["protocol_version"], 1); - assert!(!v["workflow_hash"].as_str().unwrap().is_empty()); - - // Run row is now in the store with the initial Pending checkpoint. - let store = state.store.lock().unwrap(); - let row = store.get_run(&run_id).unwrap().expect("run inserted"); - assert_eq!(row.workflow_name, "wf-s4"); - let cps = store.list_step_checkpoints(&run_id).unwrap(); - assert_eq!(cps.len(), 1); - assert_eq!(cps[0].step_id, "s1"); - assert_eq!(cps[0].status, StepStatus::Pending); - } - - #[tokio::test] - async fn submit_run_rejects_workflow_with_missing_step_source() { - let state = fresh_state(); - let app = build_router(state.clone()); - let body = serde_json::json!({ - "workflow": make_submit_workflow(), - "step_sources": {} // s1 source missing - }); - let (status, v) = post_json(&app, "/api/runs/submit", &body).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(v["error_kind"], "coord.submit.invalid_workflow"); - assert!( - v["message"] - .as_str() - .unwrap() - .contains("missing inline source"), - "expected missing-source error, got: {v}" - ); - } - - #[tokio::test] - async fn submit_run_rejects_oversized_step_source() { - let state = fresh_state(); - let app = build_router(state.clone()); - // 257 KiB — over the 256 KiB per-step cap. - let big = "x".repeat(257 * 1024); - let body = serde_json::json!({ - "workflow": make_submit_workflow(), - "step_sources": { "s1": big } - }); - let (status, v) = post_json(&app, "/api/runs/submit", &body).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(v["error_kind"], "coord.submit.invalid_workflow"); - assert!( - v["message"] - .as_str() - .unwrap() - .contains("exceeds per-step cap"), - "expected size-cap error, got: {v}" - ); - } - - #[tokio::test] - async fn run_status_returns_404_for_unknown_run_id() { - let state = fresh_state(); - let app = build_router(state.clone()); - let req = Request::builder() - .method("GET") - .uri("/api/runs/nope1234nope1234/status") - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.runs.not_found"); - } - - #[tokio::test] - async fn run_status_reflects_submitted_run() { - // Submit a run, then GET its status; the response must - // mirror the row + initial Pending checkpoint. - let state = fresh_state(); - let app = build_router(state.clone()); - let body = serde_json::json!({ - "workflow": make_submit_workflow(), - "step_sources": { "s1": "fn main() -> Int { 42 }" } - }); - let (_, v) = post_json(&app, "/api/runs/submit", &body).await; - let run_id = v["run_id"].as_str().unwrap().to_string(); - - let req = Request::builder() - .method("GET") - .uri(format!("/api/runs/{run_id}/status")) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let snap: RunStatusResponse = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(snap.run_id, run_id); - assert_eq!(snap.status, "running"); - assert_eq!(snap.step_statuses.get("s1").unwrap(), "pending"); - assert!(snap.error_msg.is_none()); - } - - // ── Sprint 0.5-S6 — approve + trigger handler tests ── - - fn make_approval_workflow() -> serde_json::Value { - // Two-step workflow: a Source step "analyze" feeds an - // ApprovalGate "human_review". Submit drives `analyze` to - // Pending; we manually mark it Completed in tests, then a - // tick opens the gate, then approve/reject closes it. - serde_json::json!({ - "schema_version": 1, - "name": "wf-s6-approve", - "version": "1.0.0", - "steps": { - "analyze": { - "kind": "source", - "source": "analyze.ax" - }, - "human_review": { - "kind": "approval_gate", - "required_role": "reviewer", - "depends_on": ["analyze"] - } - }, - "edges": [["analyze", "human_review"]] - }) - } - - async fn submit_with_open_gate(state: &CoordinatorState) -> String { - // Helper for the approve handler tests: submit, drive analyze - // Completed, tick to open the gate, return run_id. - let app = build_router(state.clone()); - let body = serde_json::json!({ - "workflow": make_approval_workflow(), - "step_sources": { "analyze": "fn main() -> Int { 1 }" } - }); - let (status, v) = post_json(&app, "/api/runs/submit", &body).await; - assert_eq!(status, StatusCode::OK, "submit failed: {v}"); - let run_id = v["run_id"].as_str().unwrap().to_string(); - - { - let store = state.store.lock().unwrap(); - let claim = store - .claim_step(&run_id, "analyze", "w", 1_000_000_000, 0) - .unwrap(); - let claim_id = match claim { - boruna_orchestrator::persistence::ClaimOutcome::Claimed { claim_id } => claim_id, - other => panic!("{other:?}"), - }; - store - .complete_step_cas(&run_id, "analyze", claim_id, "{}", "0", 1, 1) - .unwrap(); - } - - // Tick via the status endpoint (which calls advance) so the - // gate opens. - let req = Request::builder() - .method("GET") - .uri(format!("/api/runs/{run_id}/status")) - .body(Body::empty()) - .unwrap(); - app.clone().oneshot(req).await.unwrap(); - run_id - } - - #[tokio::test] - async fn approve_run_advances_gate_to_completed() { - let state = fresh_state(); - let run_id = submit_with_open_gate(&state).await; - let app = build_router(state.clone()); - - // S9: fetch the per-gate token stashed at pause-time and present it. - let token = { - let store = state.store.lock().unwrap(); - boruna_orchestrator::workflow::approval_gate_token(&store, &run_id, "human_review") - .unwrap() - .expect("open approval gate should have a stashed token") - }; - let body = serde_json::json!({ - "step_id": "human_review", - "decision": "approved", - "token": token - }); - let (status, v) = post_json(&app, &format!("/api/runs/{run_id}/approve"), &body).await; - assert_eq!(status, StatusCode::OK, "approve failed: {v}"); - assert_eq!(v["ok"], true); - - // Next status read advances the run; the gate is now Completed. - let req = Request::builder() - .method("GET") - .uri(format!("/api/runs/{run_id}/status")) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let snap: RunStatusResponse = serde_json::from_slice(&bytes).unwrap(); - assert_eq!( - snap.step_statuses.get("human_review").map(|s| s.as_str()), - Some("completed"), - "gate should be Completed after approve, got: {snap:?}" - ); - // The run is itself Completed because all steps reached terminal. - assert_eq!(snap.status, "completed"); - } - - #[tokio::test] - async fn approve_run_rejects_wrong_token() { - // S9: an open gate cannot be seized without the stashed per-gate token, - // even with a valid coordinator credential and correct step_id/decision. - let state = fresh_state(); - let run_id = submit_with_open_gate(&state).await; - let app = build_router(state.clone()); - let body = serde_json::json!({ - "step_id": "human_review", - "decision": "approved", - "token": "not-the-real-token" - }); - let (status, v) = post_json(&app, &format!("/api/runs/{run_id}/approve"), &body).await; - assert_eq!(status, StatusCode::FORBIDDEN); - assert_eq!(v["error_kind"], "coord.approval_token_invalid"); - // And a token-less body is likewise rejected. - let body2 = serde_json::json!({ "step_id": "human_review", "decision": "approved" }); - let (status2, v2) = post_json(&app, &format!("/api/runs/{run_id}/approve"), &body2).await; - assert_eq!(status2, StatusCode::FORBIDDEN); - assert_eq!(v2["error_kind"], "coord.approval_token_invalid"); - } - - #[tokio::test] - async fn approve_run_rejects_invalid_decision_string() { - let state = fresh_state(); - let run_id = submit_with_open_gate(&state).await; - let app = build_router(state.clone()); - - let body = serde_json::json!({ - "step_id": "human_review", - "decision": "maybe" - }); - let (status, v) = post_json(&app, &format!("/api/runs/{run_id}/approve"), &body).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(v["error_kind"], "coord.approve.bad_payload"); - } - - #[tokio::test] - async fn approve_run_returns_404_for_unknown_run_id() { - let state = fresh_state(); - let app = build_router(state); - let body = serde_json::json!({ - "step_id": "x", - "decision": "approved" - }); - let (status, v) = post_json(&app, "/api/runs/deadbeef0badcafe/approve", &body).await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(v["error_kind"], "coord.runs.not_found"); - } - - #[tokio::test] - async fn approve_run_rejects_double_decision() { - let state = fresh_state(); - let run_id = submit_with_open_gate(&state).await; - let app = build_router(state.clone()); - let token = { - let store = state.store.lock().unwrap(); - boruna_orchestrator::workflow::approval_gate_token(&store, &run_id, "human_review") - .unwrap() - .expect("open approval gate should have a stashed token") - }; - let body = serde_json::json!({ - "step_id": "human_review", - "decision": "approved", - "token": token - }); - let (s1, _) = post_json(&app, &format!("/api/runs/{run_id}/approve"), &body).await; - assert_eq!(s1, StatusCode::OK); - // Second approval on the same step must be rejected. - let (s2, v2) = post_json(&app, &format!("/api/runs/{run_id}/approve"), &body).await; - assert_eq!(s2, StatusCode::CONFLICT); - assert_eq!(v2["error_kind"], "coord.approve.invalid_state"); - } - - #[tokio::test] - async fn approve_run_rejects_unauthenticated_when_secret_configured() { - // Symmetric to the submit-run auth check: the approve route - // must inherit the bearer-gating from auth_middleware. - let mut state = fresh_state(); - state.config.shared_secret = Some("super-secret".into()); - let app = build_router(state); - let body = serde_json::json!({ - "step_id": "x", - "decision": "approved" - }); - let (status, v) = post_json(&app, "/api/runs/abcd0123abcd0123/approve", &body).await; - assert_eq!(status, StatusCode::UNAUTHORIZED); - assert_eq!(v["error_kind"], "coord.unauthorized"); - } - - #[tokio::test] - async fn submit_run_rejects_unauthenticated_when_secret_configured() { - // Auth middleware (sprint 0.5-S3) must guard the new - // submit endpoint just like the worker endpoints. - let mut state = fresh_state(); - state.config.shared_secret = Some("super-secret".into()); - let app = build_router(state.clone()); - let body = serde_json::json!({ - "workflow": make_submit_workflow(), - "step_sources": { "s1": "fn main() -> Int { 42 }" } - }); - let (status, v) = post_json(&app, "/api/runs/submit", &body).await; - assert_eq!(status, StatusCode::UNAUTHORIZED); - assert_eq!(v["error_kind"], "coord.unauthorized"); - } - - // ── Sprint 0.5-S7: blob-fetch route ── - - /// In-memory store wired to a real on-disk blob store at a per-test - /// tempdir. Used by the blob-route handler tests. - fn fresh_state_with_blob_store() -> (CoordinatorState, tempfile::TempDir) { - let dir = tempfile::tempdir().unwrap(); - let blobs_root = dir.path().join("blobs"); - let store = RunCheckpointStore::open_in_memory_with_blob_store(blobs_root).unwrap(); - let capability_set_hash = compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - let state = CoordinatorState { - store: Arc::new(Mutex::new(store)), - workers: Arc::new(Mutex::new(HashMap::new())), - workflow_dirs: Arc::new(Mutex::new(HashMap::new())), - capability_set_hash, - config: CoordinatorConfig { - max_lease_ttl_ms: 600_000, - poll_timeout_ms: 200, - bind_warning: None, - shared_secret: None, - mtls_required: false, - }, - start_time_ms: 0, - }; - (state, dir) - } - - fn complete_running_step_in_state( - state: &CoordinatorState, - run_id: &str, - step_id: &str, - output_json: &str, - ) -> String { - use sha2::Digest; - pending_step(state, run_id, step_id, "fn main() -> Int { 0 }"); - let store = state.store.lock().unwrap(); - let claim_id = match store - .claim_step(run_id, step_id, "wkr-test", 9_999_999_999, 1_000) - .unwrap() - { - ClaimOutcome::Claimed { claim_id } => claim_id, - other => panic!("expected Claimed, got {other:?}"), - }; - let mut h = sha2::Sha256::new(); - h.update(output_json.as_bytes()); - let hash = format!("{:x}", h.finalize()); - store - .complete_step_cas(run_id, step_id, claim_id, output_json, &hash, 1, 2_000) - .unwrap(); - hash - } - - async fn get_request(app: &Router, uri: &str) -> (StatusCode, Vec) { - let req = Request::builder() - .method("GET") - .uri(uri) - .body(Body::empty()) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - let status = resp.status(); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap() - .to_vec(); - (status, bytes) - } - - #[tokio::test] - async fn console_shell_served_without_bearer_and_carries_no_data() { - // The console SHELL must load without a bearer header (a browser - // navigation carries none) EVEN when a shared secret is set, must embed - // no run data (data is fetched client-side), and must deny framing + - // caching. - let mut state = fresh_state(); - state.config.shared_secret = Some("s3cret".into()); - { - let store = state.store.lock().unwrap(); - store - .insert_run(&RunRow { - run_id: "leaky-run-id-xyz".into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Paused, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: r#"{"default_allow":true}"#.into(), - metadata_json: "{}".into(), - }) - .unwrap(); - } - let app = build_router(state); - let req = Request::builder() - .method("GET") - .uri("/console") - .body(Body::empty()) - .unwrap(); - let resp = app.oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.headers().get("x-frame-options").unwrap(), "DENY"); - assert_eq!(resp.headers().get("cache-control").unwrap(), "no-store"); - assert!(resp - .headers() - .get("content-security-policy") - .unwrap() - .to_str() - .unwrap() - .contains("frame-ancestors")); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let html = String::from_utf8_lossy(&bytes); - assert!(html.contains("Boruna Approval Console")); - assert!( - !html.contains("leaky-run-id-xyz"), - "the shell must embed no run data" - ); - } - - #[tokio::test] - async fn api_runs_still_requires_bearer_when_secret_set() { - // Only the shell is exempt — the data endpoint the console reads stays - // behind auth. - let mut state = fresh_state(); - state.config.shared_secret = Some("s3cret".into()); - let app = build_router(state); - let (status, _) = get_request(&app, "/api/runs").await; - assert_eq!(status, StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn get_blob_returns_bytes_for_referenced_hash() { - let (state, _dir) = fresh_state_with_blob_store(); - let payload = "\"".to_string() - + &"a".repeat(boruna_orchestrator::persistence::BLOB_THRESHOLD + 1) - + "\""; - let hash = complete_running_step_in_state(&state, "RUN-1", "s1", &payload); - let app = build_router(state); - let (status, bytes) = get_request(&app, &format!("/api/runs/RUN-1/blobs/{hash}")).await; - assert_eq!(status, StatusCode::OK); - assert_eq!(bytes, payload.as_bytes()); - } - - #[tokio::test] - async fn get_blob_bad_hash_short() { - let (state, _dir) = fresh_state_with_blob_store(); - let app = build_router(state); - let (status, bytes) = get_request(&app, "/api/runs/RUN-1/blobs/abc").await; - assert_eq!(status, StatusCode::BAD_REQUEST); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.blobs.bad_hash"); - } - - #[tokio::test] - async fn get_blob_bad_hash_uppercase() { - let (state, _dir) = fresh_state_with_blob_store(); - let app = build_router(state); - // 64 chars but uppercase → format check fails before scope check. - let bad = "A".repeat(64); - let (status, bytes) = get_request(&app, &format!("/api/runs/RUN-1/blobs/{bad}")).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.blobs.bad_hash"); - } - - #[tokio::test] - async fn get_blob_not_found_unknown_hash() { - let (state, _dir) = fresh_state_with_blob_store(); - let app = build_router(state); - let bogus = "0".repeat(64); - let (status, bytes) = get_request(&app, &format!("/api/runs/RUN-1/blobs/{bogus}")).await; - assert_eq!(status, StatusCode::NOT_FOUND); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.blobs.not_found"); - } - - #[tokio::test] - async fn get_blob_run_scope_enforced() { - // Run-A produces a blob; GET on run-B for the same hash → 404. - let (state, _dir) = fresh_state_with_blob_store(); - let payload = "\"".to_string() - + &"q".repeat(boruna_orchestrator::persistence::BLOB_THRESHOLD + 1) - + "\""; - let hash = complete_running_step_in_state(&state, "RUN-A", "s1", &payload); - let app = build_router(state); - let (status, bytes) = get_request(&app, &format!("/api/runs/RUN-B/blobs/{hash}")).await; - assert_eq!(status, StatusCode::NOT_FOUND); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.blobs.not_found"); - } - - #[tokio::test] - async fn get_blob_unauthorized_no_bearer() { - let (mut state, _dir) = fresh_state_with_blob_store(); - state.config.shared_secret = Some("super-secret".into()); - let app = build_router(state); - let bogus = "0".repeat(64); - let (status, bytes) = get_request(&app, &format!("/api/runs/RUN-1/blobs/{bogus}")).await; - assert_eq!(status, StatusCode::UNAUTHORIZED); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.unauthorized"); - } - - // Sprint W2 — health endpoint + auth bypass. - - #[tokio::test] - async fn health_returns_ready_status_and_metadata() { - let state = fresh_state(); - let app = build_router(state.clone()); - let (status, bytes) = get_request(&app, "/api/health").await; - assert_eq!(status, StatusCode::OK); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["protocol_version"], 1); - assert_eq!(v["status"], "ready"); - assert_eq!(v["boruna_version"], env!("CARGO_PKG_VERSION")); - assert_eq!(v["capability_set_hash"], state.capability_set_hash); - assert!( - v["uptime_ms"].as_i64().unwrap() >= 0, - "uptime must be non-negative" - ); - } - - #[tokio::test] - async fn health_bypasses_auth_when_secret_configured() { - // Per W2 design: load balancers and external probes don't - // hold the bearer secret. /api/health must answer 200 even - // with the secret enabled. - let mut state = fresh_state(); - state.config.shared_secret = Some("super-secret-token-not-leaked".into()); - let app = build_router(state); - let (status, bytes) = get_request(&app, "/api/health").await; - assert_eq!(status, StatusCode::OK); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["status"], "ready"); - // The secret itself MUST NOT leak into health output. - let body_str = String::from_utf8_lossy(&bytes); - assert!( - !body_str.contains("super-secret-token-not-leaked"), - "shared secret leaked into /api/health response" - ); - } - - #[tokio::test] - async fn other_routes_still_require_auth_when_secret_configured() { - // Sanity: the W2 health bypass MUST NOT relax auth on - // any other route. Verify a non-health GET still 401s. - let mut state = fresh_state(); - state.config.shared_secret = Some("super-secret".into()); - let app = build_router(state); - let (status, _bytes) = get_request(&app, "/api/runs").await; - assert_eq!(status, StatusCode::UNAUTHORIZED); - } - - // ── mTLS surface (sprint W6-A) ── - - #[test] - fn parse_tls_flags_requires_all_three_or_none() { - // None of the three flags = no TLS, ok. - assert!(ServerTlsPaths::from_optional(None, None, None) - .unwrap() - .is_none()); - // All three present = ok. - let p = std::path::PathBuf::from("/tmp/x"); - let triple = - ServerTlsPaths::from_optional(Some(p.clone()), Some(p.clone()), Some(p.clone())) - .unwrap(); - assert!(triple.is_some()); - // Any partial combination is rejected. - for (a, b, c) in [ - (Some(p.clone()), None, None), - (None, Some(p.clone()), None), - (None, None, Some(p.clone())), - (Some(p.clone()), Some(p.clone()), None), - (Some(p.clone()), None, Some(p.clone())), - (None, Some(p.clone()), Some(p.clone())), - ] { - let err = ServerTlsPaths::from_optional(a, b, c).unwrap_err(); - assert!( - err.to_string().contains("must all be provided together"), - "unexpected err: {err}" - ); - } - } - - #[tokio::test] - async fn auth_middleware_rejects_when_tls_required_but_no_cert() { - // Synthesize a state with mtls_required=true but DON'T - // provide a ClientIdentity extension on the request. The - // middleware must reject with 401 + coord.unauthorized - // even when no shared_secret is configured. - let mut state = fresh_state(); - state.config.mtls_required = true; - let app = build_router(state.clone()); - let req = Request::builder() - .method("POST") - .uri("/api/workers/register") - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(®ister_payload(state.capability_set_hash.clone())).unwrap(), - )) - .unwrap(); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.unauthorized"); - } - - #[tokio::test] - async fn auth_middleware_accepts_when_tls_required_and_identity_present() { - // With mtls_required=true and a synthesized ClientIdentity - // extension, the request goes through. - let mut state = fresh_state(); - state.config.mtls_required = true; - let app = build_router(state.clone()); - let mut req = Request::builder() - .method("POST") - .uri("/api/workers/register") - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&RegisterRequest { - worker_id: Some("worker-7".into()), - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: None, - }) - .unwrap(), - )) - .unwrap(); - req.extensions_mut().insert(ClientIdentity { - common_name: "worker-7".into(), - }); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - } - - #[tokio::test] - async fn handle_register_rejects_cn_worker_id_mismatch() { - // mTLS-on. Cert says "worker-A" but body claims "worker-B". - // Must return 401 + coord.identity_mismatch. - let mut state = fresh_state(); - state.config.mtls_required = true; - let app = build_router(state.clone()); - let mut req = Request::builder() - .method("POST") - .uri("/api/workers/register") - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&RegisterRequest { - worker_id: Some("worker-B".into()), - capability_set_hash: state.capability_set_hash.clone(), - advertised_capabilities: None, - }) - .unwrap(), - )) - .unwrap(); - req.extensions_mut().insert(ClientIdentity { - common_name: "worker-A".into(), - }); - let resp = app.clone().oneshot(req).await.unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) - .await - .unwrap(); - let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(v["error_kind"], "coord.identity_mismatch"); - } - - // post1/scheduler-registry-rolling — version_compatible / semver_gte tests. - - #[test] - fn version_compatible_exact_match() { - let worker: BTreeMap = [("llm.call".to_string(), "1.0".to_string())] - .into_iter() - .collect(); - let required: BTreeMap = [("llm.call".to_string(), "1.0".to_string())] - .into_iter() - .collect(); - assert!(version_compatible(&worker, &required)); - } - - #[test] - fn version_compatible_newer_worker() { - let worker: BTreeMap = [("llm.call".to_string(), "2.0".to_string())] - .into_iter() - .collect(); - let required: BTreeMap = [("llm.call".to_string(), "1.0".to_string())] - .into_iter() - .collect(); - assert!(version_compatible(&worker, &required)); - } - - #[test] - fn version_compatible_older_worker() { - let worker: BTreeMap = [("llm.call".to_string(), "1.0".to_string())] - .into_iter() - .collect(); - let required: BTreeMap = [("llm.call".to_string(), "2.0".to_string())] - .into_iter() - .collect(); - assert!(!version_compatible(&worker, &required)); - } - - #[test] - fn version_compatible_missing_cap_defaults_1_0() { - // Worker has no version declared for "llm.call"; default is "1.0". - let worker: BTreeMap = BTreeMap::new(); - let required: BTreeMap = [("llm.call".to_string(), "1.0".to_string())] - .into_iter() - .collect(); - assert!(version_compatible(&worker, &required)); - } - - #[test] - fn cn_extraction_pulls_subject_correctly() { - // Generate a self-signed cert with rcgen, then run - // cn_from_cert_der over the DER and assert we recover the - // CN. This locks the CN parser against the same DER shape - // rcgen / openssl produce. - let mut params = rcgen::CertificateParams::new(vec!["worker-cn-extract-test".into()]) - .expect("CertificateParams::new"); - let mut dn = rcgen::DistinguishedName::new(); - dn.push(rcgen::DnType::CommonName, "worker-cn-extract-test"); - params.distinguished_name = dn; - let key_pair = rcgen::KeyPair::generate().expect("keygen"); - let cert = params.self_signed(&key_pair).expect("self-sign"); - let der = cert.der(); - let cn = cn_from_cert_der(der.as_ref()).expect("CN found"); - assert_eq!(cn, "worker-cn-extract-test"); - } -} diff --git a/crates/llmvm-cli/src/dashboard.rs b/crates/llmvm-cli/src/dashboard.rs deleted file mode 100644 index e27933b..0000000 --- a/crates/llmvm-cli/src/dashboard.rs +++ /dev/null @@ -1,1174 +0,0 @@ -//! Workflow dashboard — read-only HTTP view over `runs.db` -//! (sprint `0.4-S16`). -//! -//! See `docs/design-workflow-dashboard.md` and -//! `docs/architecture-workflow-dashboard.md` for the design rationale, -//! security posture, and route contract. -//! -//! ## Security posture -//! -//! - Loopback (127.0.0.1) by default. Operators who pass -//! `--bind 0.0.0.0` get a loud warning at startup AND a banner in -//! the rendered HTML. -//! - **No authentication.** This sprint ships dev-grade access only. -//! Operators exposing the dashboard to a network MUST front it with -//! an auth-enforcing reverse proxy. -//! - **Read-only.** Zero mutation routes. Confirmed by a regression -//! test that asserts non-GET methods on every route return 405. -//! -//! ## Stability -//! -//! Per `docs/stability.md`: **experimental**. Route paths and flag -//! names are stable; rendered HTML is not. JSON shapes inherit the -//! stability of `RunRow`, `RunRecord`, `RunOperational`, and -//! `StepCheckpoint`. - -use std::net::IpAddr; -use std::path::PathBuf; -use std::sync::{Arc, Mutex}; - -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::response::Html; -use axum::routing::get; -use axum::{Json, Router}; - -use boruna_orchestrator::persistence::{ - RunCheckpointStore, RunOperational, RunRecord, RunRow, RunStatus, StepCheckpoint, -}; -use serde::Serialize; - -/// Shared state for handlers. -/// -/// `bind_warning` is `Some(addr)` only when the dashboard was bound -/// to a non-loopback address; the index handler renders a banner in -/// that case. -#[derive(Clone)] -pub struct DashboardState { - store: Arc>, - bind_warning: Option, -} - -#[tokio::main] -pub async fn run_serve( - data_dir: PathBuf, - port: u16, - bind: IpAddr, -) -> Result<(), Box> { - let db_path = data_dir.join("runs.db"); - if !db_path.exists() { - return Err(format!( - "no runs.db at {} — run a workflow first or pass a different --data-dir", - db_path.display() - ) - .into()); - } - - let store = RunCheckpointStore::open(&db_path) - .map_err(|e| format!("failed to open {}: {e}", db_path.display()))?; - - let bind_warning = if bind.is_loopback() { - None - } else { - let msg = format!("{bind}:{port}"); - eprintln!( - "[WARNING] dashboard bound to non-loopback {msg}; \ - anyone with network access to this port can READ all run data; \ - the dashboard ships no auth — front it with an auth-enforcing reverse proxy" - ); - Some(msg) - }; - - let store_handle = Arc::new(Mutex::new(store)); - let app = dashboard_routes(store_handle, bind_warning); - - let addr = std::net::SocketAddr::new(bind, port); - eprintln!("dashboard serving on http://{addr}"); - eprintln!(" data-dir: {}", data_dir.display()); - - let listener = tokio::net::TcpListener::bind(addr).await?; - axum::serve(listener, app).await?; - Ok(()) -} - -/// Build the dashboard's read-only route surface. Public so the -/// coordinator (sprint `0.5-S2d`) can `.merge(...)` these onto -/// its own router and serve fleet visibility + distributed -/// dispatch from a single listener. -/// -/// Takes primitive args (the shared store handle and an -/// optional `bind_warning` string for the banner) instead of -/// the internal `DashboardState` so the coordinator doesn't -/// have to reach into the dashboard's internals. -pub fn dashboard_routes( - store: Arc>, - bind_warning: Option, -) -> Router { - let state = DashboardState { - store, - bind_warning, - }; - Router::new() - .route("/", get(handle_index)) - .route("/runs/{id}", get(handle_run_detail)) - .route("/api/runs", get(handle_api_runs)) - .route("/api/runs/{id}", get(handle_api_run_detail)) - .with_state(state) -} - -// ── Response shapes ── - -/// Slim list-view of a run — no `policy_json` or `metadata_json`. -/// -/// Sprint `0.4-S16` adversarial review found that returning full -/// `RunRow` in the list endpoint multiplied the disclosure surface: -/// every operator hitting `/api/runs` would see ALL runs' policies -/// and metadata. Operators sometimes embed secrets, hostnames, or -/// customer identifiers in `metadata_json`; serving them by default -/// in a no-auth dashboard is the wrong default. -/// -/// The detail endpoint (`/api/runs/:id`) still returns the full -/// record — operator drilling in is a deliberate action. -#[derive(Serialize, Debug)] -struct RunSummary { - run_id: String, - workflow_name: String, - workflow_hash: String, - status: RunStatus, - started_at_ms: i64, - updated_at_ms: i64, -} - -impl From<&RunRow> for RunSummary { - fn from(row: &RunRow) -> Self { - Self { - run_id: row.run_id.clone(), - workflow_name: row.workflow_name.clone(), - workflow_hash: row.workflow_hash.clone(), - status: row.status, - started_at_ms: row.started_at_ms, - updated_at_ms: row.updated_at_ms, - } - } -} - -#[derive(Serialize, Debug)] -struct RunsListResponse { - runs: Vec, -} - -#[derive(Serialize, Debug)] -struct RunDetailResponse { - run: RunRecord, - operational: Option, - steps: Vec, -} - -// ── Handlers ── - -async fn handle_index(State(state): State) -> Result, StatusCode> { - let runs = { - let store = state - .store - .lock() - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - store - .list_runs() - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? - }; - Ok(Html(render_index(&runs, state.bind_warning.as_deref()))) -} - -async fn handle_run_detail( - State(state): State, - Path(id): Path, -) -> Result, StatusCode> { - let (run, operational, steps) = load_run_detail(&state, &id)?; - // Sprint 0.5-S7b: resolve per-step output for the HTML render. - // For inline outputs we read the bytes; for blob-stored outputs - // we hand the renderer the ref so it can emit a link to the - // S7 blob route without fetching the bytes (avoids slurping - // multi-MB blobs into a dashboard render). - let outputs = resolve_step_outputs_for_render(&state, &id, &steps)?; - Ok(Html(render_detail( - &run, - operational.as_ref(), - &steps, - &outputs, - state.bind_warning.as_deref(), - ))) -} - -async fn handle_api_runs( - State(state): State, -) -> Result, StatusCode> { - let runs = { - let store = state - .store - .lock() - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - store - .list_runs() - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? - }; - let summaries: Vec = runs.iter().map(RunSummary::from).collect(); - Ok(Json(RunsListResponse { runs: summaries })) -} - -async fn handle_api_run_detail( - State(state): State, - Path(id): Path, -) -> Result, StatusCode> { - let (run, operational, steps) = load_run_detail(&state, &id)?; - Ok(Json(RunDetailResponse { - run, - operational, - steps, - })) -} - -fn load_run_detail( - state: &DashboardState, - id: &str, -) -> Result<(RunRecord, Option, Vec), StatusCode> { - let store = state - .store - .lock() - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - let run = store - .get_run_record(id) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? - .ok_or(StatusCode::NOT_FOUND)?; - let operational = store - .get_run_operational(id) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - let steps = store - .list_step_checkpoints(id) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - Ok((run, operational, steps)) -} - -/// Per-step output as it should render in the HTML detail page. -/// Sprint 0.5-S7b — distinguishes inline-rendered text from a -/// blob-link placeholder so `render_detail` can emit different -/// markup without re-querying the persistence layer. -#[derive(Debug, Clone)] -enum StepOutputDisplay { - /// No output yet (Pending / Running / paused / Failed-without-output). - None, - /// Output stored inline; the value is the JSON-encoded text. - /// Renderer applies truncation + html_escape. - Inline(String), - /// Output stored in the blob store. The hash is the - /// `output_blob_ref` column value. Renderer emits a link to - /// the coord/dashboard's blob route — does NOT fetch bytes - /// (large blobs would bloat the dashboard render). - Blob(String), -} - -/// Resolve the per-step output rendering for the HTML detail page. -/// Reads through `read_step_output` for inline outputs; for -/// blob-stored steps it short-circuits to the ref (which lives on -/// the StepCheckpoint already) so we don't pull large bytes into -/// memory just to render a link. -fn resolve_step_outputs_for_render( - state: &DashboardState, - run_id: &str, - steps: &[StepCheckpoint], -) -> Result, StatusCode> { - let store = state - .store - .lock() - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - let mut out = Vec::with_capacity(steps.len()); - for cp in steps { - // Blob-stored: skip the byte read; we want a link, not the bytes. - if let Some(hash) = &cp.output_blob_ref { - out.push(StepOutputDisplay::Blob(hash.clone())); - continue; - } - // Inline (or no output yet) — read_step_output handles both, - // returning Some(json) for completed-inline and None for - // pending/running/failed-without-output. - match store.read_step_output(run_id, &cp.step_id) { - Ok(Some(json)) => out.push(StepOutputDisplay::Inline(json)), - Ok(None) => out.push(StepOutputDisplay::None), - Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), - } - } - Ok(out) -} - -// ── Rendering ── - -/// Escape HTML special characters. Every value rendered into the -/// HTML output must go through this helper. Operator-controlled -/// run_ids and workflow_names are operational state but could in -/// principle contain XSS payloads (especially when run_ids come -/// from external triggers). -fn html_escape(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for c in s.chars() { - match c { - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '&' => out.push_str("&"), - '"' => out.push_str("""), - '\'' => out.push_str("'"), - _ => out.push(c), - } - } - out -} - -const PAGE_STYLE: &str = r#" - -"#; - -/// Top navigation bar shared by both dashboard pages. `active` is the -/// nav key of the current page ("runs" for the list; detail pages are -/// also under "runs") so the matching link gets an `aria-current` cue. -fn render_header(active: &str) -> String { - let cur = |k: &str| { - if k == active { - r#" aria-current="page""# - } else { - "" - } - }; - format!( - r#"
- Boruna Workflow Dashboard - -
"#, - r = cur("runs"), - j = cur("api"), - ) -} - -fn render_banner(bind_warning: Option<&str>) -> String { - match bind_warning { - Some(addr) => format!( - r#""#, - html_escape(addr) - ), - None => String::new(), - } -} - -fn render_index(runs: &[RunRow], bind_warning: Option<&str>) -> String { - let banner = render_banner(bind_warning); - let mut body = String::new(); - body.push_str("

Workflow runs

"); - body.push_str( - r#"

Every workflow execution recorded in this Boruna instance. Each row is one -run; click its Run ID to see per-step progress, outputs, and errors. Times are UTC.

"#, - ); - body.push_str(&format!( - r#"

{} run{} total

"#, - runs.len(), - if runs.len() == 1 { "" } else { "s" } - )); - - if runs.is_empty() { - body.push_str(r#"

No runs yet. Start one with boruna workflow run ..., then reload this page.

"#); - } else { - body.push_str(r#"
"#); - body.push_str( - r#""#, - ); - body.push_str(""); - body.push_str( - r#""#, - ); - body.push_str(""); - for run in runs { - let status_class = format!("status-{}", run.status.as_str()); - body.push_str(&format!( - r#""#, - html_escape(&run.run_id), - html_escape(&run.run_id), - html_escape(&run.workflow_name), - status_class, - run.status.as_str(), - format_ms(run.started_at_ms), - format_ms(run.updated_at_ms), - )); - } - body.push_str("
Status: running - paused - completed - failed
Run IDWorkflowStatusStartedUpdated
{}{}{}{}{}
"); - } - - body.push_str(r#"

Machine-readable list: GET /api/runs

"#); - - wrap_page("Boruna runs", &render_header("runs"), &banner, &body) -} - -/// Maximum chars of inline output rendered in the HTML detail -/// page. Sprint 0.5-S7b. Truncated outputs append `…` to signal -/// elision. The full bytes are still in the persistence layer -/// (or the blob store); operators can fetch via `boruna evidence -/// inspect` or the JSON API for the full content. -const HTML_INLINE_OUTPUT_MAX: usize = 256; - -/// Visible chars of a blob hash in the HTML detail page link -/// label. Sprint 0.5-S7b. The full 64-char hash is preserved in -/// the link target; the visible label is shortened so the table -/// stays readable. -const HTML_BLOB_HASH_LABEL_LEN: usize = 16; - -fn render_detail( - run: &RunRecord, - operational: Option<&RunOperational>, - steps: &[StepCheckpoint], - outputs: &[StepOutputDisplay], - bind_warning: Option<&str>, -) -> String { - let banner = render_banner(bind_warning); - let mut body = String::new(); - body.push_str(r#"

← All runs

"#); - body.push_str(&format!("

Run {}

", html_escape(&run.run_id))); - body.push_str( - r#"

One workflow run. The summary below identifies which workflow ran and its -current state; the steps table shows each step's progress, retry count, output, and any error.

"#, - ); - - body.push_str(r#"
"#); - body.push_str(&format!( - r#""#, - html_escape(&run.workflow_name) - )); - body.push_str(&format!( - r#""#, - h = html_escape(&run.workflow_hash) - )); - if let Some(op) = operational { - let status_class = format!("status-{}", op.transient_status.as_str()); - body.push_str(&format!( - r#""#, - status_class, - op.transient_status.as_str() - )); - body.push_str(&format!( - r#""#, - format_ms(op.started_at_ms) - )); - body.push_str(&format!( - r#""#, - format_ms(op.updated_at_ms) - )); - } - if let Some(t) = &run.terminal_status { - body.push_str(&format!( - r#""#, - c = t.as_str() - )); - } - body.push_str("
Workflow{}
Workflow hash{h}
Status{}
Started{}
Updated{}
Terminal{c}
"); - - body.push_str(&format!("

Steps ({})

", steps.len())); - if steps.is_empty() { - body.push_str( - r#"

No step checkpoints recorded — the run may not have started - any steps yet, or it ran in a mode that does not persist per-step checkpoints.

"#, - ); - } else { - body.push_str(r#"
"#); - body.push_str( - r#""#, - ); - body.push_str(""); - body.push_str( - r#""#, - ); - body.push_str(""); - for (step, output) in steps.iter().zip(outputs.iter()) { - let status_class = format!("status-{}", step.status.as_str()); - let started = step.started_at_ms.map(format_ms).unwrap_or_default(); - let ended = step.ended_at_ms.map(format_ms).unwrap_or_default(); - let error = step - .error_msg - .as_deref() - .map(html_escape) - .unwrap_or_default(); - let output_cell = render_output_cell(&run.run_id, output); - body.push_str(&format!( - r#""#, - html_escape(&step.step_id), - status_class, - step.status.as_str(), - step.attempt_count, - started, - ended, - output_cell, - error, - )); - } - body.push_str("
Output shows the recorded JSON result (long values are truncated with … and - large ones link out as a blob). Attempts counts retries. Times are UTC.
StepStatusAttemptsStartedEndedOutputError
{}{}{}{}{}{}{}
"); - } - - body.push_str(&format!( - r#"

Machine-readable detail: GET /api/runs/{}

"#, - html_escape(&run.run_id) - )); - - wrap_page( - &format!("Run {}", run.run_id), - &render_header("runs"), - &banner, - &body, - ) -} - -/// Render a single step's Output cell. Sprint 0.5-S7b. -/// -/// - `StepOutputDisplay::None` → render `—` (em dash). -/// - `StepOutputDisplay::Inline(json)` → render in a `` block, -/// truncated to [`HTML_INLINE_OUTPUT_MAX`] chars (UTF-8 char -/// boundary safe). Contents pass through `html_escape`. -/// - `StepOutputDisplay::Blob(hash)` → render `[blob: …]` -/// linked to `/api/runs/{run_id}/blobs/{hash}`. The full hash is -/// in the URL; the visible label is shortened so the table stays -/// readable. -fn render_output_cell(run_id: &str, output: &StepOutputDisplay) -> String { - match output { - StepOutputDisplay::None => "—".to_string(), - StepOutputDisplay::Inline(json) => { - let truncated = truncate_chars(json, HTML_INLINE_OUTPUT_MAX); - let suffix = if truncated.len() < json.len() { - "…" - } else { - "" - }; - format!("{}{}", html_escape(truncated), suffix) - } - StepOutputDisplay::Blob(hash) => { - // Defensive: only build the link if hash matches the - // expected 64-lowercase-hex shape. Otherwise the - // blob-route will reject; render plain text. - let valid = hash.len() == 64 - && hash - .bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)); - if !valid { - return format!("[blob: {}]", html_escape(hash)); - } - let label_len = hash.len().min(HTML_BLOB_HASH_LABEL_LEN); - let short = &hash[..label_len]; - format!( - r#"[blob: {}…]"#, - html_escape(run_id), - html_escape(hash), - html_escape(short), - ) - } - } -} - -/// Truncate `s` to at most `max_chars` Unicode scalar values -/// (NOT bytes), staying on a char boundary so `html_escape` -/// receives a valid `&str`. Returns a borrowed prefix. -fn truncate_chars(s: &str, max_chars: usize) -> &str { - let mut end = s.len(); - for (count, (i, _)) in s.char_indices().enumerate() { - if count == max_chars { - end = i; - break; - } - } - &s[..end] -} - -fn wrap_page(title: &str, header: &str, banner: &str, body: &str) -> String { - format!( - r#" - -{}{} -{}{}
{}
-
Read-only view over runs.db. This dashboard cannot start, stop, or -change runs — manage workflows with the boruna CLI. It ships no authentication; keep it on -127.0.0.1 unless fronted by an auth-enforcing proxy.
- -"#, - html_escape(title), - PAGE_STYLE, - header, - banner, - body - ) -} - -/// Format a Unix epoch ms timestamp as ISO-8601 UTC with a `Z` -/// suffix. We don't pull `chrono` for this since `boruna-cli` -/// doesn't already depend on it; manual computation is fine for -/// a 19-character string. -fn format_ms(ms: i64) -> String { - // Negative ms (clock skew or bad data) would produce malformed - // output via the year-walking loop below. The 0 sentinel - // already returns empty for missing timestamps; treat negatives - // the same way. Caught by adversarial review for sprint - // 0.4-S16. - if ms <= 0 { - return String::new(); - } - // Convert to days since 1970-01-01 + intra-day seconds. - let total_secs = ms.div_euclid(1000); - let day_secs = total_secs.rem_euclid(86_400); - let mut days = total_secs.div_euclid(86_400); - let hours = (day_secs / 3600) as u32; - let minutes = ((day_secs % 3600) / 60) as u32; - let seconds = (day_secs % 60) as u32; - - // Convert days-since-epoch to YYYY-MM-DD via the standard - // proleptic-Gregorian algorithm. - let mut year: i64 = 1970; - loop { - let dy = if is_leap(year) { 366 } else { 365 }; - if days < dy { - break; - } - days -= dy; - year += 1; - } - let mut month: u32 = 1; - let days_in_month = |m: u32, y: i64| -> i64 { - match m { - 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, - 4 | 6 | 9 | 11 => 30, - 2 if is_leap(y) => 29, - 2 => 28, - _ => 0, - } - }; - while days >= days_in_month(month, year) { - days -= days_in_month(month, year); - month += 1; - } - let day = (days + 1) as u32; - format!("{year:04}-{month:02}-{day:02}T{hours:02}:{minutes:02}:{seconds:02}Z") -} - -fn is_leap(y: i64) -> bool { - (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 -} - -#[cfg(test)] -mod tests { - use super::*; - use boruna_orchestrator::persistence::{ClaimOutcome, RunStatus, StepStatus}; - - fn fresh_store() -> RunCheckpointStore { - RunCheckpointStore::open_in_memory().expect("open in-memory store") - } - - fn sample_run(id: &str, name: &str, status: RunStatus) -> RunRow { - RunRow { - run_id: id.into(), - workflow_name: name.into(), - workflow_hash: "x".into(), - status, - started_at_ms: 1_700_000_000_000, - updated_at_ms: 1_700_000_000_500, - policy_json: "{}".into(), - metadata_json: "{}".into(), - } - } - - fn sample_step(run_id: &str, step_id: &str, status: StepStatus) -> StepCheckpoint { - StepCheckpoint { - run_id: run_id.into(), - step_id: step_id.into(), - status, - output_json: None, - output_hash: None, - started_at_ms: Some(1_700_000_001_000), - ended_at_ms: Some(1_700_000_002_000), - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - } - } - - fn state_with(store: RunCheckpointStore, bind_warning: Option<&str>) -> DashboardState { - DashboardState { - store: Arc::new(Mutex::new(store)), - bind_warning: bind_warning.map(String::from), - } - } - - // ── Index handler ── - - #[tokio::test] - async fn handle_index_empty_store_renders_empty_table() { - let state = state_with(fresh_store(), None); - let html = handle_index(State(state)).await.unwrap().0; - assert!(html.contains("0 runs total")); - assert!(html.contains("No runs yet")); - } - - #[tokio::test] - async fn handle_index_renders_runs_grouped_by_status() { - let store = fresh_store(); - store - .insert_run(&sample_run("r1", "wf-a", RunStatus::Running)) - .unwrap(); - store - .insert_run(&sample_run("r2", "wf-b", RunStatus::Completed)) - .unwrap(); - store - .insert_run(&sample_run("r3", "wf-a", RunStatus::Paused)) - .unwrap(); - let state = state_with(store, None); - let html = handle_index(State(state)).await.unwrap().0; - assert!(html.contains("3 runs total")); - assert!(html.contains("r1") && html.contains("r2") && html.contains("r3")); - assert!(html.contains("running") && html.contains("paused") && html.contains("completed")); - } - - #[tokio::test] - async fn handle_index_html_escapes_run_ids() { - let store = fresh_store(); - store - .insert_run(&sample_run( - "", - "wf", - RunStatus::Running, - )) - .unwrap(); - let state = state_with(store, None); - let html = handle_index(State(state)).await.unwrap().0; - assert!(html.contains("<script>")); - assert!(!html.contains(""), - ); - let html = render_outputs_page(&data); - assert!(!html.contains("")); - assert!(html.contains("<script>")); - } - - #[test] - fn render_outputs_page_empty() { - let data = make_data(true); - let html = render_outputs_page(&data); - assert!(html.contains("0 steps")); - assert!(html.contains("No outputs found")); - } - - #[test] - fn describe_event_workflow_started() { - let event = AuditEvent::WorkflowStarted { - workflow_hash: String::from("wh"), - policy_hash: String::from("ph"), - }; - let (step, etype, detail) = describe_event(&event); - assert_eq!(step, ""); - assert_eq!(etype, "WorkflowStarted"); - assert!(detail.contains("wh")); - } -} diff --git a/crates/llmvm-cli/src/main.rs b/crates/llmvm-cli/src/main.rs index 4c576cb..ba8c7ca 100644 --- a/crates/llmvm-cli/src/main.rs +++ b/crates/llmvm-cli/src/main.rs @@ -15,24 +15,14 @@ use boruna_vm::capability_gateway::{CapabilityGateway, Policy, ReplayHandler}; use boruna_vm::replay::EventLog; use boruna_vm::vm::Vm; -#[cfg(feature = "serve")] -mod coordinator; -#[cfg(feature = "serve")] -mod dashboard; mod doctor; mod evidence_diff; -#[cfg(feature = "serve")] -mod evidence_serve; mod format; mod provider_registry; mod repl; mod scaffold; -#[cfg(feature = "serve")] -mod serve; mod size; mod skills; -#[cfg(feature = "serve")] -mod worker; mod workflow_eval; #[derive(Parser)] @@ -268,27 +258,6 @@ enum Command { /// stable `error_kind` taxonomy. #[command(subcommand)] Policy(PolicyCommand), - /// Workflow dashboard — read-only HTTP view over `runs.db` - /// (sprint 0.4-S16). Requires `--features serve`. See - /// `docs/design-workflow-dashboard.md` for the security - /// posture (loopback by default, no auth). - #[cfg(feature = "serve")] - #[command(subcommand)] - Dashboard(DashboardCommand), - /// Distributed-execution coordinator — HTTP server that - /// dispatches workflow steps to remote workers (sprint - /// 0.5-S2b, ADR 002). Requires `--features serve`. Loopback - /// default; **no authentication** — front with reverse - /// proxy if exposed publicly. - #[cfg(feature = "serve")] - #[command(subcommand)] - Coordinator(CoordinatorCommand), - /// Distributed-execution worker — polls a coordinator for - /// claimable steps, executes them, reports results (sprint - /// 0.5-S2b, ADR 002). Requires `--features serve`. - #[cfg(feature = "serve")] - #[command(subcommand)] - Worker(WorkerCommand), /// Migration tooling beta (sprint `W5-C`). Upgrades pre-1.0 /// Boruna artifacts to the current on-disk format. See /// `docs/guides/migration.md` for the coverage matrix and @@ -316,208 +285,6 @@ enum Command { }, } -#[cfg(feature = "serve")] -#[derive(Subcommand)] -enum CoordinatorCommand { - /// Serve the coordinator HTTP routes. - Serve { - /// Persistent data directory holding `runs.db`. Same - /// fallback chain as `boruna workflow run`. - #[arg(long)] - data_dir: Option, - /// Listen port (default 8090). - #[arg(long, default_value = "8090")] - port: u16, - /// Bind address. Defaults to `127.0.0.1`. Pass `0.0.0.0` - /// to expose on all interfaces (you accept the - /// no-auth-on-LAN consequences). - #[arg(long, default_value = "127.0.0.1")] - bind: String, - /// Cap on lease TTL workers can request (default 5 min). - #[arg(long, default_value = "300000")] - max_lease_ttl_ms: u64, - /// Long-poll wait timeout for `/api/work/claim` - /// (default 30 s). - #[arg(long, default_value = "30000")] - poll_timeout_ms: u64, - /// Background lease-expiry sweep interval in - /// milliseconds (default 30 s). Lower = faster - /// recovery from worker crashes; higher = less DB - /// churn under steady-state. Minimum 100 ms (lower - /// values are clamped + a warning is logged). - #[arg(long, default_value = "30000")] - sweep_interval_ms: u64, - /// Shared-secret bearer token for HTTP authentication - /// (sprint `0.5-S3`). When set, every coord HTTP route - /// requires `Authorization: Bearer ` header. - /// Generate via `openssl rand -hex 32`. Falls back to - /// `BORUNA_COORD_SECRET` env var. When unset, no auth - /// is enforced — operators binding to a non-loopback - /// address without a secret get a loud stderr warning - /// (the no-auth posture remains backwards-compatible - /// for loopback-only deployments). - #[arg(long, env = "BORUNA_COORD_SECRET")] - shared_secret: Option, - /// Server certificate chain (PEM) for mTLS (sprint - /// `W6-A`). Required together with `--tls-key` and - /// `--tls-client-ca`; passing only some is a startup - /// error. Operators generate certs out-of-band — see - /// `docs/guides/coord-mtls.md`. - #[arg(long)] - tls_cert: Option, - /// Server private key (PEM) for mTLS. Required with - /// `--tls-cert` and `--tls-client-ca`. - #[arg(long)] - tls_key: Option, - /// Trust root for verifying CLIENT certificates (PEM). - /// When all three TLS flags are set the coord requires - /// every connection to present a client cert chained to - /// this root. The cert subject CN drives worker identity - /// and is matched against any `worker_id` in the request - /// body — mismatch returns `coord.identity_mismatch`. - #[arg(long)] - tls_client_ca: Option, - }, - /// Drive a submit-only workflow run to terminal status by - /// computing downstream-ready successors as workers complete - /// steps and writing fresh Pending checkpoints. Sprint - /// `0.5-S2f`: client-side multi-wave advancement. Operates on - /// the same `runs.db` the coordinator process uses; must run - /// on a host with filesystem access to `--data-dir`. - /// - /// Idempotent on restart — kill and re-invoke at any point; - /// the run continues from where it was left. - /// - /// Exit codes: - /// - 0 — run reached `Completed` status. - /// - 1 — run reached `Failed` status. - /// - 2 — invalid arguments, run not found, missing - /// `workflow_def` in metadata, or unsupported step kind - /// (approval/external_trigger in non-first wave). - /// - 3 — `--max-wait-secs` budget exceeded before terminal. - Wait { - /// Run id to drive to terminal status (returned by - /// `boruna workflow run --submit-only`). - run_id: String, - /// Persistent data directory holding `runs.db`. Same - /// fallback chain as `boruna workflow run`. Must match - /// the coordinator process's `--data-dir`. - #[arg(long)] - data_dir: Option, - /// Polling interval in milliseconds. Minimum 100 ms - /// (lower values are clamped + a warning is logged). - #[arg(long, default_value = "500")] - poll_interval_ms: u64, - /// Maximum total wait duration in seconds. `0` = - /// unlimited. Useful for CI test timeouts. - #[arg(long, default_value = "0")] - max_wait_secs: u64, - }, -} - -#[cfg(feature = "serve")] -#[derive(Subcommand)] -enum WorkerCommand { - /// Run a worker that polls the named coordinator for work. - Run { - /// Coordinator base URL, e.g. - /// `http://coord.internal:8090`. Sprint W2: accepts a - /// comma-separated list of URLs for HA failover at - /// registration time, e.g. - /// `http://coord-1:8090,http://coord-2:8090`. The worker - /// tries URLs in order and registers against the first - /// reachable one. After successful registration the - /// worker sticks to that coord for its lifetime — operator - /// restarts pick a different healthy URL. - #[arg(long)] - coordinator: String, - /// Optional worker id; auto-generated if absent. - #[arg(long)] - worker_id: Option, - /// Lease TTL the worker requests on each claim. - /// Coordinator may cap this. - #[arg(long, default_value = "300000")] - lease_ttl_ms: u64, - /// Long-poll timeout the worker tells the coordinator - /// to wait before returning 204. - #[arg(long, default_value = "30000")] - poll_timeout_ms: u64, - /// Shared-secret bearer token for HTTP authentication - /// (sprint `0.5-S3`). MUST match the coordinator's - /// `--shared-secret`. Falls back to `BORUNA_COORD_SECRET` - /// env var. When unset, no `Authorization` header is - /// sent — only works when the coord also has no secret. - #[arg(long, env = "BORUNA_COORD_SECRET")] - shared_secret: Option, - /// Sprint `W3-A` — comma-separated capability names this - /// worker advertises (e.g. `--advertise-caps net.fetch,db.query`). - /// When set, the coordinator only routes steps whose - /// policy-required capabilities are a subset of this list. - /// When omitted (or empty), the worker behaves as a - /// full-fleet worker (the pre-W3-A default). Capability - /// names must match `boruna_bytecode::Capability::ALL` - /// exactly; unknown names cause registration to fail - /// with `coord.unknown_capability`. - /// - /// **Operational metadata only** — placement filter, not - /// a security gate. The VM's capability gateway remains - /// the security boundary. - #[arg(long)] - advertise_caps: Option, - /// Per-capability version declarations: `llm.call=2.0,net.fetch=1.5`. - /// Matched against per-step `required_capability_versions` in the - /// workflow DAG. Workers without a declared version for a capability - /// default to `"1.0"`. Coordinate with `--advertise-caps` — versions - /// for undeclared capabilities are ignored by the coordinator. - #[arg(long)] - advertise_cap_versions: Option, - /// Client certificate chain (PEM) for mTLS (sprint - /// `W6-A`). Required together with `--tls-key` and - /// `--tls-server-ca`; passing only some is a startup - /// error. The cert's subject CN MUST match - /// `--worker-id` (case-insensitive) when both are set — - /// mismatch surfaces as `coord.identity_mismatch` at - /// registration time. - #[arg(long)] - tls_cert: Option, - /// Client private key (PEM) for mTLS. Required with - /// `--tls-cert` and `--tls-server-ca`. - #[arg(long)] - tls_key: Option, - /// Trust root for verifying the COORD's server - /// certificate (PEM). Required with `--tls-cert` and - /// `--tls-key`. - #[arg(long)] - tls_server_ca: Option, - }, -} - -#[cfg(feature = "serve")] -#[derive(Subcommand)] -enum DashboardCommand { - /// Serve a read-only dashboard over HTTP. - /// - /// Loopback (127.0.0.1) by default. Pass `--bind 0.0.0.0` to - /// expose on the LAN; the dashboard ships no auth, so any - /// public bind MUST be fronted by an auth-enforcing reverse - /// proxy. - Serve { - /// Persistent data directory holding `runs.db`. Same - /// fallback chain as `boruna workflow run` / - /// `metrics export`. - #[arg(long)] - data_dir: Option, - /// Listen port (default 8080). - #[arg(long, default_value = "8080")] - port: u16, - /// Bind address. Defaults to `127.0.0.1`. Pass `0.0.0.0` - /// to expose on all interfaces (you accept the - /// no-auth-on-LAN consequences). - #[arg(long, default_value = "127.0.0.1")] - bind: String, - }, -} - #[derive(Subcommand)] enum PolicyCommand { /// Strict-validate a policy file. Exits 0 on ok, 2 on @@ -1247,15 +1014,6 @@ enum EvidenceCommand { #[arg(long, value_name = "N")] parallelism: Option, }, - /// Start a local web UI to browse an evidence bundle (post1-T-4.4). - /// Requires the `serve` feature. - Serve { - /// Evidence bundle directory. - dir: PathBuf, - /// Port to listen on (default: 4444). - #[arg(long, default_value = "4444")] - port: u16, - }, /// Compare two evidence bundles side-by-side (post1-evidence-diff). /// Reports differences in workflow metadata, step outputs, audit event /// counts, and verification status. @@ -1339,15 +1097,6 @@ enum FrameworkCommand { /// Cycle log file (JSON). log: PathBuf, }, - /// Serve a framework app as a web page (requires --features serve). - #[cfg(feature = "serve")] - Serve { - /// Source file (.ax) - file: PathBuf, - /// Port to listen on. - #[arg(short, long, default_value = "3000")] - port: u16, - }, } /// CLI entry point. @@ -1647,12 +1396,6 @@ fn run(cli: Cli) -> Result<(), Box> { process::exit(code); } } - #[cfg(feature = "serve")] - Command::Dashboard(d) => run_dashboard(d, env_arg)?, - #[cfg(feature = "serve")] - Command::Coordinator(c) => run_coordinator(c, env_arg)?, - #[cfg(feature = "serve")] - Command::Worker(w) => run_worker_cmd(w)?, Command::Migrate { kind, path, @@ -1699,148 +1442,6 @@ fn run_migrate( Ok(()) } -#[cfg(feature = "serve")] -fn run_coordinator( - cmd: CoordinatorCommand, - env_arg: Option<&str>, -) -> Result<(), Box> { - match cmd { - CoordinatorCommand::Serve { - data_dir, - port, - bind, - max_lease_ttl_ms, - poll_timeout_ms, - sweep_interval_ms, - shared_secret, - tls_cert, - tls_key, - tls_client_ca, - } => { - #[cfg(feature = "persist-sqlite")] - { - let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); - let bind_addr: std::net::IpAddr = bind - .parse() - .map_err(|e| format!("invalid --bind address {bind:?}: {e}"))?; - let tls_config = - coordinator::ServerTlsPaths::from_optional(tls_cert, tls_key, tls_client_ca)?; - coordinator::run_serve( - resolved, - port, - bind_addr, - max_lease_ttl_ms, - poll_timeout_ms, - sweep_interval_ms, - shared_secret, - tls_config, - )?; - } - #[cfg(not(feature = "persist-sqlite"))] - { - let _ = ( - data_dir, - port, - bind, - max_lease_ttl_ms, - poll_timeout_ms, - sweep_interval_ms, - shared_secret, - tls_cert, - tls_key, - tls_client_ca, - ); - return Err("`coordinator serve` requires the `persist-sqlite` feature".into()); - } - } - CoordinatorCommand::Wait { - run_id, - data_dir, - poll_interval_ms, - max_wait_secs, - } => { - #[cfg(feature = "persist-sqlite")] - { - let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); - let exit_code = - coordinator::run_wait(resolved, run_id, poll_interval_ms, max_wait_secs)?; - if exit_code != 0 { - std::process::exit(exit_code); - } - } - #[cfg(not(feature = "persist-sqlite"))] - { - let _ = (run_id, data_dir, poll_interval_ms, max_wait_secs); - return Err("`coordinator wait` requires the `persist-sqlite` feature".into()); - } - } - } - Ok(()) -} - -#[cfg(feature = "serve")] -fn run_worker_cmd(cmd: WorkerCommand) -> Result<(), Box> { - match cmd { - WorkerCommand::Run { - coordinator, - worker_id, - lease_ttl_ms, - poll_timeout_ms, - shared_secret, - advertise_caps, - advertise_cap_versions, - tls_cert, - tls_key, - tls_server_ca, - } => { - let advertised = worker::parse_advertise_caps(advertise_caps.as_deref()); - let cap_versions = worker::parse_cap_versions(advertise_cap_versions.as_deref()); - let tls_config = - worker::ClientTlsPaths::from_optional(tls_cert, tls_key, tls_server_ca)?; - worker::run_worker( - coordinator, - worker_id, - lease_ttl_ms, - poll_timeout_ms, - shared_secret, - advertised, - cap_versions, - tls_config, - )?; - } - } - Ok(()) -} - -#[cfg(feature = "serve")] -fn run_dashboard( - cmd: DashboardCommand, - env_arg: Option<&str>, -) -> Result<(), Box> { - match cmd { - DashboardCommand::Serve { - data_dir, - port, - bind, - } => { - #[cfg(feature = "persist-sqlite")] - { - let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); - let bind_addr: std::net::IpAddr = bind - .parse() - .map_err(|e| format!("invalid --bind address {bind:?}: {e}"))?; - dashboard::run_serve(resolved, port, bind_addr)?; - } - #[cfg(not(feature = "persist-sqlite"))] - { - let _ = (data_dir, port, bind); - return Err("`dashboard serve` requires the `persist-sqlite` feature".into()); - } - } - } - Ok(()) -} - fn run_policy(cmd: PolicyCommand) -> i32 { use boruna_vm::policy_validate; match cmd { @@ -2682,10 +2283,6 @@ fn main() -> Int {{ println!("cycles: {}", harness.cycle()); println!("state: {}", harness.state()); } - #[cfg(feature = "serve")] - FrameworkCommand::Serve { file, port } => { - serve::run_serve(file, port)?; - } FrameworkCommand::Replay { file, log } => { let source = fs::read_to_string(&file)?; let log_json = fs::read_to_string(&log)?; @@ -3191,29 +2788,16 @@ fn run_workflow( // the conventional code (0/1/2) and skip the rest of // the local-run flow. if let Some(coord_url) = coordinator { - #[cfg(feature = "serve")] - { - let exit = crate::coordinator::run_remote( - &def, - &dir, - &policy_obj, - &coord_url, - coord_token.as_deref(), - coord_poll_interval_ms, - coord_max_wait_secs, - )?; - process::exit(exit); - } - #[cfg(not(feature = "serve"))] - { - let _ = ( - coord_url, - coord_token, - coord_poll_interval_ms, - coord_max_wait_secs, - ); - return Err("`workflow run --coordinator` requires the `serve` feature".into()); - } + let _ = ( + coord_url, + coord_token, + coord_poll_interval_ms, + coord_max_wait_secs, + ); + return Err("`workflow run --coordinator` is no longer supported — \ + distributed coordinator execution has been removed; \ + run workflows locally" + .into()); } let options = RunOptions { @@ -3498,29 +3082,10 @@ fn run_workflow( token, } => { if let Some(url) = coordinator { - #[cfg(feature = "serve")] - { - crate::coordinator::send_approve_remote( - &url, - coord_token.as_deref(), - &run_id, - &step_id, - "approved", - None, - token.as_deref().unwrap_or_default(), - )?; - println!( - "approval recorded for step '{step_id}' in run '{run_id}' \ - via coordinator {url}." - ); - } - #[cfg(not(feature = "serve"))] - { - let _ = (url, coord_token, token); - return Err( - "`workflow approve --coordinator` requires the `serve` feature".into(), - ); - } + let _ = (url, coord_token, token); + return Err("`workflow approve --coordinator` is no longer supported — \ + distributed coordinator execution has been removed" + .into()); } else { let _ = token; // local approve is operator-trusted; no gate token #[cfg(feature = "persist-sqlite")] @@ -3558,29 +3123,10 @@ fn run_workflow( token, } => { if let Some(url) = coordinator { - #[cfg(feature = "serve")] - { - crate::coordinator::send_approve_remote( - &url, - coord_token.as_deref(), - &run_id, - &step_id, - "rejected", - reason.as_deref(), - token.as_deref().unwrap_or_default(), - )?; - println!( - "rejection recorded for step '{step_id}' in run '{run_id}' \ - via coordinator {url}." - ); - } - #[cfg(not(feature = "serve"))] - { - let _ = (url, coord_token, reason, token); - return Err( - "`workflow reject --coordinator` requires the `serve` feature".into(), - ); - } + let _ = (url, coord_token, reason, token); + return Err("`workflow reject --coordinator` is no longer supported — \ + distributed coordinator execution has been removed" + .into()); } else { let _ = token; // local reject is operator-trusted; no gate token #[cfg(feature = "persist-sqlite")] @@ -3639,28 +3185,10 @@ fn run_workflow( .map_err(|e| format!("--payload is not valid JSON: {e}"))?; if let Some(url) = coordinator { - #[cfg(feature = "serve")] - { - crate::coordinator::send_trigger_remote( - &url, - coord_token.as_deref(), - &run_id, - &step_id, - &token, - &payload_str, - )?; - println!( - "trigger recorded for step '{step_id}' in run '{run_id}' \ - via coordinator {url}." - ); - } - #[cfg(not(feature = "serve"))] - { - let _ = (url, coord_token); - return Err( - "`workflow trigger --coordinator` requires the `serve` feature".into(), - ); - } + let _ = (url, coord_token); + return Err("`workflow trigger --coordinator` is no longer supported — \ + distributed coordinator execution has been removed" + .into()); } else { #[cfg(feature = "persist-sqlite")] { @@ -4702,20 +4230,6 @@ fn run_evidence( parallelism, )?; } - EvidenceCommand::Serve { dir, port } => { - #[cfg(feature = "serve")] - { - let rt = tokio::runtime::Runtime::new()?; - rt.block_on(evidence_serve::serve(&dir, port))?; - } - #[cfg(not(feature = "serve"))] - { - let _ = (dir, port); - return Err("`evidence serve` requires the `serve` feature — \ - build with: cargo build --features boruna-cli/serve" - .into()); - } - } EvidenceCommand::Diff { bundle_a, bundle_b, diff --git a/crates/llmvm-cli/src/serve.rs b/crates/llmvm-cli/src/serve.rs deleted file mode 100644 index eb6648f..0000000 --- a/crates/llmvm-cli/src/serve.rs +++ /dev/null @@ -1,506 +0,0 @@ -use std::fs; -use std::path::PathBuf; -use std::sync::{Arc, Mutex}; - -use axum::extract::{Form, State}; -use axum::response::Html; -use axum::routing::{get, post}; -use axum::Router; - -use boruna_bytecode::Value; -use boruna_compiler::ast::{BinOp, Block, Expr, Item, Pattern, Stmt}; -use boruna_framework::runtime::AppMessage; -use boruna_framework::testing::TestHarness; - -use crate::parse_message; - -// --------------------------------------------------------------------------- -// Shared state -// --------------------------------------------------------------------------- - -struct AppContext { - harness: TestHarness, - source_path: PathBuf, - message_tags: Vec, - state_fields: Vec, - cycle_log_display: Vec, -} - -#[derive(Clone)] -struct CycleEntry { - cycle: u64, - tag: String, - state_after: String, - effects: Vec, -} - -type SharedState = Arc>; - -// --------------------------------------------------------------------------- -// Entry point -// --------------------------------------------------------------------------- - -#[tokio::main] -pub async fn run_serve(file: PathBuf, port: u16) -> Result<(), Box> { - let ctx = build_context(&file)?; - let shared: SharedState = Arc::new(Mutex::new(ctx)); - - let app = Router::new() - .route("/", get(handle_index)) - .route("/send", post(handle_send)) - .route("/reset", post(handle_reset)) - .route("/api/state", get(handle_api_state)) - .with_state(shared); - - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port)); - eprintln!("serving on http://{addr}"); - - let listener = tokio::net::TcpListener::bind(addr).await?; - axum::serve(listener, app).await?; - Ok(()) -} - -fn build_context(file: &PathBuf) -> Result> { - let source = fs::read_to_string(file)?; - let harness = TestHarness::from_source(&source)?; - let message_tags = discover_message_tags(&source); - let state_fields = discover_state_fields(&source); - Ok(AppContext { - harness, - source_path: file.clone(), - message_tags, - state_fields, - cycle_log_display: Vec::new(), - }) -} - -// --------------------------------------------------------------------------- -// Handlers -// --------------------------------------------------------------------------- - -async fn handle_index(State(state): State) -> Html { - let ctx = state.lock().unwrap(); - Html(render_page(&ctx)) -} - -#[derive(serde::Deserialize)] -pub struct SendForm { - tag: String, - payload: Option, -} - -async fn handle_send(State(state): State, Form(form): Form) -> Html { - let mut ctx = state.lock().unwrap(); - - let payload_str = form.payload.as_deref().unwrap_or("0"); - let (_, payload) = parse_message(&format!("_:{payload_str}")); - let msg = AppMessage::new(&form.tag, payload); - - let entry = match ctx.harness.send(msg) { - Ok((state_val, effects)) => { - let effect_strs: Vec = effects - .iter() - .map(|e| e.kind.as_str().to_string()) - .collect(); - let cycle = ctx.harness.cycle(); - let state_after = format_state(&state_val, &ctx.state_fields); - CycleEntry { - cycle, - tag: form.tag.clone(), - state_after, - effects: effect_strs, - } - } - Err(e) => { - let cycle = ctx.harness.cycle(); - CycleEntry { - cycle, - tag: format!("ERROR: {}", form.tag), - state_after: format!("{e}"), - effects: vec![], - } - } - }; - ctx.cycle_log_display.push(entry); - // Keep last 20 entries - if ctx.cycle_log_display.len() > 20 { - let start = ctx.cycle_log_display.len() - 20; - ctx.cycle_log_display = ctx.cycle_log_display[start..].to_vec(); - } - - Html(render_page(&ctx)) -} - -async fn handle_reset(State(state): State) -> Html { - let mut ctx = state.lock().unwrap(); - match build_context(&ctx.source_path.clone()) { - Ok(new_ctx) => *ctx = new_ctx, - Err(e) => { - ctx.cycle_log_display.push(CycleEntry { - cycle: 0, - tag: "RESET ERROR".into(), - state_after: format!("{e}"), - effects: vec![], - }); - } - } - Html(render_page(&ctx)) -} - -async fn handle_api_state(State(state): State) -> axum::Json { - let ctx = state.lock().unwrap(); - axum::Json(serde_json::json!({ - "cycle": ctx.harness.cycle(), - "state": format!("{}", ctx.harness.state()), - "snapshot": ctx.harness.snapshot(), - })) -} - -// --------------------------------------------------------------------------- -// HTML rendering -// --------------------------------------------------------------------------- - -fn escape_html(s: &str) -> String { - s.replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) -} - -fn format_state(val: &Value, field_names: &[String]) -> String { - match val { - Value::Record { fields, .. } => { - let pairs: Vec = fields - .iter() - .enumerate() - .map(|(i, v)| { - let label = field_names - .get(i) - .cloned() - .unwrap_or_else(|| format!("[{i}]")); - format!("{label}: {v}") - }) - .collect(); - pairs.join(", ") - } - other => format!("{other}"), - } -} - -fn render_page(ctx: &AppContext) -> String { - let state_html = render_state_table(ctx.harness.state(), &ctx.state_fields); - - let buttons_html: String = ctx.message_tags.iter().map(|tag| { - format!( - r#"
- - - -
"#, - tag = escape_html(tag), - ) - }).collect(); - - let view_html = match ctx.harness.view() { - Ok(v) => escape_html(&format!("{v}")), - Err(e) => format!("view error: {}", escape_html(&e.to_string())), - }; - - let log_html: String = ctx - .cycle_log_display - .iter() - .rev() - .map(|entry| { - let effects = if entry.effects.is_empty() { - String::new() - } else { - format!( - " [{}]", - escape_html(&entry.effects.join(", ")) - ) - }; - format!( - "{}{}{}{}", - entry.cycle, - escape_html(&entry.tag), - escape_html(&entry.state_after), - effects, - ) - }) - .collect(); - - format!( - r##" - - - -Boruna — {title} - - - -

Boruna Framework

-

{file} — cycle {cycle}

- -
-

State

- {state_html} -
- -
-

Messages

-
{buttons_html}
-
- - - -
-
- -
-

View

- 
{view_html}
-
- -
-

Cycle Log

- - - {log_html} -
#MessageState AfterEffects
-
- -
-
-
-
-
- -"##, - title = escape_html( - &ctx.source_path - .file_stem() - .map(|s| s.to_string_lossy().to_string()) - .unwrap_or_default() - ), - file = escape_html(&ctx.source_path.display().to_string()), - cycle = ctx.harness.cycle(), - state_html = state_html, - buttons_html = buttons_html, - view_html = view_html, - log_html = log_html, - ) -} - -fn render_state_table(val: &Value, field_names: &[String]) -> String { - match val { - Value::Record { fields, .. } => { - let rows: String = fields - .iter() - .enumerate() - .map(|(i, v)| { - let label = field_names - .get(i) - .cloned() - .unwrap_or_else(|| format!("[{i}]")); - format!( - "{}{}", - escape_html(&label), - escape_html(&format!("{v}")), - ) - }) - .collect(); - format!("{rows}
") - } - other => format!("
{}
", escape_html(&format!("{other}"))), - } -} - -// --------------------------------------------------------------------------- -// Message tag auto-discovery -// --------------------------------------------------------------------------- - -fn discover_message_tags(source: &str) -> Vec { - let tokens = match boruna_compiler::lexer::lex(source) { - Ok(t) => t, - Err(_) => return vec![], - }; - let program = match boruna_compiler::parser::parse(tokens) { - Ok(p) => p, - Err(_) => return vec![], - }; - - let mut tags = Vec::new(); - - // Find the update() function - for item in &program.items { - if let Item::Function(f) = item { - if f.name == "update" { - collect_tags_from_block(&f.body, &mut tags); - } - } - } - - tags.sort(); - tags.dedup(); - tags -} - -fn collect_tags_from_block(block: &Block, tags: &mut Vec) { - for stmt in &block.stmts { - match stmt { - Stmt::Let { value, .. } => collect_tags_from_expr(value, tags), - Stmt::Assign { value, .. } => collect_tags_from_expr(value, tags), - Stmt::Expr(e) => collect_tags_from_expr(e, tags), - Stmt::Return(Some(e)) => collect_tags_from_expr(e, tags), - Stmt::Return(None) => {} - Stmt::While { condition, body } => { - collect_tags_from_expr(condition, tags); - collect_tags_from_block(body, tags); - } - Stmt::For { iter, body, .. } => { - collect_tags_from_expr(iter, tags); - collect_tags_from_block(body, tags); - } - } - } -} - -fn collect_tags_from_expr(expr: &Expr, tags: &mut Vec) { - match expr { - // msg.tag == "literal" or "literal" == msg.tag - Expr::Binary { - op: BinOp::Eq, - left, - right, - } => { - if is_msg_tag_access(left) { - if let Expr::StringLit(s) = right.as_ref() { - tags.push(s.clone()); - } - } - if is_msg_tag_access(right) { - if let Expr::StringLit(s) = left.as_ref() { - tags.push(s.clone()); - } - } - collect_tags_from_expr(left, tags); - collect_tags_from_expr(right, tags); - } - - // match msg.tag { "add" => ..., "remove" => ... } - Expr::Match { value, arms } => { - let is_tag_match = is_msg_tag_access(value); - for arm in arms { - if is_tag_match { - if let Pattern::StringLit(s) = &arm.pattern { - tags.push(s.clone()); - } - } - collect_tags_from_expr(&arm.body, tags); - } - collect_tags_from_expr(value, tags); - } - - Expr::If { - condition, - then_block, - else_block, - } => { - collect_tags_from_expr(condition, tags); - collect_tags_from_block(then_block, tags); - if let Some(eb) = else_block { - collect_tags_from_block(eb, tags); - } - } - Expr::Binary { left, right, .. } => { - collect_tags_from_expr(left, tags); - collect_tags_from_expr(right, tags); - } - Expr::Unary { expr, .. } => collect_tags_from_expr(expr, tags), - Expr::Call { func, args } => { - collect_tags_from_expr(func, tags); - for a in args { - collect_tags_from_expr(a, tags); - } - } - Expr::FieldAccess { object, .. } => collect_tags_from_expr(object, tags), - Expr::Record { fields, spread, .. } => { - for (_, e) in fields { - collect_tags_from_expr(e, tags); - } - if let Some(s) = spread { - collect_tags_from_expr(s, tags); - } - } - Expr::List(items) => { - for e in items { - collect_tags_from_expr(e, tags); - } - } - Expr::SomeExpr(e) | Expr::OkExpr(e) | Expr::ErrExpr(e) | Expr::Spawn(e) | Expr::Emit(e) => { - collect_tags_from_expr(e, tags); - } - Expr::Send { target, message } => { - collect_tags_from_expr(target, tags); - collect_tags_from_expr(message, tags); - } - Expr::Block(b) => collect_tags_from_block(b, tags), - Expr::EnumVariant { - payload: Some(e), .. - } => collect_tags_from_expr(e, tags), - _ => {} - } -} - -/// Returns true if the expression is `msg.tag` (or any ident ending in .tag). -fn is_msg_tag_access(expr: &Expr) -> bool { - matches!(expr, Expr::FieldAccess { field, .. } if field == "tag") -} - -// --------------------------------------------------------------------------- -// State field discovery -// --------------------------------------------------------------------------- - -fn discover_state_fields(source: &str) -> Vec { - let tokens = match boruna_compiler::lexer::lex(source) { - Ok(t) => t, - Err(_) => return vec![], - }; - let program = match boruna_compiler::parser::parse(tokens) { - Ok(p) => p, - Err(_) => return vec![], - }; - - for item in &program.items { - if let Item::TypeDef(td) = item { - if td.name == "State" { - if let boruna_compiler::ast::TypeDefKind::Record(fields) = &td.kind { - return fields.iter().map(|(name, _)| name.clone()).collect(); - } - } - } - } - - vec![] -} diff --git a/crates/llmvm-cli/src/worker.rs b/crates/llmvm-cli/src/worker.rs deleted file mode 100644 index d60bac8..0000000 --- a/crates/llmvm-cli/src/worker.rs +++ /dev/null @@ -1,722 +0,0 @@ -//! Distributed-execution worker (sprint `0.5-S2b`). Polls a -//! coordinator over HTTP for claimable steps, compiles + runs -//! the step's `.ax` source, reports the result back. -//! -//! See `docs/design-coordinator-worker-http.md` and -//! `docs/architecture-coordinator-worker-http.md`. - -use std::path::PathBuf; -use std::time::Duration; - -use boruna_bytecode::{compute_capability_set_hash, Value}; -use boruna_vm::capability_gateway::{CapabilityGateway, Policy}; -use boruna_vm::vm::Vm; -use sha2::{Digest, Sha256}; - -use crate::coordinator::{ - CapabilityAdvertisement, CompleteRequest, ErrorBody, FailRequest, HeartbeatRequest, - RegisterRequest, RegisterResponse, WorkItem, -}; - -const HEARTBEAT_INTERVAL_MS: u64 = 10_000; - -/// File-path bundle for the worker's mTLS client config (sprint -/// `W6-A`). All three paths required together — partial sets are -/// a startup error so half-configured TLS doesn't silently fall -/// back to plaintext. -#[derive(Debug, Clone)] -pub struct ClientTlsPaths { - pub cert: PathBuf, - pub key: PathBuf, - pub server_ca: PathBuf, -} - -impl ClientTlsPaths { - pub fn from_optional( - cert: Option, - key: Option, - server_ca: Option, - ) -> Result, Box> { - match (cert, key, server_ca) { - (None, None, None) => Ok(None), - (Some(cert), Some(key), Some(server_ca)) => Ok(Some(Self { - cert, - key, - server_ca, - })), - _ => Err("--tls-cert, --tls-key, --tls-server-ca must all be provided together".into()), - } - } -} - -/// Read a cert PEM and key PEM and concatenate them in the format -/// `reqwest::Identity::from_pem` expects (cert blocks then key -/// block, separated by newlines). Reqwest's parser is fine with -/// multi-cert chains followed by a single key. -fn read_pem_pair( - cert: &std::path::Path, - key: &std::path::Path, -) -> Result, Box> { - let mut cert_bytes = - std::fs::read(cert).map_err(|e| format!("read --tls-cert {}: {e}", cert.display()))?; - let key_bytes = - std::fs::read(key).map_err(|e| format!("read --tls-key {}: {e}", key.display()))?; - if !cert_bytes.ends_with(b"\n") { - cert_bytes.push(b'\n'); - } - cert_bytes.extend_from_slice(&key_bytes); - Ok(cert_bytes) -} - -/// Conditionally attach the `Authorization: Bearer ` header -/// to a reqwest request when a shared-secret is configured. When -/// `secret` is `None`, the request is returned unchanged — the -/// pre-0.5-S3 no-auth behavior. Sprint `0.5-S3`. -fn add_bearer(req: reqwest::RequestBuilder, secret: &Option) -> reqwest::RequestBuilder { - match secret { - Some(s) => req.bearer_auth(s), - None => req, - } -} - -/// Sprint `W3-A` — parse a comma-separated `--advertise-caps` -/// value into the wire-shape `Vec`. Empty or whitespace-only -/// input maps to `None` (full-fleet behavior, matches the absent -/// flag). Trims whitespace around each element and drops empty -/// fragments produced by trailing commas. -pub fn parse_advertise_caps(raw: Option<&str>) -> Option> { - let s = raw?.trim(); - if s.is_empty() { - return None; - } - let names: Vec = s - .split(',') - .map(|p| p.trim().to_string()) - .filter(|p| !p.is_empty()) - .collect(); - if names.is_empty() { - None - } else { - Some(names) - } -} - -/// Parse `--advertise-cap-versions` value of the form -/// `"llm.call=2.0,net.fetch=1.5"` into a `BTreeMap`. -/// Unknown or malformed pairs are silently dropped. -pub fn parse_cap_versions(raw: Option<&str>) -> std::collections::BTreeMap { - let Some(s) = raw else { - return std::collections::BTreeMap::new(); - }; - s.split(',') - .filter_map(|pair| { - let (k, v) = pair.split_once('=')?; - let k = k.trim().to_string(); - let v = v.trim().to_string(); - if k.is_empty() || v.is_empty() { - None - } else { - Some((k, v)) - } - }) - .collect() -} - -#[derive(Clone)] -struct WorkerHandle { - coord_url: String, - worker_id: String, - session_token: String, - client: reqwest::Client, - lease_ttl_ms: u64, - /// Reserved for 0.5-S2c: a future tighter coupling between - /// the worker's claim long-poll and the coordinator's - /// poll_timeout_ms cap. Today the worker's reqwest client - /// timeout is poll_timeout_ms + 30 s buffer. - #[allow(dead_code)] - poll_timeout_ms: u64, - /// Shared-secret bearer token (sprint `0.5-S3`). When - /// `Some`, every HTTP request to the coordinator carries - /// `Authorization: Bearer `. When `None`, no - /// auth header is sent — only works when the coordinator - /// has no secret configured (legacy/loopback deployments). - shared_secret: Option, -} - -/// Parse a `--coordinator` value that may carry one or more -/// comma-separated coord URLs into a normalized `Vec`. -/// Sprint W2: multi-URL workers fail over at registration time. -/// -/// Whitespace around commas is trimmed; empty entries are -/// dropped; trailing slashes are removed for stable comparison -/// in error messages. Returns `Err` if the value parses to zero -/// usable URLs (e.g. `--coordinator " , "` or empty string). -pub fn parse_coordinator_urls(raw: &str) -> Result, String> { - let urls: Vec = raw - .split(',') - .map(|s| s.trim().trim_end_matches('/').to_string()) - .filter(|s| !s.is_empty()) - .collect(); - if urls.is_empty() { - return Err(format!( - "--coordinator must be one or more comma-separated URLs, got '{raw}'" - )); - } - Ok(urls) -} - -#[allow(clippy::too_many_arguments)] -#[tokio::main] -pub async fn run_worker( - coordinator: String, - worker_id: Option, - lease_ttl_ms: u64, - poll_timeout_ms: u64, - shared_secret: Option, - advertised_capabilities: Option>, - cap_versions: std::collections::BTreeMap, - tls_paths: Option, -) -> Result<(), Box> { - let coord_urls = parse_coordinator_urls(&coordinator)?; - let mut builder = reqwest::Client::builder() - // Long-poll buffer: client timeout MUST be greater than - // server-side poll_timeout_ms so a 30s long-poll doesn't - // trip a 10s default client timeout. - .timeout(Duration::from_millis(poll_timeout_ms + 30_000)); - if let Some(tls) = &tls_paths { - let identity_pem = read_pem_pair(&tls.cert, &tls.key)?; - let identity = reqwest::Identity::from_pem(&identity_pem) - .map_err(|e| format!("client cert+key parse: {e}"))?; - let ca_pem = std::fs::read(&tls.server_ca) - .map_err(|e| format!("read --tls-server-ca {}: {e}", tls.server_ca.display()))?; - let ca = - reqwest::Certificate::from_pem(&ca_pem).map_err(|e| format!("server CA parse: {e}"))?; - builder = builder - .use_rustls_tls() - .identity(identity) - .add_root_certificate(ca); - eprintln!( - "worker mTLS: cert={} key={} server-ca={}", - tls.cert.display(), - tls.key.display(), - tls.server_ca.display() - ); - } - let client = builder.build()?; - - let capability_set_hash = compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - - // Sprint W2: register against the first reachable URL in the - // operator-supplied list. After registration succeeds, we - // "stick" to that coord for the lifetime of this worker - // process — `session_token` is per-coord, so mid-session - // failover would require re-register against a different - // coord. Operators recover from a sticky-coord crash by - // restarting the worker; on next start it picks the next - // healthy URL. This is the standard k8s liveness-probe model. - // - // Sprint W3-A: every register attempt carries the same - // `advertised_capabilities` payload so any winning coord - // applies the same placement filter. - let mut last_err: Option = None; - let (winning_url, reg) = { - let mut found: Option<(String, RegisterResponse)> = None; - for candidate in &coord_urls { - let register_url = format!("{}/api/workers/register", candidate); - let req = client.post(®ister_url).json(&RegisterRequest { - worker_id: worker_id.clone(), - capability_set_hash: capability_set_hash.clone(), - advertised_capabilities: advertised_capabilities.as_ref().map(|names| { - names - .iter() - .map(|n| { - if let Some(ver) = cap_versions.get(n) { - CapabilityAdvertisement::Versioned { - name: n.clone(), - version: ver.clone(), - } - } else { - CapabilityAdvertisement::Legacy(n.clone()) - } - }) - .collect() - }), - }); - match add_bearer(req, &shared_secret).send().await { - Ok(reg_resp) => { - let status = reg_resp.status(); - if !status.is_success() { - let body: ErrorBody = reg_resp.json().await.unwrap_or(ErrorBody { - protocol_version: 1, - error_kind: "coord.invalid_request".into(), - message: "registration failed; could not parse error body".into(), - current_claim_id: None, - current_status: None, - expected_hash: None, - max_bytes: None, - }); - last_err = Some(format!( - "register against {candidate} returned {status}: {} ({})", - body.error_kind, body.message - )); - if coord_urls.len() > 1 { - eprintln!( - "worker register: {candidate} returned {status}; trying next" - ); - } - continue; - } - let reg: RegisterResponse = reg_resp.json().await?; - found = Some((candidate.clone(), reg)); - break; - } - Err(e) => { - last_err = Some(format!("connect to {candidate} failed: {e}")); - if coord_urls.len() > 1 { - eprintln!( - "worker register: connect to {candidate} failed ({e}); trying next" - ); - } - continue; - } - } - } - match found { - Some(t) => t, - None => { - let detail = last_err.unwrap_or_else(|| "no candidates tried".into()); - return Err(format!( - "worker could not register against any of {} coord URL(s): {detail}", - coord_urls.len() - ) - .into()); - } - } - }; - eprintln!( - "worker {} registered with coordinator {} (selected from {} candidate(s))", - reg.worker_id, - winning_url, - coord_urls.len() - ); - - let handle = WorkerHandle { - coord_url: winning_url, - worker_id: reg.worker_id, - session_token: reg.session_token, - client, - lease_ttl_ms, - poll_timeout_ms, - shared_secret, - }; - - // Spawn heartbeat task. - let hb = handle.clone(); - let hb_task = tokio::spawn(async move { - let mut tick = tokio::time::interval(Duration::from_millis(HEARTBEAT_INTERVAL_MS)); - // First tick fires immediately; skip it. - tick.tick().await; - loop { - tick.tick().await; - let req = hb - .client - .post(format!("{}/api/workers/heartbeat", hb.coord_url)) - .json(&HeartbeatRequest { - worker_id: hb.worker_id.clone(), - session_token: hb.session_token.clone(), - }); - let _ = add_bearer(req, &hb.shared_secret).send().await; - } - }); - - let result = main_loop(handle).await; - hb_task.abort(); - result -} - -async fn main_loop(handle: WorkerHandle) -> Result<(), Box> { - // Floor on the empty-claim retry interval. The coordinator - // does its own server-side long-poll; if a misconfigured or - // proxied coordinator returns 204 instantly, this floor - // prevents the worker from CPU-spinning at full rate. - // Adversarial review caught the busy-spin (F2) — without - // this sleep, an instant 204 produces ~thousands of HTTP - // requests per second indefinitely. - let empty_backoff = Duration::from_millis(100); - loop { - match claim_one(&handle).await { - Ok(None) => { - tokio::time::sleep(empty_backoff).await; - } - Ok(Some(work)) => { - let result = execute_step(&work); - match result { - Ok((output_json, output_hash)) => { - report_complete(&handle, &work, output_json, output_hash).await?; - } - Err(error_msg) => { - report_fail(&handle, &work, error_msg).await?; - } - } - } - Err(e) => { - eprintln!("worker {} claim error: {e}", handle.worker_id); - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - } -} - -async fn claim_one(handle: &WorkerHandle) -> Result, Box> { - let url = format!( - "{}/api/work/claim?worker_id={}&session_token={}&lease_ttl_ms={}", - handle.coord_url, - urlencoding_simple(&handle.worker_id), - urlencoding_simple(&handle.session_token), - handle.lease_ttl_ms - ); - let resp = add_bearer(handle.client.get(&url), &handle.shared_secret) - .send() - .await?; - let status = resp.status(); - if status.as_u16() == 204 { - return Ok(None); - } - if !status.is_success() { - let body: ErrorBody = resp.json().await.unwrap_or(ErrorBody { - protocol_version: 1, - error_kind: "coord.invalid_request".into(), - message: format!("claim failed with {status}"), - current_claim_id: None, - current_status: None, - expected_hash: None, - max_bytes: None, - }); - return Err(format!("claim {status}: {} ({})", body.error_kind, body.message).into()); - } - let item: WorkItem = resp.json().await?; - Ok(Some(item)) -} - -/// Compile + run the step's `.ax` source under the work item's -/// policy. Returns `(output_json, output_hash)` on success or an -/// error message on failure. -/// -/// Policy parsing goes through the strict validator from sprint -/// `0.4-S15` (`boruna_vm::policy_validate::parse`) so workers -/// reject the same shapes the CLI rejects, with the same stable -/// `error_kind` strings. This closes the validate-vs-execute -/// drift surface at the worker boundary. -fn execute_step(work: &WorkItem) -> Result<(String, String), String> { - let policy: Policy = boruna_vm::policy_validate::parse(&work.policy_json) - .map_err(|e| format!("policy parse: {e}"))?; - let module = boruna_compiler::compile(&work.step_id, &work.source) - .map_err(|e| format!("compile: {e}"))?; - let gateway = CapabilityGateway::new(policy); - let mut vm = Vm::new(module, gateway); - let value = vm.run().map_err(|e| format!("runtime: {e}"))?; - let output_json = value_to_json(&value); - let mut hasher = Sha256::new(); - hasher.update(output_json.as_bytes()); - let digest = hasher.finalize(); - let mut hex = String::with_capacity(7 + 64); - hex.push_str("sha256:"); - for b in digest { - hex.push_str(&format!("{b:02x}")); - } - Ok((output_json, hex)) -} - -fn value_to_json(v: &Value) -> String { - // For the MVP: serialize via the existing `format_value` - // shape used by the MCP `boruna_run` tool. - match v { - Value::Int(n) => n.to_string(), - Value::Float(f) => f.to_string(), - Value::String(s) => serde_json::to_string(s).unwrap(), - Value::Bool(b) => b.to_string(), - Value::Unit => "null".into(), - Value::None => r#"{"option":"None"}"#.into(), - _ => serde_json::to_string(&format!("{v:?}")).unwrap(), - } -} - -async fn report_complete( - handle: &WorkerHandle, - work: &WorkItem, - output_json: String, - output_hash: String, -) -> Result<(), Box> { - let url = format!("{}/api/work/complete", handle.coord_url); - let body = CompleteRequest { - worker_id: handle.worker_id.clone(), - session_token: handle.session_token.clone(), - run_id: work.run_id.clone(), - step_id: work.step_id.clone(), - claim_id: work.claim_id, - output_json, - output_hash, - attempt_count: 1, - }; - let resp = add_bearer(handle.client.post(&url).json(&body), &handle.shared_secret) - .send() - .await?; - if resp.status().is_success() { - return Ok(()); - } - let status = resp.status(); - let err: ErrorBody = resp.json().await.unwrap_or(ErrorBody { - protocol_version: 1, - error_kind: "coord.invalid_request".into(), - message: format!("complete failed with {status}"), - current_claim_id: None, - current_status: None, - expected_hash: None, - max_bytes: None, - }); - eprintln!( - "worker {} complete returned {}: {} ({})", - handle.worker_id, status, err.error_kind, err.message - ); - // Adversarial-review F1: do NOT silently swallow non-success - // responses. Distinguish three cases: - // - // 1. `coord.lease_expired` (409) — per ADR 002 the - // coordinator has already re-dispatched the step to - // another worker. Discard our work; do NOT report_fail - // (that would race with the new claim). Just log + move - // on. - // - // 2. `coord.output_too_large` (413) and other - // output-validation errors — the work is genuinely done - // but the coordinator can't accept the output. Re-running - // the same source produces the same output, so retry is - // pointless. Report as a step failure so the run can - // progress (or terminal-fail per retry policy). - // - // 3. Anything else (5xx, network error after retry, unknown - // error_kind) — best-effort: report_fail so the step - // doesn't strand. Caller's retry policy decides whether - // to re-attempt. - match err.error_kind.as_str() { - "coord.lease_expired" => Ok(()), - _ => { - // Map the failure into a step-fail report so the row - // doesn't sit in Running until lease expiry. - let fail_msg = format!( - "report_complete rejected by coordinator: {} ({})", - err.error_kind, err.message - ); - report_fail(handle, work, fail_msg).await - } - } -} - -async fn report_fail( - handle: &WorkerHandle, - work: &WorkItem, - error_msg: String, -) -> Result<(), Box> { - let url = format!("{}/api/work/fail", handle.coord_url); - let body = FailRequest { - worker_id: handle.worker_id.clone(), - session_token: handle.session_token.clone(), - run_id: work.run_id.clone(), - step_id: work.step_id.clone(), - claim_id: work.claim_id, - error_msg, - attempt_count: 1, - }; - let resp = add_bearer(handle.client.post(&url).json(&body), &handle.shared_secret) - .send() - .await?; - if !resp.status().is_success() { - let status = resp.status(); - eprintln!("worker {} fail returned {}", handle.worker_id, status); - } - Ok(()) -} - -/// Minimal URL-encoding for the small alphabet our worker_ids -/// and session_tokens use (alphanumerics, hyphens, underscores). -/// Avoids pulling a urlencode dep. -fn urlencoding_simple(s: &str) -> String { - s.chars() - .map(|c| match c { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(), - _ => format!("%{:02X}", c as u32), - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn execute_pure_function_returns_int_output() { - let work = WorkItem { - protocol_version: 1, - run_id: "r".into(), - step_id: "s".into(), - claim_id: 1, - lease_expires_at_ms: 0, - source: "fn main() -> Int { 42 }\n".into(), - policy_json: r#"{"default_allow":true}"#.into(), - inputs_json: None, - }; - let (output_json, output_hash) = execute_step(&work).unwrap(); - assert_eq!(output_json, "42"); - assert!(output_hash.starts_with("sha256:")); - } - - #[test] - fn execute_returns_deterministic_hash_for_same_input() { - let work = WorkItem { - protocol_version: 1, - run_id: "r".into(), - step_id: "s".into(), - claim_id: 1, - lease_expires_at_ms: 0, - source: "fn main() -> Int { 1 + 2 }\n".into(), - policy_json: r#"{"default_allow":true}"#.into(), - inputs_json: None, - }; - let (out1, hash1) = execute_step(&work).unwrap(); - let (out2, hash2) = execute_step(&work).unwrap(); - assert_eq!(out1, out2); - assert_eq!(hash1, hash2); - } - - #[test] - fn execute_compile_error_returns_err() { - let work = WorkItem { - protocol_version: 1, - run_id: "r".into(), - step_id: "s".into(), - claim_id: 1, - lease_expires_at_ms: 0, - source: "@@@ not valid".into(), - policy_json: r#"{"default_allow":true}"#.into(), - inputs_json: None, - }; - let err = execute_step(&work).unwrap_err(); - assert!(err.contains("compile")); - } - - #[test] - fn urlencoding_passes_through_safe_chars() { - assert_eq!(urlencoding_simple("wkr-abc123"), "wkr-abc123"); - assert_eq!(urlencoding_simple("hello world"), "hello%20world"); - } - - // Sprint W2 — coordinator URL parsing for HA failover. - - #[test] - fn parse_single_coord_url_keeps_one_entry() { - let urls = parse_coordinator_urls("http://coord:8090").unwrap(); - assert_eq!(urls, vec!["http://coord:8090".to_string()]); - } - - #[test] - fn parse_strips_trailing_slash_for_stable_logging() { - let urls = parse_coordinator_urls("http://coord:8090/").unwrap(); - assert_eq!(urls, vec!["http://coord:8090".to_string()]); - } - - #[test] - fn parse_multiple_coord_urls_preserves_order() { - let urls = - parse_coordinator_urls("http://coord-1:8090,http://coord-2:8090,http://coord-3:8090") - .unwrap(); - assert_eq!( - urls, - vec![ - "http://coord-1:8090".to_string(), - "http://coord-2:8090".to_string(), - "http://coord-3:8090".to_string(), - ] - ); - } - - #[test] - fn parse_tolerates_whitespace_around_commas() { - let urls = parse_coordinator_urls("http://a:1 , http://b:2 , http://c:3").unwrap(); - assert_eq!( - urls, - vec![ - "http://a:1".to_string(), - "http://b:2".to_string(), - "http://c:3".to_string(), - ] - ); - } - - #[test] - fn parse_drops_empty_entries() { - let urls = parse_coordinator_urls("http://a:1,,http://b:2").unwrap(); - assert_eq!( - urls, - vec!["http://a:1".to_string(), "http://b:2".to_string()] - ); - } - - #[test] - fn parse_empty_string_is_an_error() { - assert!(parse_coordinator_urls("").is_err()); - assert!(parse_coordinator_urls(" , ").is_err()); - assert!(parse_coordinator_urls(",,,").is_err()); - } - - // Sprint W3-A — advertised capabilities parsing. - - #[test] - fn parse_advertise_caps_absent_or_empty_returns_none() { - assert_eq!(parse_advertise_caps(None), None); - assert_eq!(parse_advertise_caps(Some("")), None); - assert_eq!(parse_advertise_caps(Some(" ")), None); - assert_eq!(parse_advertise_caps(Some(",,, ,")), None); - } - - #[test] - fn parse_advertise_caps_splits_and_trims() { - assert_eq!( - parse_advertise_caps(Some("net.fetch, db.query ,fs.read")), - Some(vec![ - "net.fetch".into(), - "db.query".into(), - "fs.read".into() - ]) - ); - } - - // post1/scheduler-registry-rolling — cap version parsing. - - #[test] - fn parse_cap_versions_empty_returns_empty_map() { - assert!(parse_cap_versions(None).is_empty()); - assert!(parse_cap_versions(Some("")).is_empty()); - } - - #[test] - fn parse_cap_versions_single() { - let m = parse_cap_versions(Some("llm.call=2.0")); - assert_eq!(m.get("llm.call").map(|s| s.as_str()), Some("2.0")); - assert_eq!(m.len(), 1); - } - - #[test] - fn parse_cap_versions_multiple() { - let m = parse_cap_versions(Some("llm.call=2.0,net.fetch=1.5")); - assert_eq!(m.get("llm.call").map(|s| s.as_str()), Some("2.0")); - assert_eq!(m.get("net.fetch").map(|s| s.as_str()), Some("1.5")); - assert_eq!(m.len(), 2); - } -} diff --git a/crates/llmvm-cli/tests/cli_coordinator_mtls.rs b/crates/llmvm-cli/tests/cli_coordinator_mtls.rs deleted file mode 100644 index 40f086d..0000000 --- a/crates/llmvm-cli/tests/cli_coordinator_mtls.rs +++ /dev/null @@ -1,440 +0,0 @@ -//! End-to-end mTLS surface integration tests for the -//! coordinator (sprint `W6-A`). Generates a self-signed CA + -//! server cert + client cert in a tempdir using `rcgen`, spins -//! up `boruna coordinator serve` with mTLS enabled, and asserts -//! the four required adversarial properties: -//! -//! 1. A connection without a client cert is rejected at the TLS -//! handshake. -//! 2. A connection with a client cert from a DIFFERENT CA is -//! rejected at the TLS handshake. -//! 3. A valid client cert succeeds; the cert subject CN drives -//! the recorded `worker_id` on registration. -//! 4. A valid cert with a body `worker_id` that does NOT match -//! the cert CN returns 401 `coord.identity_mismatch`. -//! -//! Only compiled when `--features serve` is enabled. - -#![cfg(feature = "serve")] - -use std::io::Write; -use std::net::{TcpListener, TcpStream}; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use boruna_orchestrator::persistence::{ - RunCheckpointStore, RunRow, RunStatus, StepCheckpoint, StepStatus, -}; - -fn boruna_bin() -> &'static str { - env!("CARGO_BIN_EXE_boruna") -} - -fn pick_free_port() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); - let port = listener.local_addr().unwrap().port(); - drop(listener); - port -} - -fn wait_for_server(port: u16) { - let deadline = Instant::now() + Duration::from_secs(15); - while Instant::now() < deadline { - if TcpStream::connect_timeout( - &format!("127.0.0.1:{port}").parse().unwrap(), - Duration::from_millis(200), - ) - .is_ok() - { - std::thread::sleep(Duration::from_millis(150)); - return; - } - std::thread::sleep(Duration::from_millis(50)); - } - panic!("server on port {port} never came up within 15s"); -} - -fn populate_pending_step(data_dir: &Path) { - std::fs::create_dir_all(data_dir).unwrap(); - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let metadata_json = serde_json::json!({ - "step_sources": { "noop": "fn main() -> Int { 0 }\n" } - }) - .to_string(); - store - .insert_run(&RunRow { - run_id: "run-init".into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Running, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: r#"{"default_allow":true}"#.into(), - metadata_json, - }) - .unwrap(); - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: "run-init".into(), - step_id: "noop".into(), - status: StepStatus::Pending, - output_json: None, - output_hash: None, - started_at_ms: None, - ended_at_ms: None, - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); -} - -fn kill_child(mut child: Child) { - let _ = child.kill(); - let _ = child.wait(); -} - -/// A complete on-disk certificate bundle for the test. -struct CertBundle { - /// CA cert PEM path (used both as server's client-CA and as - /// the worker's server-CA — the same self-signed root signs - /// both ends in this test). - ca_pem: PathBuf, - server_cert_pem: PathBuf, - server_key_pem: PathBuf, - /// Per-test client cert + key. Subject CN = `client_cn`. - client_cert_pem: PathBuf, - client_key_pem: PathBuf, - client_cn: String, -} - -fn generate_certs(dir: &Path, client_cn: &str) -> CertBundle { - use rcgen::{ - BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, KeyPair, - KeyUsagePurpose, - }; - - // 1. CA. - let mut ca_params = CertificateParams::new(Vec::::new()).unwrap(); - ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); - ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; - let mut ca_dn = DistinguishedName::new(); - ca_dn.push(DnType::CommonName, "boruna-mtls-test-CA"); - ca_params.distinguished_name = ca_dn; - let ca_key = KeyPair::generate().unwrap(); - let ca_cert = ca_params.self_signed(&ca_key).unwrap(); - - // 2. Server cert (CN=localhost, SAN includes IP 127.0.0.1). - let mut server_params = - CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()]).unwrap(); - let mut server_dn = DistinguishedName::new(); - server_dn.push(DnType::CommonName, "localhost"); - server_params.distinguished_name = server_dn; - let server_key = KeyPair::generate().unwrap(); - let server_cert = server_params - .signed_by(&server_key, &ca_cert, &ca_key) - .unwrap(); - - // 3. Client cert with the requested CN. - let mut client_params = CertificateParams::new(vec![client_cn.to_string()]).unwrap(); - let mut client_dn = DistinguishedName::new(); - client_dn.push(DnType::CommonName, client_cn); - client_params.distinguished_name = client_dn; - let client_key = KeyPair::generate().unwrap(); - let client_cert = client_params - .signed_by(&client_key, &ca_cert, &ca_key) - .unwrap(); - - let ca_pem = dir.join("ca.pem"); - let server_cert_pem = dir.join("server-cert.pem"); - let server_key_pem = dir.join("server-key.pem"); - let client_cert_pem = dir.join("client-cert.pem"); - let client_key_pem = dir.join("client-key.pem"); - write_pem(&ca_pem, &ca_cert.pem()); - write_pem(&server_cert_pem, &server_cert.pem()); - write_pem(&server_key_pem, &server_key.serialize_pem()); - write_pem(&client_cert_pem, &client_cert.pem()); - write_pem(&client_key_pem, &client_key.serialize_pem()); - - CertBundle { - ca_pem, - server_cert_pem, - server_key_pem, - client_cert_pem, - client_key_pem, - client_cn: client_cn.to_string(), - } -} - -fn write_pem(path: &Path, content: &str) { - let mut f = std::fs::File::create(path).unwrap(); - f.write_all(content.as_bytes()).unwrap(); -} - -fn spawn_mtls_coordinator(data_dir: &Path, certs: &CertBundle) -> (Child, u16) { - let port = pick_free_port(); - let child = Command::new(boruna_bin()) - .args([ - "coordinator", - "serve", - "--data-dir", - data_dir.to_str().unwrap(), - "--port", - &port.to_string(), - "--max-lease-ttl-ms", - "60000", - "--poll-timeout-ms", - "200", - "--tls-cert", - certs.server_cert_pem.to_str().unwrap(), - "--tls-key", - certs.server_key_pem.to_str().unwrap(), - "--tls-client-ca", - certs.ca_pem.to_str().unwrap(), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn mTLS coordinator"); - wait_for_server(port); - (child, port) -} - -/// Build a blocking reqwest client with a specific client cert -/// and trust root. -fn build_https_client( - cert_pem: &Path, - key_pem: &Path, - server_ca_pem: &Path, -) -> reqwest::blocking::Client { - let mut id_pem = std::fs::read(cert_pem).unwrap(); - if !id_pem.ends_with(b"\n") { - id_pem.push(b'\n'); - } - id_pem.extend_from_slice(&std::fs::read(key_pem).unwrap()); - let identity = reqwest::Identity::from_pem(&id_pem).expect("identity"); - let ca_pem = std::fs::read(server_ca_pem).unwrap(); - let ca = reqwest::Certificate::from_pem(&ca_pem).expect("server CA"); - reqwest::blocking::Client::builder() - .use_rustls_tls() - .identity(identity) - .add_root_certificate(ca) - .timeout(Duration::from_secs(10)) - .build() - .expect("https client") -} - -/// Connect with no client cert at all and confirm the TLS -/// handshake does not produce an HTTP response. Uses a low-level -/// rustls ClientConfig with no client identity. The server-side -/// `WebPkiClientVerifier` requires a client cert; the handshake -/// should fail with a "certificate required" alert. -fn assert_handshake_rejected_no_cert(port: u16, server_ca_pem: &Path) { - use std::io::Read; - - use rustls::pki_types::ServerName; - use rustls::{ClientConfig, RootCertStore}; - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - - let ca_pem = std::fs::read(server_ca_pem).unwrap(); - let mut reader = std::io::Cursor::new(ca_pem); - let mut roots = RootCertStore::empty(); - let certs: Vec<_> = rustls_pemfile::certs(&mut reader) - .collect::>() - .unwrap(); - for c in certs { - roots.add(c).unwrap(); - } - let config = ClientConfig::builder() - .with_root_certificates(roots) - .with_no_client_auth(); - let server_name = ServerName::try_from("localhost").unwrap(); - let mut client = rustls::ClientConnection::new(Arc::new(config), server_name).unwrap(); - let mut sock = TcpStream::connect(("127.0.0.1", port)).expect("tcp connect"); - sock.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - sock.set_write_timeout(Some(Duration::from_secs(5))) - .unwrap(); - let mut tls = rustls::Stream::new(&mut client, &mut sock); - let req = format!( - "POST /api/workers/register HTTP/1.1\r\nHost: localhost:{port}\r\nContent-Type: application/json\r\nContent-Length: 2\r\n\r\n{{}}" - ); - // Try to drive the handshake to completion. Rustls buffers - // application data until handshake is done; the failure - // surfaces on the next read. Either write OR read failing - // proves the handshake was rejected. - let write_res = tls.write_all(req.as_bytes()); - let mut buf = [0u8; 64]; - let read_res = tls.read(&mut buf); - let read_zero = matches!(read_res, Ok(0)); - assert!( - write_res.is_err() || read_res.is_err() || read_zero, - "expected handshake rejection without client cert; \ - write={write_res:?} read={read_res:?}" - ); -} - -fn cap_hash() -> String { - boruna_bytecode::compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ) -} - -#[test] -fn mtls_no_client_cert_handshake_rejected() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path()); - let cert_dir = dir.path().join("certs"); - std::fs::create_dir_all(&cert_dir).unwrap(); - let certs = generate_certs(&cert_dir, "worker-A"); - let (child, port) = spawn_mtls_coordinator(dir.path(), &certs); - let result = std::panic::catch_unwind(|| { - assert_handshake_rejected_no_cert(port, &certs.ca_pem); - }); - kill_child(child); - result.expect("handshake-rejection assertion"); -} - -#[test] -fn mtls_client_cert_from_different_ca_handshake_rejected() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path()); - let cert_dir = dir.path().join("certs"); - let foreign_dir = dir.path().join("foreign"); - std::fs::create_dir_all(&cert_dir).unwrap(); - std::fs::create_dir_all(&foreign_dir).unwrap(); - let certs = generate_certs(&cert_dir, "worker-A"); - // Generate a SECOND, untrusted CA + client cert. - let foreign = generate_certs(&foreign_dir, "worker-A"); - let (child, port) = spawn_mtls_coordinator(dir.path(), &certs); - - // Use the foreign client cert against the server's trusted - // CA. The handshake should fail because the server's - // WebPkiClientVerifier doesn't trust the foreign CA. - let client = build_https_client( - &foreign.client_cert_pem, - &foreign.client_key_pem, - &certs.ca_pem, - ); - let body = serde_json::json!({ - "worker_id": "worker-A", - "capability_set_hash": cap_hash(), - }) - .to_string(); - let url = format!("https://localhost:{port}/api/workers/register"); - let res = client - .post(&url) - .header("content-type", "application/json") - .body(body) - .send(); - kill_child(child); - // We expect a transport-level error (handshake failure), not - // an HTTP response. - assert!( - res.is_err(), - "expected handshake rejection for foreign-CA cert; got {res:?}" - ); -} - -#[test] -fn mtls_valid_client_cert_succeeds_and_cn_drives_worker_id() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path()); - let cert_dir = dir.path().join("certs"); - std::fs::create_dir_all(&cert_dir).unwrap(); - let certs = generate_certs(&cert_dir, "worker-good-7"); - let (child, port) = spawn_mtls_coordinator(dir.path(), &certs); - let client = build_https_client(&certs.client_cert_pem, &certs.client_key_pem, &certs.ca_pem); - // No worker_id in body — exercise the CN-drives-identity - // branch. - let body = serde_json::json!({ - "capability_set_hash": cap_hash(), - }) - .to_string(); - let url = format!("https://localhost:{port}/api/workers/register"); - let res = client - .post(&url) - .header("content-type", "application/json") - .body(body) - .send(); - let status = res.as_ref().map(|r| r.status().as_u16()).unwrap_or(0); - let text = res - .map(|r| r.text().unwrap_or_default()) - .unwrap_or_default(); - kill_child(child); - assert_eq!(status, 200, "resp: {text}"); - let v: serde_json::Value = serde_json::from_str(&text).expect("json"); - assert_eq!(v["worker_id"], certs.client_cn); -} - -#[test] -fn mtls_cn_mismatch_returns_identity_mismatch() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path()); - let cert_dir = dir.path().join("certs"); - std::fs::create_dir_all(&cert_dir).unwrap(); - let certs = generate_certs(&cert_dir, "worker-A"); - let (child, port) = spawn_mtls_coordinator(dir.path(), &certs); - let client = build_https_client(&certs.client_cert_pem, &certs.client_key_pem, &certs.ca_pem); - // Body says we are worker-B but our cert CN is worker-A. - let body = serde_json::json!({ - "worker_id": "worker-B", - "capability_set_hash": cap_hash(), - }) - .to_string(); - let url = format!("https://localhost:{port}/api/workers/register"); - let res = client - .post(&url) - .header("content-type", "application/json") - .body(body) - .send(); - let status = res.as_ref().map(|r| r.status().as_u16()).unwrap_or(0); - let text = res - .map(|r| r.text().unwrap_or_default()) - .unwrap_or_default(); - kill_child(child); - assert_eq!(status, 401, "resp: {text}"); - let v: serde_json::Value = serde_json::from_str(&text).expect("json"); - assert_eq!(v["error_kind"], "coord.identity_mismatch"); -} - -#[test] -fn mtls_partial_flags_are_rejected_at_startup() { - // Pass --tls-cert without --tls-key/--tls-client-ca and - // confirm the process exits with the typed error message - // (project §1: half-configured TLS rejected at parse). - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path()); - let cert_dir = dir.path().join("certs"); - std::fs::create_dir_all(&cert_dir).unwrap(); - let certs = generate_certs(&cert_dir, "worker-A"); - let port = pick_free_port(); - let out = Command::new(boruna_bin()) - .args([ - "coordinator", - "serve", - "--data-dir", - dir.path().to_str().unwrap(), - "--port", - &port.to_string(), - "--tls-cert", - certs.server_cert_pem.to_str().unwrap(), - ]) - .output() - .expect("invoke"); - assert!(!out.status.success()); - let stderr = String::from_utf8_lossy(&out.stderr); - assert!( - stderr.contains("must all be provided together"), - "stderr: {stderr}" - ); -} diff --git a/crates/llmvm-cli/tests/cli_coordinator_worker.rs b/crates/llmvm-cli/tests/cli_coordinator_worker.rs deleted file mode 100644 index 6231606..0000000 --- a/crates/llmvm-cli/tests/cli_coordinator_worker.rs +++ /dev/null @@ -1,1718 +0,0 @@ -//! End-to-end CLI integration tests for `boruna coordinator -//! serve` + `boruna worker run` (sprint 0.5-S2b). Spawns the -//! binary in both modes and asserts the protocol works -//! end-to-end. -//! -//! Only compiled when `--features serve` is enabled. - -#![cfg(feature = "serve")] - -use std::io::{BufRead, BufReader, Read, Write}; -use std::net::{TcpListener, TcpStream}; -use std::path::Path; -use std::process::{Child, Command, Stdio}; -use std::time::{Duration, Instant}; - -use boruna_orchestrator::persistence::{ - RunCheckpointStore, RunRow, RunStatus, StepCheckpoint, StepStatus, -}; - -fn boruna_bin() -> &'static str { - env!("CARGO_BIN_EXE_boruna") -} - -/// Connect to a freshly-spawned server with a small retry budget. -/// Sprint W10 — pre-existing flaky surface flagged by W9-D's local -/// runs: even after `wait_for_server` returns, a busy CI runner can -/// race the listener-backlog state and surface ConnectionRefused on -/// the first real request. Retrying fixes the race without changing -/// what the test verifies (HTTP responses, not connect timing). -/// Mirrors the W8 fix in `cli_dashboard.rs::http_request`. -fn connect_with_retries(port: u16) -> TcpStream { - let mut last_err: Option = None; - for attempt in 0..5 { - match TcpStream::connect_timeout( - &format!("127.0.0.1:{port}").parse().unwrap(), - Duration::from_millis(500), - ) { - Ok(s) => return s, - Err(e) => { - last_err = Some(e); - std::thread::sleep(Duration::from_millis(50 * (attempt + 1))); - } - } - } - panic!( - "connect to 127.0.0.1:{port} failed after 5 retries; last err: {}", - last_err - .as_ref() - .map(|e| e.to_string()) - .unwrap_or_else(|| "unknown".into()) - ); -} - -fn pick_free_port() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); - let port = listener.local_addr().unwrap().port(); - drop(listener); - port -} - -fn wait_for_server(port: u16) { - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline { - if TcpStream::connect_timeout( - &format!("127.0.0.1:{port}").parse().unwrap(), - Duration::from_millis(200), - ) - .is_ok() - { - // Brief wait for the server's HTTP layer to be ready - // after TCP accept. - std::thread::sleep(Duration::from_millis(100)); - return; - } - std::thread::sleep(Duration::from_millis(50)); - } - panic!("server on port {port} never came up within 10s"); -} - -fn http_request(port: u16, method: &str, path: &str, body: Option<&str>) -> (u16, String) { - let mut stream = connect_with_retries(port); - stream - .set_read_timeout(Some(Duration::from_secs(10))) - .unwrap(); - let body = body.unwrap_or(""); - let req = format!( - "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream.write_all(req.as_bytes()).expect("write"); - let mut reader = BufReader::new(&stream); - let mut status_line = String::new(); - reader.read_line(&mut status_line).expect("read status"); - let parts: Vec<&str> = status_line.split_whitespace().collect(); - let code: u16 = parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0); - loop { - let mut line = String::new(); - let n = reader.read_line(&mut line).unwrap_or(0); - if n == 0 || line == "\r\n" || line == "\n" { - break; - } - } - let mut body = String::new(); - let _ = reader.read_to_string(&mut body); - let _ = stream.shutdown(std::net::Shutdown::Both); - (code, body) -} - -fn populate_pending_step(data_dir: &Path, run_id: &str, step_id: &str, source: &str) { - std::fs::create_dir_all(data_dir).unwrap(); - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let metadata_json = serde_json::json!({ - "step_sources": { step_id: source } - }) - .to_string(); - store - .insert_run(&RunRow { - run_id: run_id.into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Running, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: r#"{"default_allow":true}"#.into(), - metadata_json, - }) - .unwrap(); - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: run_id.into(), - step_id: step_id.into(), - status: StepStatus::Pending, - output_json: None, - output_hash: None, - started_at_ms: None, - ended_at_ms: None, - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); -} - -fn spawn_coordinator(data_dir: &Path, max_lease_ttl_ms: u64, poll_timeout_ms: u64) -> (Child, u16) { - spawn_coordinator_with_sweep(data_dir, max_lease_ttl_ms, poll_timeout_ms, 30_000) -} - -fn spawn_coordinator_with_sweep( - data_dir: &Path, - max_lease_ttl_ms: u64, - poll_timeout_ms: u64, - sweep_interval_ms: u64, -) -> (Child, u16) { - let port = pick_free_port(); - let child = Command::new(boruna_bin()) - .args([ - "coordinator", - "serve", - "--data-dir", - data_dir.to_str().unwrap(), - "--port", - &port.to_string(), - "--max-lease-ttl-ms", - &max_lease_ttl_ms.to_string(), - "--poll-timeout-ms", - &poll_timeout_ms.to_string(), - "--sweep-interval-ms", - &sweep_interval_ms.to_string(), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn coordinator"); - wait_for_server(port); - (child, port) -} - -fn spawn_worker(coord_url: &str, worker_id: &str, lease_ttl_ms: u64) -> Child { - Command::new(boruna_bin()) - .args([ - "worker", - "run", - "--coordinator", - coord_url, - "--worker-id", - worker_id, - "--lease-ttl-ms", - &lease_ttl_ms.to_string(), - "--poll-timeout-ms", - "1000", - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn worker") -} - -fn kill_child(mut child: Child) { - let _ = child.kill(); - let _ = child.wait(); -} - -/// Spawn a coordinator with an auth shared-secret. Sprint 0.5-S3. -fn spawn_coordinator_with_secret(data_dir: &Path, secret: &str) -> (Child, u16) { - let port = pick_free_port(); - let child = Command::new(boruna_bin()) - .args([ - "coordinator", - "serve", - "--data-dir", - data_dir.to_str().unwrap(), - "--port", - &port.to_string(), - "--max-lease-ttl-ms", - "60000", - "--poll-timeout-ms", - "200", - "--shared-secret", - secret, - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn coordinator with secret"); - wait_for_server(port); - (child, port) -} - -fn http_request_with_auth( - port: u16, - method: &str, - path: &str, - body: Option<&str>, - bearer: Option<&str>, -) -> (u16, String) { - let mut stream = connect_with_retries(port); - stream - .set_read_timeout(Some(Duration::from_secs(10))) - .unwrap(); - let body = body.unwrap_or(""); - let auth_header = match bearer { - Some(b) => format!("Authorization: Bearer {b}\r\n"), - None => String::new(), - }; - let req = format!( - "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n{auth_header}Content-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream.write_all(req.as_bytes()).expect("write"); - let mut reader = BufReader::new(&stream); - let mut status_line = String::new(); - reader.read_line(&mut status_line).expect("read status"); - let parts: Vec<&str> = status_line.split_whitespace().collect(); - let code: u16 = parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0); - loop { - let mut line = String::new(); - let n = reader.read_line(&mut line).unwrap_or(0); - if n == 0 || line == "\r\n" || line == "\n" { - break; - } - } - let mut body = String::new(); - let _ = reader.read_to_string(&mut body); - let _ = stream.shutdown(std::net::Shutdown::Both); - (code, body) -} - -#[test] -fn coord_register_returns_worker_id_and_session_token() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-init", "noop", "fn main() -> Int { 0 }\n"); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - let cap_hash = boruna_bytecode::compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - let body = serde_json::json!({ - "capability_set_hash": cap_hash, - }) - .to_string(); - let (code, resp) = http_request(port, "POST", "/api/workers/register", Some(&body)); - kill_child(child); - assert_eq!(code, 200, "resp: {resp}"); - let v: serde_json::Value = serde_json::from_str(&resp).expect("json"); - assert_eq!(v["protocol_version"], 1); - assert!(v["worker_id"].as_str().unwrap().starts_with("wkr-")); - assert!(v["session_token"].as_str().unwrap().starts_with("sess-")); -} - -#[test] -fn coord_register_rejects_binary_mismatch() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-init", "noop", "fn main() -> Int { 0 }\n"); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - let body = serde_json::json!({ - "capability_set_hash": "sha256:bogus", - }) - .to_string(); - let (code, resp) = http_request(port, "POST", "/api/workers/register", Some(&body)); - kill_child(child); - assert_eq!(code, 409); - let v: serde_json::Value = serde_json::from_str(&resp).expect("json"); - assert_eq!(v["error_kind"], "coord.binary_mismatch"); - assert!(v["expected_hash"].is_string()); -} - -#[test] -fn coord_missing_data_dir_exits_nonzero() { - let dir = tempfile::tempdir().unwrap(); - let bogus = dir.path().join("does-not-exist"); - let port = pick_free_port(); - let out = Command::new(boruna_bin()) - .args([ - "coordinator", - "serve", - "--data-dir", - bogus.to_str().unwrap(), - "--port", - &port.to_string(), - ]) - .output() - .expect("invoke"); - assert!(!out.status.success()); - let stderr = String::from_utf8_lossy(&out.stderr); - assert!(stderr.contains("runs.db"), "stderr: {stderr}"); -} - -#[test] -fn coord_oversize_body_returns_413() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-init", "noop", "fn main() -> Int { 0 }\n"); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - // Build a body just over 8 MiB. - let oversize = "x".repeat(8 * 1024 * 1024 + 100); - let body = serde_json::json!({ - "worker_id": "ghost", - "session_token": "x", - "run_id": "r", - "step_id": "s", - "claim_id": 1, - "output_json": oversize, - "output_hash": "h", - "attempt_count": 1, - }) - .to_string(); - let (code, _resp) = http_request(port, "POST", "/api/work/complete", Some(&body)); - kill_child(child); - // Axum's DefaultBodyLimit returns 413 for oversized bodies. - assert_eq!(code, 413, "expected 413, got {code}"); -} - -#[test] -fn worker_subprocess_completes_step_end_to_end() { - // The MVP smoke test: spawn coordinator + worker, pre-populate - // a single Pending step, wait for the worker to claim+execute+ - // complete, assert the row's final state. - let dir = tempfile::tempdir().unwrap(); - populate_pending_step( - dir.path(), - "run-smoke", - "compute", - "fn main() -> Int { 1 + 2 + 3 + 4 }\n", - ); - let (coord_child, port) = spawn_coordinator(dir.path(), 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker_child = spawn_worker(&coord_url, "smoke-worker", 30_000); - - // Poll runs.db until the step transitions to Completed (or - // timeout). - let db_path = dir.path().join("runs.db"); - let deadline = Instant::now() + Duration::from_secs(15); - let mut seen_completed = false; - let mut last_status = String::new(); - while Instant::now() < deadline { - let store = RunCheckpointStore::open(&db_path).unwrap(); - let cps = store.list_step_checkpoints("run-smoke").unwrap(); - if let Some(cp) = cps.first() { - last_status = cp.status.as_str().into(); - if cp.status == StepStatus::Completed { - seen_completed = true; - assert_eq!(cp.output_json.as_deref(), Some("10")); - assert!(cp.output_hash.as_deref().unwrap().starts_with("sha256:")); - assert!(cp.worker_id.is_none()); - assert!(cp.lease_expires_at_ms.is_none()); - assert_eq!(cp.claim_id, 1); - break; - } - } - drop(store); - std::thread::sleep(Duration::from_millis(100)); - } - kill_child(worker_child); - kill_child(coord_child); - assert!( - seen_completed, - "step never reached Completed; last status: {last_status}" - ); -} - -#[test] -fn worker_kill_mid_step_lease_expires_then_reclaim() { - // Flagship regression: prove that a step claimed by some - // worker that never completes (lease expires) is reclaimed - // by a fresh worker via the HTTP path. Asserts the final - // row's `claim_id == 2`, proving the CAS state machine - // worked end-to-end through the coordinator's claim - // endpoint. - // - // Adversarial-review fix (F5): the prior version spawned a - // real worker A subprocess, which was racy under fast CI - // (worker A could complete before the lease expired, - // turning the test into a no-op). The deterministic - // version simulates worker A's claim by inserting a - // Running checkpoint with an already-expired - // `lease_expires_at` directly into runs.db. The - // coordinator's startup sweep + the explicit - // `expire_leases_and_requeue` call moves the row back to - // Pending; only then does worker B come up via the HTTP - // path and reclaim. - use boruna_orchestrator::persistence::StepCheckpoint; - let dir = tempfile::tempdir().unwrap(); - let metadata_json = serde_json::json!({ - "step_sources": { "step1": "fn main() -> Int { 99 }\n" } - }) - .to_string(); - std::fs::create_dir_all(dir.path()).unwrap(); - let store = RunCheckpointStore::open(&dir.path().join("runs.db")).unwrap(); - store - .insert_run(&RunRow { - run_id: "run-race".into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Running, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: r#"{"default_allow":true}"#.into(), - metadata_json, - }) - .unwrap(); - // First insert as Pending so claim_step can transition it. - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: "run-race".into(), - step_id: "step1".into(), - status: StepStatus::Pending, - output_json: None, - output_hash: None, - started_at_ms: None, - ended_at_ms: None, - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); - // Simulate "worker A claimed and was killed": call the - // public claim_step API with a `lease_expires_at` that's - // already in the past relative to wall-clock-now. The - // coordinator's startup sweep will then expire this lease. - store - .claim_step("run-race", "step1", "worker-A", 1, 0) - .unwrap(); - drop(store); - - // Spawn the coordinator — its startup sweep should expire - // the stale lease and re-enqueue the step as Pending. - let (coord_child, port) = spawn_coordinator(dir.path(), 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let db_path = dir.path().join("runs.db"); - - // Verify the coordinator's startup sweep ran. - let store = RunCheckpointStore::open(&db_path).unwrap(); - let cp = store - .list_step_checkpoints("run-race") - .unwrap() - .pop() - .unwrap(); - assert_eq!( - cp.status, - StepStatus::Pending, - "coordinator startup sweep should have requeued the stale-lease row" - ); - assert_eq!(cp.worker_id, None); - assert_eq!(cp.claim_id, 1, "claim_id preserved across requeue"); - drop(store); - - // Spawn worker B; it should claim (claim_id=2) and complete. - let worker_b = spawn_worker(&coord_url, "worker-B", 60_000); - - let deadline = Instant::now() + Duration::from_secs(15); - let mut completed = false; - while Instant::now() < deadline { - let store = RunCheckpointStore::open(&db_path).unwrap(); - let cps = store.list_step_checkpoints("run-race").unwrap(); - if let Some(cp) = cps.first() { - if cp.status == StepStatus::Completed { - completed = true; - assert_eq!(cp.claim_id, 2, "expected claim_id=2 after reclaim"); - assert_eq!(cp.output_json.as_deref(), Some("99")); - break; - } - } - drop(store); - std::thread::sleep(Duration::from_millis(100)); - } - kill_child(worker_b); - kill_child(coord_child); - assert!(completed, "worker B never reclaimed and completed"); -} - -#[test] -fn cli_workflow_run_submit_only_then_worker_completes() { - // Sprint 0.5-S2e: full end-to-end via the marquee CLI - // path. Spawn coordinator + worker. Use - // `boruna workflow run --submit-only` against a 1-step - // workflow on disk (real workflow.json + .ax file). - // Assert: - // 1. `workflow run --submit-only` exits 0. - // 2. The step transitions Pending → Running → - // Completed via the worker. - // 3. The output_json matches the expected value (proof - // that the `.ax` source flowed through metadata, - // coordinator, worker, and back). - let dir = tempfile::tempdir().unwrap(); - let data_dir = dir.path().join("data"); - let wf_dir = dir.path().join("wf"); - std::fs::create_dir_all(&wf_dir).unwrap(); - std::fs::write(wf_dir.join("step1.ax"), "fn main() -> Int { 7 }\n").unwrap(); - std::fs::write( - wf_dir.join("workflow.json"), - r#"{ - "schema_version": 1, - "name": "submit-test", - "version": "1.0.0", - "steps": { - "step1": { - "kind": "source", - "source": "step1.ax", - "capabilities": [], - "outputs": {"result": "Int"} - } - }, - "edges": [] - }"#, - ) - .unwrap(); - - // Pre-create the data dir so the coordinator can open - // runs.db. We bootstrap by running submit-only first - // (which creates runs.db), then start the coordinator. - std::fs::create_dir_all(&data_dir).unwrap(); - - // Submit the workflow (creates runs.db + Pending step1). - let submit_out = Command::new(boruna_bin()) - .args([ - "workflow", - "run", - wf_dir.to_str().unwrap(), - "--data-dir", - data_dir.to_str().unwrap(), - "--submit-only", - ]) - .output() - .expect("invoke boruna workflow run --submit-only"); - assert!( - submit_out.status.success(), - "submit failed: stderr={}", - String::from_utf8_lossy(&submit_out.stderr) - ); - - // Find the run_id by scanning runs.db. - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let runs = store.list_runs().unwrap(); - assert_eq!(runs.len(), 1, "expected exactly one run after submit"); - let run_id = runs[0].run_id.clone(); - drop(store); - - // Spawn coordinator + worker and wait for the step to complete. - let (coord_child, port) = spawn_coordinator(&data_dir, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "submit-worker", 30_000); - - let deadline = Instant::now() + Duration::from_secs(15); - let mut completed = false; - let mut last_state = String::new(); - while Instant::now() < deadline { - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let cps = store.list_step_checkpoints(&run_id).unwrap(); - last_state = format!( - "{:?}", - cps.iter() - .map(|c| (c.step_id.as_str(), c.status)) - .collect::>() - ); - if let Some(cp) = cps.first() { - if cp.status == StepStatus::Completed { - completed = true; - assert_eq!(cp.output_json.as_deref(), Some("7")); - assert_eq!(cp.claim_id, 1); - break; - } - } - drop(store); - std::thread::sleep(Duration::from_millis(100)); - } - kill_child(worker); - kill_child(coord_child); - assert!( - completed, - "step never reached Completed; last state: {last_state}" - ); -} - -// ── coordinator wait (sprint 0.5-S2f) ── - -/// Build a 3-step fan-in workflow on disk for the wait-driver tests. -/// `s1` and `s2` are wave-1 source steps; `s3` depends on both. -/// `s3.ax` body controls success/failure for the failed-run test. -fn make_fan_in_workflow_on_disk(wf_dir: &Path, s3_body: &str) { - std::fs::create_dir_all(wf_dir).unwrap(); - std::fs::write(wf_dir.join("s1.ax"), "fn main() -> Int { 1 }\n").unwrap(); - std::fs::write(wf_dir.join("s2.ax"), "fn main() -> Int { 2 }\n").unwrap(); - std::fs::write(wf_dir.join("s3.ax"), s3_body).unwrap(); - std::fs::write( - wf_dir.join("workflow.json"), - r#"{ - "schema_version": 1, - "name": "wait-test", - "version": "1.0.0", - "steps": { - "s1": {"kind": "source", "source": "s1.ax", "capabilities": [], "outputs": {"result": "Int"}}, - "s2": {"kind": "source", "source": "s2.ax", "capabilities": [], "outputs": {"result": "Int"}}, - "s3": {"kind": "source", "source": "s3.ax", "capabilities": [], "outputs": {"result": "Int"}} - }, - "edges": [["s1", "s3"], ["s2", "s3"]] - }"#, - ) - .unwrap(); -} - -fn submit_only(data_dir: &Path, wf_dir: &Path) -> String { - let out = Command::new(boruna_bin()) - .args([ - "workflow", - "run", - wf_dir.to_str().unwrap(), - "--data-dir", - data_dir.to_str().unwrap(), - "--submit-only", - ]) - .output() - .expect("invoke boruna workflow run --submit-only"); - assert!( - out.status.success(), - "submit failed: stderr={}", - String::from_utf8_lossy(&out.stderr) - ); - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let runs = store.list_runs().unwrap(); - assert_eq!(runs.len(), 1, "expected exactly one run after submit"); - runs[0].run_id.clone() -} - -fn spawn_wait(data_dir: &Path, run_id: &str) -> Child { - Command::new(boruna_bin()) - .args([ - "coordinator", - "wait", - run_id, - "--data-dir", - data_dir.to_str().unwrap(), - "--poll-interval-ms", - "100", - "--max-wait-secs", - "30", - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn coordinator wait") -} - -#[test] -fn cli_coordinator_wait_drives_multi_wave_to_completion() { - // Sprint 0.5-S2f marquee test. Submit a 3-step fan-in workflow - // (s1, s2 → s3) and prove the `coordinator wait` driver advances - // wave-2 (s3) to Pending after wave-1 completes, the worker picks - // it up, and the run reaches Completed with all 3 steps Completed. - let dir = tempfile::tempdir().unwrap(); - let data_dir = dir.path().join("data"); - let wf_dir = dir.path().join("wf"); - std::fs::create_dir_all(&data_dir).unwrap(); - make_fan_in_workflow_on_disk(&wf_dir, "fn main() -> Int { 3 }\n"); - - let run_id = submit_only(&data_dir, &wf_dir); - - let (coord_child, port) = spawn_coordinator(&data_dir, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "wait-worker", 30_000); - - // Run wait synchronously and wait for it to exit. - let wait_out = Command::new(boruna_bin()) - .args([ - "coordinator", - "wait", - &run_id, - "--data-dir", - data_dir.to_str().unwrap(), - "--poll-interval-ms", - "100", - "--max-wait-secs", - "30", - ]) - .output() - .expect("invoke coordinator wait"); - - kill_child(worker); - kill_child(coord_child); - - assert!( - wait_out.status.success(), - "wait exited non-zero; status={:?}\nstdout={}\nstderr={}", - wait_out.status.code(), - String::from_utf8_lossy(&wait_out.stdout), - String::from_utf8_lossy(&wait_out.stderr), - ); - - // All 3 steps Completed; correct outputs. - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let cps = store.list_step_checkpoints(&run_id).unwrap(); - assert_eq!(cps.len(), 3, "got {} checkpoints", cps.len()); - for cp in &cps { - assert_eq!( - cp.status, - StepStatus::Completed, - "step {} not Completed: {:?}", - cp.step_id, - cp.status - ); - } - let outputs: std::collections::BTreeMap<&str, &str> = cps - .iter() - .map(|c| (c.step_id.as_str(), c.output_json.as_deref().unwrap_or(""))) - .collect(); - assert_eq!(outputs["s1"], "1"); - assert_eq!(outputs["s2"], "2"); - assert_eq!(outputs["s3"], "3"); - - // Wait stdout should contain transitions per step. - let stdout = String::from_utf8_lossy(&wait_out.stdout); - assert!(stdout.contains("step s1"), "stdout missing s1: {stdout}"); - assert!(stdout.contains("step s3"), "stdout missing s3: {stdout}"); - assert!( - stdout.contains("completed"), - "stdout missing completed: {stdout}" - ); -} - -#[test] -fn cli_coordinator_wait_resumes_after_kill() { - // Kill the wait process between waves; re-invoke; the run still - // completes. Proves the wait driver is stateless on the client - // side — all state lives in runs.db. - let dir = tempfile::tempdir().unwrap(); - let data_dir = dir.path().join("data"); - let wf_dir = dir.path().join("wf"); - std::fs::create_dir_all(&data_dir).unwrap(); - make_fan_in_workflow_on_disk(&wf_dir, "fn main() -> Int { 3 }\n"); - - let run_id = submit_only(&data_dir, &wf_dir); - - let (coord_child, port) = spawn_coordinator(&data_dir, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "resume-worker", 30_000); - - // First wait: kill it after a short delay (likely between - // waves but state survives regardless). - let wait1 = spawn_wait(&data_dir, &run_id); - std::thread::sleep(Duration::from_millis(800)); - kill_child(wait1); - - // Second wait: drive to terminal. - let wait_out = Command::new(boruna_bin()) - .args([ - "coordinator", - "wait", - &run_id, - "--data-dir", - data_dir.to_str().unwrap(), - "--poll-interval-ms", - "100", - "--max-wait-secs", - "30", - ]) - .output() - .expect("invoke wait #2"); - - kill_child(worker); - kill_child(coord_child); - - assert!( - wait_out.status.success(), - "wait #2 exited non-zero; status={:?}\nstdout={}\nstderr={}", - wait_out.status.code(), - String::from_utf8_lossy(&wait_out.stdout), - String::from_utf8_lossy(&wait_out.stderr), - ); - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let cps = store.list_step_checkpoints(&run_id).unwrap(); - assert_eq!(cps.len(), 3); - for cp in &cps { - assert_eq!(cp.status, StepStatus::Completed); - } -} - -#[test] -fn cli_coordinator_wait_two_concurrent_waits_converge() { - // CORR-6 from 0.5-S2f: two `coordinator wait` processes against - // the same run_id must each converge to exit 0 (Completed). The - // race-safe persistence primitive (`insert_pending_step_if_absent`, - // ON CONFLICT DO NOTHING) guarantees only one wait wins each - // Pending insert; both observe the same terminal state. - let dir = tempfile::tempdir().unwrap(); - let data_dir = dir.path().join("data"); - let wf_dir = dir.path().join("wf"); - std::fs::create_dir_all(&data_dir).unwrap(); - make_fan_in_workflow_on_disk(&wf_dir, "fn main() -> Int { 3 }\n"); - - let run_id = submit_only(&data_dir, &wf_dir); - let (coord_child, port) = spawn_coordinator(&data_dir, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "concurrent-waits-worker", 30_000); - - // Spawn two `coordinator wait` children racing on the same run_id. - let wait1 = spawn_wait(&data_dir, &run_id); - let wait2 = spawn_wait(&data_dir, &run_id); - - // Each wait runs in its own process; collect their exit codes - // by spawning a third invocation that we wait on synchronously - // (it will see the same terminal state). Then ensure both - // background waits also converge. - let synchronous_wait = Command::new(boruna_bin()) - .args([ - "coordinator", - "wait", - &run_id, - "--data-dir", - data_dir.to_str().unwrap(), - "--poll-interval-ms", - "100", - "--max-wait-secs", - "30", - ]) - .output() - .expect("invoke synchronous wait"); - - // Background waits should also exit 0 — they race against the - // synchronous one but all see the same terminal state. Use - // try_wait with a short retry to confirm without leaking. - let _ = wait1.id(); - let _ = wait2.id(); - let mut wait1_status = None; - let mut wait2_status = None; - let deadline = Instant::now() + Duration::from_secs(15); - let mut wait1 = wait1; - let mut wait2 = wait2; - while Instant::now() < deadline { - if wait1_status.is_none() { - if let Ok(Some(s)) = wait1.try_wait() { - wait1_status = Some(s); - } - } - if wait2_status.is_none() { - if let Ok(Some(s)) = wait2.try_wait() { - wait2_status = Some(s); - } - } - if wait1_status.is_some() && wait2_status.is_some() { - break; - } - std::thread::sleep(Duration::from_millis(100)); - } - // Kill any wait still running (may have lost the race to detect - // terminal). Then the synchronous wait above is the one we - // assert against — its exit code is the contract. - if wait1_status.is_none() { - kill_child(wait1); - } - if wait2_status.is_none() { - kill_child(wait2); - } - kill_child(worker); - kill_child(coord_child); - - assert!( - synchronous_wait.status.success(), - "synchronous wait exited non-zero: status={:?}\nstdout={}\nstderr={}", - synchronous_wait.status.code(), - String::from_utf8_lossy(&synchronous_wait.stdout), - String::from_utf8_lossy(&synchronous_wait.stderr), - ); - - // Verify the run reached Completed; both background waits, if - // they exited, exited 0. - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - let cps = store.list_step_checkpoints(&run_id).unwrap(); - assert_eq!(cps.len(), 3); - for cp in &cps { - assert_eq!(cp.status, StepStatus::Completed); - } - if let Some(s) = wait1_status { - assert_eq!(s.code(), Some(0), "wait1 exited non-zero"); - } - if let Some(s) = wait2_status { - assert_eq!(s.code(), Some(0), "wait2 exited non-zero"); - } -} - -#[test] -fn cli_coordinator_wait_exits_zero_immediately_for_already_completed_run() { - // Drive a run to Completed via the marquee path, then re-invoke - // wait against the same run_id. The second wait should exit 0 - // immediately on the first tick (no polling loop). - let dir = tempfile::tempdir().unwrap(); - let data_dir = dir.path().join("data"); - let wf_dir = dir.path().join("wf"); - std::fs::create_dir_all(&data_dir).unwrap(); - make_fan_in_workflow_on_disk(&wf_dir, "fn main() -> Int { 3 }\n"); - - let run_id = submit_only(&data_dir, &wf_dir); - let (coord_child, port) = spawn_coordinator(&data_dir, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "completed-worker", 30_000); - - // First wait: drive to Completed. - let _ = Command::new(boruna_bin()) - .args([ - "coordinator", - "wait", - &run_id, - "--data-dir", - data_dir.to_str().unwrap(), - "--poll-interval-ms", - "100", - "--max-wait-secs", - "30", - ]) - .output() - .expect("invoke wait #1"); - - // Second wait: should exit 0 immediately. Use a short max-wait - // budget — if the loop spins without detecting Completed on the - // first tick, this would time out (exit 3) instead. - let started = Instant::now(); - let wait_out = Command::new(boruna_bin()) - .args([ - "coordinator", - "wait", - &run_id, - "--data-dir", - data_dir.to_str().unwrap(), - "--poll-interval-ms", - "5000", // long poll; if we hit it, the test fails - "--max-wait-secs", - "10", - ]) - .output() - .expect("invoke wait #2"); - let elapsed = started.elapsed(); - - kill_child(worker); - kill_child(coord_child); - - assert_eq!( - wait_out.status.code(), - Some(0), - "expected exit 0; got {:?}\nstdout={}\nstderr={}", - wait_out.status.code(), - String::from_utf8_lossy(&wait_out.stdout), - String::from_utf8_lossy(&wait_out.stderr), - ); - assert!( - elapsed < Duration::from_secs(3), - "wait took {elapsed:?}; expected immediate exit on first tick" - ); -} - -#[test] -fn cli_coordinator_wait_exits_nonzero_on_failed_run() { - // s3 has a deliberately broken .ax (missing main); worker fails it. - // Wait should exit non-zero (1 = run Failed). - let dir = tempfile::tempdir().unwrap(); - let data_dir = dir.path().join("data"); - let wf_dir = dir.path().join("wf"); - std::fs::create_dir_all(&data_dir).unwrap(); - // Compile-error body — the worker should fail this step. - make_fan_in_workflow_on_disk(&wf_dir, "this is not valid ax syntax\n"); - - let run_id = submit_only(&data_dir, &wf_dir); - - let (coord_child, port) = spawn_coordinator(&data_dir, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "fail-worker", 30_000); - - let wait_out = Command::new(boruna_bin()) - .args([ - "coordinator", - "wait", - &run_id, - "--data-dir", - data_dir.to_str().unwrap(), - "--poll-interval-ms", - "100", - "--max-wait-secs", - "30", - ]) - .output() - .expect("invoke wait"); - - kill_child(worker); - kill_child(coord_child); - - assert_eq!( - wait_out.status.code(), - Some(1), - "expected exit code 1 (run Failed); got {:?}\nstdout={}\nstderr={}", - wait_out.status.code(), - String::from_utf8_lossy(&wait_out.stdout), - String::from_utf8_lossy(&wait_out.stderr), - ); -} - -#[test] -fn coord_bg_sweep_requeues_expired_lease() { - // Sprint 0.5-S2c: prove the background sweep fires - // periodically and requeues stale leases without - // requiring a coordinator restart. - use boruna_orchestrator::persistence::StepCheckpoint; - let dir = tempfile::tempdir().unwrap(); - let metadata_json = serde_json::json!({ - "step_sources": { "step1": "fn main() -> Int { 1 }\n" } - }) - .to_string(); - std::fs::create_dir_all(dir.path()).unwrap(); - let store = RunCheckpointStore::open(&dir.path().join("runs.db")).unwrap(); - store - .insert_run(&RunRow { - run_id: "run-bg".into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Running, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: r#"{"default_allow":true}"#.into(), - metadata_json, - }) - .unwrap(); - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: "run-bg".into(), - step_id: "step1".into(), - status: StepStatus::Pending, - output_json: None, - output_hash: None, - started_at_ms: None, - ended_at_ms: None, - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); - drop(store); - - // Spawn coordinator with a fast sweep interval (200 ms). - let (coord_child, _port) = spawn_coordinator_with_sweep(dir.path(), 60_000, 1_000, 200); - - // Give the coordinator a moment past startup, then create - // a stale claim using the persistence API directly. The - // claim's lease expires in the past (`lease_expires_at=1`). - std::thread::sleep(Duration::from_millis(300)); - let store = RunCheckpointStore::open(&dir.path().join("runs.db")).unwrap(); - let outcome = store - .claim_step("run-bg", "step1", "ghost-worker", 1, 0) - .unwrap(); - assert!(matches!( - outcome, - boruna_orchestrator::persistence::ClaimOutcome::Claimed { .. } - )); - drop(store); - - // Poll for status flip (deadline 5s — plenty of margin - // even under parallel-test CPU contention; sweep interval - // is 200 ms). - let deadline = Instant::now() + Duration::from_secs(5); - let mut cp = None; - while Instant::now() < deadline { - let store = RunCheckpointStore::open(&dir.path().join("runs.db")).unwrap(); - let candidate = store - .list_step_checkpoints("run-bg") - .unwrap() - .pop() - .unwrap(); - drop(store); - if candidate.status == StepStatus::Pending { - cp = Some(candidate); - break; - } - cp = Some(candidate); - std::thread::sleep(Duration::from_millis(100)); - } - kill_child(coord_child); - let cp = cp.expect("step row not found"); - - assert_eq!( - cp.status, - StepStatus::Pending, - "background sweep should have requeued the stale-lease row" - ); - assert_eq!(cp.worker_id, None); - assert_eq!(cp.lease_expires_at_ms, None); - // claim_id is preserved across requeue (per 0.5-S2a contract). - assert_eq!(cp.claim_id, 1); -} - -#[test] -fn coord_serve_responds_to_dashboard_index() { - // Sprint 0.5-S2d: the coordinator now serves the - // dashboard's read routes on the same listener. - let dir = tempfile::tempdir().unwrap(); - populate_pending_step( - dir.path(), - "run-merged", - "step1", - "fn main() -> Int { 1 }\n", - ); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - let (code, body) = http_request(port, "GET", "/", None); - kill_child(child); - assert_eq!(code, 200, "body: {body}"); - assert!(body.contains(" Int { 1 }\n"); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - let (code, body) = http_request(port, "GET", "/api/runs", None); - kill_child(child); - assert_eq!(code, 200); - let v: serde_json::Value = serde_json::from_str(&body).expect("json"); - assert!(v["runs"].is_array()); - assert_eq!(v["runs"][0]["run_id"], "run-api"); - // Slim RunSummary contract from 0.4-S16: no policy/metadata - // leakage even on the merged listener. - let json = body.clone(); - assert!(!json.contains("policy_json")); - assert!(!json.contains("metadata_json")); -} - -#[test] -fn coord_serve_handles_both_coord_and_dashboard_routes_on_same_listener() { - // Adversarial-review gap: existing tests exercise coord - // routes OR dashboard routes against a coord process, - // never both against the same running instance. This - // test proves the merge actually works at runtime — both - // route trees coexist on one port. - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-merge", "step1", "fn main() -> Int { 1 }\n"); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - - // 1. Hit a coord route — register a worker. - let cap_hash = boruna_bytecode::compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - let body = serde_json::json!({"capability_set_hash": cap_hash}).to_string(); - let (coord_code, coord_resp) = http_request(port, "POST", "/api/workers/register", Some(&body)); - assert_eq!(coord_code, 200, "coord route failed: {coord_resp}"); - - // 2. Hit a dashboard route — list runs — on the SAME - // listener. Note: same port; new connection (our HTTP - // helper closes after each request). - let (dash_code, dash_resp) = http_request(port, "GET", "/api/runs", None); - assert_eq!(dash_code, 200, "dashboard route failed: {dash_resp}"); - let v: serde_json::Value = serde_json::from_str(&dash_resp).expect("json"); - assert_eq!(v["runs"][0]["run_id"], "run-merge"); - - // 3. Hit both again to ensure neither broke the other's - // state. - let (h_code, _) = http_request( - port, - "POST", - "/api/workers/heartbeat", - Some(&serde_json::json!({ - "worker_id": serde_json::from_str::(&coord_resp).unwrap()["worker_id"], - "session_token": serde_json::from_str::(&coord_resp).unwrap()["session_token"], - }).to_string()), - ); - assert_eq!(h_code, 200, "heartbeat failed after dashboard call"); - let (idx_code, _) = http_request(port, "GET", "/", None); - assert_eq!(idx_code, 200, "index failed after coord call"); - - kill_child(child); -} - -#[test] -fn coord_serve_dashboard_404_for_unknown_run() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-x", "step1", "fn main() -> Int { 1 }\n"); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - let (code, _) = http_request(port, "GET", "/runs/no-such-id", None); - let (api_code, _) = http_request(port, "GET", "/api/runs/no-such-id", None); - kill_child(child); - assert_eq!(code, 404); - assert_eq!(api_code, 404); -} - -#[test] -fn worker_completes_two_step_linear_dag() { - // Sprint 0.5-S2c: prove the protocol scales beyond a - // single step. Pre-populate two Pending steps; the - // worker claims+completes both. Note: the coordinator - // does NOT yet do DAG advancement (that's 0.5-S2d), so - // this test pre-populates BOTH steps as Pending up - // front. In practice the operator's wave loop would do - // this via separate calls to upsert_step_checkpoint as - // each step's dependency is satisfied. - use boruna_orchestrator::persistence::StepCheckpoint; - let dir = tempfile::tempdir().unwrap(); - let metadata_json = serde_json::json!({ - "step_sources": { - "step1": "fn main() -> Int { 10 }\n", - "step2": "fn main() -> Int { 20 }\n", - } - }) - .to_string(); - std::fs::create_dir_all(dir.path()).unwrap(); - let store = RunCheckpointStore::open(&dir.path().join("runs.db")).unwrap(); - store - .insert_run(&RunRow { - run_id: "run-2step".into(), - workflow_name: "wf".into(), - workflow_hash: "h".into(), - status: RunStatus::Running, - started_at_ms: 0, - updated_at_ms: 0, - policy_json: r#"{"default_allow":true}"#.into(), - metadata_json, - }) - .unwrap(); - for step_id in ["step1", "step2"] { - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: "run-2step".into(), - step_id: step_id.into(), - status: StepStatus::Pending, - output_json: None, - output_hash: None, - started_at_ms: None, - ended_at_ms: None, - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); - } - drop(store); - - let (coord_child, port) = spawn_coordinator(dir.path(), 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "two-step-worker", 30_000); - - let db_path = dir.path().join("runs.db"); - let deadline = Instant::now() + Duration::from_secs(15); - let mut both_completed = false; - let mut last_state = String::new(); - while Instant::now() < deadline { - let store = RunCheckpointStore::open(&db_path).unwrap(); - let cps = store.list_step_checkpoints("run-2step").unwrap(); - last_state = format!( - "{:?}", - cps.iter() - .map(|c| (c.step_id.as_str(), c.status)) - .collect::>() - ); - if cps.len() == 2 && cps.iter().all(|c| c.status == StepStatus::Completed) { - both_completed = true; - // Verify the per-step outputs. - for cp in &cps { - let expected_output = match cp.step_id.as_str() { - "step1" => "10", - "step2" => "20", - _ => panic!("unexpected step_id {}", cp.step_id), - }; - assert_eq!(cp.output_json.as_deref(), Some(expected_output)); - assert!(cp.output_hash.as_deref().unwrap().starts_with("sha256:")); - assert_eq!(cp.claim_id, 1); - } - break; - } - drop(store); - std::thread::sleep(Duration::from_millis(100)); - } - kill_child(worker); - kill_child(coord_child); - assert!( - both_completed, - "expected both steps Completed within 15s; last state: {last_state}" - ); -} - -// ── shared-secret auth (sprint 0.5-S3) ── - -#[test] -fn coord_with_secret_rejects_request_without_bearer() { - // Coord configured with --shared-secret. A naked request to - // /api/workers/register without Authorization header → 401. - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-init", "noop", "fn main() -> Int { 0 }\n"); - let secret = "test-secret-32-hex-chars-aaaa"; - let (child, port) = spawn_coordinator_with_secret(dir.path(), secret); - let cap_hash = boruna_bytecode::compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - let body = serde_json::json!({ "capability_set_hash": cap_hash }).to_string(); - let (code, resp) = - http_request_with_auth(port, "POST", "/api/workers/register", Some(&body), None); - kill_child(child); - assert_eq!(code, 401, "resp: {resp}"); - let v: serde_json::Value = serde_json::from_str(&resp).expect("json"); - assert_eq!(v["error_kind"], "coord.unauthorized"); - assert_eq!(v["protocol_version"], 1); -} - -#[test] -fn coord_with_secret_rejects_request_with_wrong_bearer() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-init", "noop", "fn main() -> Int { 0 }\n"); - let secret = "the-real-secret-aaaaaaaaaaaa"; - let wrong = "the-wrong-secret-bbbbbbbbbbbb"; - let (child, port) = spawn_coordinator_with_secret(dir.path(), secret); - let cap_hash = boruna_bytecode::compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - let body = serde_json::json!({ "capability_set_hash": cap_hash }).to_string(); - let (code, resp) = http_request_with_auth( - port, - "POST", - "/api/workers/register", - Some(&body), - Some(wrong), - ); - kill_child(child); - assert_eq!(code, 401, "resp: {resp}"); - let v: serde_json::Value = serde_json::from_str(&resp).expect("json"); - assert_eq!(v["error_kind"], "coord.unauthorized"); -} - -#[test] -fn coord_with_secret_accepts_request_with_correct_bearer() { - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-init", "noop", "fn main() -> Int { 0 }\n"); - let secret = "matching-secret-xxxxxxxxxxx"; - let (child, port) = spawn_coordinator_with_secret(dir.path(), secret); - let cap_hash = boruna_bytecode::compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - let body = serde_json::json!({ "capability_set_hash": cap_hash }).to_string(); - let (code, resp) = http_request_with_auth( - port, - "POST", - "/api/workers/register", - Some(&body), - Some(secret), - ); - kill_child(child); - assert_eq!(code, 200, "resp: {resp}"); - let v: serde_json::Value = serde_json::from_str(&resp).expect("json"); - assert_eq!(v["protocol_version"], 1); - assert!(v["worker_id"].as_str().unwrap().starts_with("wkr-")); -} - -#[test] -fn coord_without_secret_accepts_unauth_request_no_regression() { - // Existing test surface: when no --shared-secret, no auth required. - // This duplicates `coord_register_returns_worker_id_and_session_token` - // explicitly via the auth-aware HTTP helper to lock the no-regression - // contract. Sprint 0.5-S3 must not break loopback-only deployments. - let dir = tempfile::tempdir().unwrap(); - populate_pending_step(dir.path(), "run-init", "noop", "fn main() -> Int { 0 }\n"); - let (child, port) = spawn_coordinator(dir.path(), 60_000, 200); - let cap_hash = boruna_bytecode::compute_capability_set_hash( - boruna_bytecode::Capability::ALL - .iter() - .map(|c| (c.name().to_string(), c.version().to_string())) - .collect::>() - .iter() - .map(|(n, v)| (n.as_str(), v.as_str())), - ); - let body = serde_json::json!({ "capability_set_hash": cap_hash }).to_string(); - let (code, _resp) = - http_request_with_auth(port, "POST", "/api/workers/register", Some(&body), None); - kill_child(child); - assert_eq!(code, 200); -} - -// ── Sprint 0.5-S4 — `workflow run --coordinator` end-to-end ── - -fn write_single_step_workflow(wf_dir: &Path, body: &str) { - std::fs::create_dir_all(wf_dir).unwrap(); - std::fs::write(wf_dir.join("s1.ax"), body).unwrap(); - std::fs::write( - wf_dir.join("workflow.json"), - r#"{ - "schema_version": 1, - "name": "remote-run-test", - "version": "1.0.0", - "steps": { - "s1": {"kind": "source", "source": "s1.ax", "capabilities": [], "outputs": {"result": "Int"}} - }, - "edges": [] - }"#, - ) - .unwrap(); -} - -#[test] -fn cli_workflow_run_coordinator_drives_remote_run_to_completion() { - // End-to-end: operator's CLI submits a workflow over HTTP to a - // remote coordinator (different data-dir), the coordinator - // dispatches it to a connected worker, the worker runs the - // step, and the CLI's polling loop sees Completed and exits 0. - let tmp = tempfile::tempdir().unwrap(); - let coord_data = tmp.path().join("coord-data"); - std::fs::create_dir_all(&coord_data).unwrap(); - // Coordinator's `serve` requires runs.db to already exist (the - // pre-0.5-S4 model assumed an operator had submitted at least - // one workflow locally first). Touch the schema by opening the - // store and dropping it. - drop(RunCheckpointStore::open(&coord_data.join("runs.db")).unwrap()); - let wf_dir = tmp.path().join("wf"); - write_single_step_workflow(&wf_dir, "fn main() -> Int { 7 }\n"); - - let (coord_child, port) = spawn_coordinator(&coord_data, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "remote-run-worker", 30_000); - - let out = Command::new(boruna_bin()) - .args([ - "workflow", - "run", - wf_dir.to_str().unwrap(), - "--coordinator", - &coord_url, - "--coord-poll-interval-ms", - "200", - "--coord-max-wait-secs", - "30", - ]) - .output() - .expect("invoke boruna workflow run --coordinator"); - - kill_child(worker); - kill_child(coord_child); - - let stdout = String::from_utf8_lossy(&out.stdout); - let stderr = String::from_utf8_lossy(&out.stderr); - assert!( - out.status.success(), - "workflow run --coordinator failed:\nstdout={stdout}\nstderr={stderr}" - ); - assert_eq!(out.status.code(), Some(0)); - assert!( - stdout.contains("step s1: completed") && stdout.contains(": completed"), - "expected completion lines in stdout, got: {stdout}" - ); -} - -#[test] -fn cli_workflow_run_coordinator_exits_1_on_step_failure() { - // Mirror of the success case but with a step source that fails - // at runtime. Exit code must be 1 (Failed), not 2 (timeout). - let tmp = tempfile::tempdir().unwrap(); - let coord_data = tmp.path().join("coord-data"); - std::fs::create_dir_all(&coord_data).unwrap(); - // Coordinator's `serve` requires runs.db to already exist (the - // pre-0.5-S4 model assumed an operator had submitted at least - // one workflow locally first). Touch the schema by opening the - // store and dropping it. - drop(RunCheckpointStore::open(&coord_data.join("runs.db")).unwrap()); - let wf_dir = tmp.path().join("wf"); - // Use match exhaustion to force a runtime failure. - write_single_step_workflow( - &wf_dir, - r#"fn main() -> Int { match 99 { 0 => 0, 1 => 1 } } -"#, - ); - - let (coord_child, port) = spawn_coordinator(&coord_data, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "remote-run-fail-worker", 30_000); - - let out = Command::new(boruna_bin()) - .args([ - "workflow", - "run", - wf_dir.to_str().unwrap(), - "--coordinator", - &coord_url, - "--coord-poll-interval-ms", - "200", - "--coord-max-wait-secs", - "30", - ]) - .output() - .expect("invoke boruna workflow run --coordinator"); - - kill_child(worker); - kill_child(coord_child); - - let stdout = String::from_utf8_lossy(&out.stdout); - assert_eq!( - out.status.code(), - Some(1), - "expected exit 1 (Failed), got {:?}\nstdout={stdout}", - out.status.code() - ); - assert!( - stdout.contains("failed"), - "expected 'failed' line in stdout, got: {stdout}" - ); -} - -#[test] -fn cli_workflow_approve_via_coordinator_advances_remote_run() { - // Sprint 0.5-S6: end-to-end approval gate over HTTP. - // 1. Submit a workflow with an approval gate via --coordinator. - // The CLI submits + polls; while it polls in the foreground, - // a worker drives `analyze` to Completed; the gate then - // opens (AwaitingApproval) and the foreground CLI keeps - // polling. - // 2. From a separate process we run `workflow approve --coordinator` - // against the open gate. - // 3. The foreground CLI sees Completed and exits 0. - let tmp = tempfile::tempdir().unwrap(); - let coord_data = tmp.path().join("coord-data"); - std::fs::create_dir_all(&coord_data).unwrap(); - drop(RunCheckpointStore::open(&coord_data.join("runs.db")).unwrap()); - let wf_dir = tmp.path().join("wf"); - std::fs::create_dir_all(&wf_dir).unwrap(); - std::fs::write(wf_dir.join("analyze.ax"), "fn main() -> Int { 1 }\n").unwrap(); - std::fs::write( - wf_dir.join("workflow.json"), - r#"{ - "schema_version": 1, - "name": "approve-test", - "version": "1.0.0", - "steps": { - "analyze": {"kind": "source", "source": "analyze.ax", "capabilities": [], "outputs": {"result": "Int"}}, - "human_review": {"kind": "approval_gate", "required_role": "reviewer", "depends_on": ["analyze"], "capabilities": [], "outputs": {}} - }, - "edges": [["analyze", "human_review"]] - }"#, - ) - .unwrap(); - - let (coord_child, port) = spawn_coordinator(&coord_data, 60_000, 1_000); - let coord_url = format!("http://127.0.0.1:{port}"); - let worker = spawn_worker(&coord_url, "approve-worker", 30_000); - - // Spawn the foreground `workflow run --coordinator` in the - // background so the approval can race-in while it polls. - let mut run_child = Command::new(boruna_bin()) - .args([ - "workflow", - "run", - wf_dir.to_str().unwrap(), - "--coordinator", - &coord_url, - "--coord-poll-interval-ms", - "200", - "--coord-max-wait-secs", - "30", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("spawn workflow run"); - - // Wait for analyze to finish + gate to open (poll the dashboard - // store directly to find the run_id). - let deadline = Instant::now() + Duration::from_secs(20); - let mut run_id = String::new(); - while Instant::now() < deadline { - let store = RunCheckpointStore::open(&coord_data.join("runs.db")).unwrap(); - let runs = store.list_runs().unwrap(); - if let Some(r) = runs.first() { - let cps = store.list_step_checkpoints(&r.run_id).unwrap(); - if cps - .iter() - .any(|c| c.step_id == "human_review" && c.status == StepStatus::AwaitingApproval) - { - run_id = r.run_id.clone(); - break; - } - } - drop(store); - std::thread::sleep(Duration::from_millis(100)); - } - assert!(!run_id.is_empty(), "gate never opened within 20s"); - - // S9: read the per-gate approval token stashed when the gate opened and - // pass it to `workflow approve` — the coordinator now requires it. - let approval_token = { - let store = RunCheckpointStore::open(&coord_data.join("runs.db")).unwrap(); - boruna_orchestrator::workflow::approval_gate_token(&store, &run_id, "human_review") - .unwrap() - .expect("open approval gate should have a stashed token") - }; - - // Approve via remote CLI. - let approve = Command::new(boruna_bin()) - .args([ - "workflow", - "approve", - &run_id, - "human_review", - "--coordinator", - &coord_url, - "--token", - &approval_token, - ]) - .output() - .expect("invoke workflow approve --coordinator"); - assert!( - approve.status.success(), - "approve failed: stderr={}", - String::from_utf8_lossy(&approve.stderr) - ); - - // Foreground CLI should now exit 0. - let exit = run_child.wait().expect("wait foreground run"); - kill_child(worker); - kill_child(coord_child); - assert_eq!( - exit.code(), - Some(0), - "foreground CLI did not exit 0 (got {:?})", - exit.code() - ); -} - -#[test] -fn cli_workflow_run_coordinator_rejects_data_dir_combo() { - // --coordinator and --data-dir are mutually exclusive at the - // clap level. clap should refuse before any side effect. - let tmp = tempfile::tempdir().unwrap(); - let wf_dir = tmp.path().join("wf"); - write_single_step_workflow(&wf_dir, "fn main() -> Int { 1 }\n"); - let out = Command::new(boruna_bin()) - .args([ - "workflow", - "run", - wf_dir.to_str().unwrap(), - "--coordinator", - "http://127.0.0.1:1", - "--data-dir", - tmp.path().to_str().unwrap(), - ]) - .output() - .expect("invoke boruna workflow run"); - assert!( - !out.status.success(), - "expected clap to reject --coordinator + --data-dir" - ); - let stderr = String::from_utf8_lossy(&out.stderr); - assert!( - stderr.contains("cannot be used with") || stderr.contains("conflicts"), - "expected conflict error, got: {stderr}" - ); -} diff --git a/crates/llmvm-cli/tests/cli_dashboard.rs b/crates/llmvm-cli/tests/cli_dashboard.rs deleted file mode 100644 index 2d7e103..0000000 --- a/crates/llmvm-cli/tests/cli_dashboard.rs +++ /dev/null @@ -1,297 +0,0 @@ -//! End-to-end CLI integration tests for `boruna dashboard serve` -//! (sprint 0.4-S16). These tests spawn the binary, hit the running -//! HTTP server, and assert end-to-end behavior. -//! -//! Only compiled when `--features serve` is enabled. - -#![cfg(feature = "serve")] - -use std::io::{BufRead, BufReader, Read, Write}; -use std::net::{Shutdown, TcpListener, TcpStream}; -use std::path::Path; -use std::process::{Child, Command, Stdio}; -use std::time::{Duration, Instant}; - -fn boruna_bin() -> &'static str { - env!("CARGO_BIN_EXE_boruna") -} - -/// Find a free TCP port on the loopback interface. -fn pick_free_port() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); - let port = listener.local_addr().unwrap().port(); - drop(listener); - port -} - -/// Wait for the server to become reachable on `127.0.0.1:port`. -/// Returns the running child handle on success. -fn wait_for_server(port: u16) { - let deadline = Instant::now() + Duration::from_secs(5); - while Instant::now() < deadline { - if TcpStream::connect_timeout( - &format!("127.0.0.1:{port}").parse().unwrap(), - Duration::from_millis(200), - ) - .is_ok() - { - return; - } - std::thread::sleep(Duration::from_millis(50)); - } - panic!("server on port {port} never came up within 5s"); -} - -/// Hand-rolled minimal HTTP/1.1 GET to keep test deps out of the -/// CLI Cargo.toml. Returns `(status_code, body)`. -fn http_get(port: u16, path: &str) -> (u16, String) { - http_request(port, "GET", path) -} - -fn http_request(port: u16, method: &str, path: &str) -> (u16, String) { - // Retry-on-ConnectionRefused: the port-pick → spawn → wait_for_server - // → http_request chain has a small race window on busy CI runners - // (kernel re-assigning ephemeral ports, child process re-binding). - // Observed once on the self-hosted runner under v1.0.0-rc2 push. - // Retrying a few times with backoff is the standard fix; the test - // is exercising HTTP responses, not connect-establishment timing. - let mut last_err: Option = None; - let mut stream = (0..5) - .find_map(|attempt| { - match TcpStream::connect_timeout( - &format!("127.0.0.1:{port}").parse().unwrap(), - Duration::from_millis(500), - ) { - Ok(s) => Some(s), - Err(e) => { - last_err = Some(e); - std::thread::sleep(Duration::from_millis(50 * (attempt + 1))); - None - } - } - }) - .unwrap_or_else(|| { - panic!( - "connect to 127.0.0.1:{port} failed after 5 retries; last err: {}", - last_err - .as_ref() - .map(|e| e.to_string()) - .unwrap_or_else(|| "unknown".into()) - ) - }); - stream - .set_read_timeout(Some(Duration::from_secs(3))) - .unwrap(); - let req = - format!("{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n"); - stream.write_all(req.as_bytes()).expect("write request"); - let mut reader = BufReader::new(&stream); - let mut status_line = String::new(); - reader.read_line(&mut status_line).expect("read status"); - // Parse "HTTP/1.1 200 OK\r\n" - let parts: Vec<&str> = status_line.split_whitespace().collect(); - let code: u16 = parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0); - // Skip headers until blank line - loop { - let mut line = String::new(); - let n = reader.read_line(&mut line).unwrap_or(0); - if n == 0 || line == "\r\n" || line == "\n" { - break; - } - } - let mut body = String::new(); - let _ = reader.read_to_string(&mut body); - let _ = stream.shutdown(Shutdown::Both); - (code, body) -} - -/// Spawn the dashboard, return the running child + port. -/// Caller MUST call `kill_child` when done. -fn spawn_dashboard(data_dir: &Path) -> (Child, u16) { - let port = pick_free_port(); - let child = Command::new(boruna_bin()) - .args([ - "dashboard", - "serve", - "--data-dir", - data_dir.to_str().unwrap(), - "--port", - &port.to_string(), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn boruna dashboard"); - wait_for_server(port); - (child, port) -} - -fn kill_child(mut child: Child) { - let _ = child.kill(); - let _ = child.wait(); -} - -/// Populate a runs.db with a single run + step using the -/// orchestrator's persistence APIs directly. Faster than running -/// an actual workflow. -fn populate_db(data_dir: &Path) { - use boruna_orchestrator::persistence::{ - RunCheckpointStore, RunRow, RunStatus, StepCheckpoint, StepStatus, - }; - std::fs::create_dir_all(data_dir).unwrap(); - let store = RunCheckpointStore::open(&data_dir.join("runs.db")).unwrap(); - store - .insert_run(&RunRow { - run_id: "r-test".into(), - workflow_name: "etl".into(), - workflow_hash: "deadbeef".into(), - status: RunStatus::Running, - started_at_ms: 1_700_000_000_000, - updated_at_ms: 1_700_000_000_500, - policy_json: "{}".into(), - metadata_json: "{}".into(), - }) - .unwrap(); - store - .upsert_step_checkpoint(&StepCheckpoint { - run_id: "r-test".into(), - step_id: "extract".into(), - status: StepStatus::Completed, - output_json: None, - output_hash: None, - started_at_ms: Some(1_700_000_001_000), - ended_at_ms: Some(1_700_000_002_000), - error_msg: None, - attempt_count: 1, - worker_id: None, - lease_expires_at_ms: None, - claim_id: 0, - output_blob_ref: None, - }) - .unwrap(); -} - -#[test] -fn cli_dashboard_serve_responds_to_index() { - let dir = tempfile::tempdir().unwrap(); - populate_db(dir.path()); - let (child, port) = spawn_dashboard(dir.path()); - let (code, body) = http_get(port, "/"); - kill_child(child); - assert_eq!(code, 200); - assert!(body.contains(" Date: Sat, 18 Jul 2026 11:33:28 +0300 Subject: [PATCH 2/2] refactor(cli): remove the now-dead --coordinator flags entirely Follow-up to the HTTP-layer removal: the `workflow run/approve/reject/trigger` commands still carried `--coordinator` / `--coord-token` (and run's poll/wait, approve/reject's per-gate `--token`) flags that only errored. Removed them outright, dropped the neutralized dispatch branches, and cleaned the `conflicts_with = "coordinator"` on `--data-dir`. Kept `trigger --token` (the local external-trigger token, still required). Local commands unchanged. Verified: boruna-cli build + 95 tests pass (0 failed), clippy -D warnings clean, fmt clean. --- crates/llmvm-cli/src/main.rs | 255 +++++++++-------------------------- 1 file changed, 63 insertions(+), 192 deletions(-) diff --git a/crates/llmvm-cli/src/main.rs b/crates/llmvm-cli/src/main.rs index ba8c7ca..291ed8a 100644 --- a/crates/llmvm-cli/src/main.rs +++ b/crates/llmvm-cli/src/main.rs @@ -584,44 +584,6 @@ enum WorkflowCommand { /// deferred to a future sprint. #[arg(long, conflicts_with_all = ["ephemeral", "skip_if_running"])] submit_only: bool, - /// Submit the workflow to a remote coordinator over HTTP and - /// poll for terminal status (sprint `0.5-S4`). The CI runner - /// does NOT need filesystem access to the cluster's data-dir; - /// the workflow.json + every Source-kind step's `.ax` body - /// are inlined into the submit payload. Bearer token via - /// `--coord-token` or the `BORUNA_TOKEN` env var when the - /// cluster is auth-gated (0.5-S3). - /// - /// Mutually exclusive with `--data-dir` (different - /// operational model entirely), `--ephemeral`, - /// `--submit-only`, and `--skip-if-running`. Exit codes - /// match `coordinator wait`: `0` Completed, `1` Failed, - /// `2` timeout / submit-failed. - #[arg( - long, - value_name = "URL", - conflicts_with_all = ["data_dir", "ephemeral", "submit_only", "skip_if_running"] - )] - coordinator: Option, - /// Bearer token for the coordinator's auth middleware (sprint - /// `0.5-S3`). Only meaningful with `--coordinator`. Falls back - /// to the `BORUNA_TOKEN` env var. Omit if the cluster is - /// loopback / unauthenticated. - #[arg(long, value_name = "BEARER", env = "BORUNA_TOKEN")] - coord_token: Option, - /// How often to poll `/api/runs/{run_id}/status` when running - /// against `--coordinator`. Defaults to `1000` ms; clamped - /// silently to a `500`-ms floor matching `coordinator wait`. - /// Ignored without `--coordinator`. - #[arg(long, default_value = "1000")] - coord_poll_interval_ms: u64, - /// Maximum total wall-clock time to wait for terminal status - /// in `--coordinator` mode. `0` (default) means wait - /// indefinitely — same posture as `coordinator wait`. On - /// timeout the CLI exits with `2` and the run keeps going - /// on the cluster. - #[arg(long, default_value = "0")] - coord_max_wait_secs: u64, /// CI/CD safety check: refuse to run if the on-disk def's /// workflow_hash doesn't match this value (case-insensitive /// 64-char SHA-256 hex). Capture via `boruna workflow @@ -669,26 +631,8 @@ enum WorkflowCommand { run_id: String, /// Step id of the approval gate to approve. step_id: String, - #[arg(long, conflicts_with = "coordinator")] + #[arg(long)] data_dir: Option, - /// Sprint 0.5-S6: drive a remote coordinator over HTTP instead - /// of mutating a local data-dir. POSTs to - /// `/api/runs/{run_id}/approve`. Mutually exclusive with - /// `--data-dir`. Bearer token via `--coord-token` or - /// `BORUNA_TOKEN` env var. - #[arg(long, value_name = "URL")] - coordinator: Option, - /// Bearer token for the coordinator's auth middleware. Falls - /// back to the `BORUNA_TOKEN` env var. Only meaningful with - /// `--coordinator`. - #[arg(long, value_name = "BEARER", env = "BORUNA_TOKEN")] - coord_token: Option, - /// Per-gate approval token stashed at pause-time (finding S9). Required - /// by a remote coordinator's `/approve` — without it the gate cannot be - /// approved. Distinct from `--coord-token` (the auth bearer). Retrieve - /// it from the paused run's status. Only meaningful with `--coordinator`. - #[arg(long, value_name = "TOKEN")] - token: Option, }, /// Reject a paused approval-gate step. Records a rejection sentinel; /// `boruna workflow resume ` will then halt the run as @@ -700,19 +644,8 @@ enum WorkflowCommand { /// resumed run's step error_msg. #[arg(long)] reason: Option, - #[arg(long, conflicts_with = "coordinator")] + #[arg(long)] data_dir: Option, - /// Sprint 0.5-S6: drive a remote coordinator over HTTP. POSTs - /// to `/api/runs/{run_id}/approve` with `decision: "rejected"`. - #[arg(long, value_name = "URL")] - coordinator: Option, - #[arg(long, value_name = "BEARER", env = "BORUNA_TOKEN")] - coord_token: Option, - /// Per-gate approval token stashed at pause-time (finding S9), required - /// by a remote coordinator's `/approve`. Only meaningful with - /// `--coordinator`. - #[arg(long, value_name = "TOKEN")] - token: Option, }, /// Trigger a paused external_trigger step (sprint 0.3-S15). Records /// the supplied payload as the step's output and primes resume to @@ -743,15 +676,8 @@ enum WorkflowCommand { /// exclusive with `--payload`. Useful for large webhook bodies. #[arg(long)] payload_file: Option, - #[arg(long, conflicts_with = "coordinator")] + #[arg(long)] data_dir: Option, - /// Sprint 0.5-S6: drive a remote coordinator over HTTP. POSTs - /// to `/api/runs/{run_id}/trigger`. Mutually exclusive with - /// `--data-dir`. - #[arg(long, value_name = "URL")] - coordinator: Option, - #[arg(long, value_name = "BEARER", env = "BORUNA_TOKEN")] - coord_token: Option, }, /// Show the full state of a single run: row, step checkpoints, and /// approval-gate decisions. Use `--json` for machine-readable output @@ -2733,10 +2659,6 @@ fn run_workflow( concurrency, skip_if_running, submit_only, - coordinator, - coord_token, - coord_poll_interval_ms, - coord_max_wait_secs, expect_workflow_hash, bundle_storage, providers, @@ -2782,24 +2704,6 @@ fn run_workflow( } }; - // Sprint 0.5-S4: --coordinator branches off the local- - // run path entirely. Build the inline submit payload, - // POST it, then poll status until terminal. Exit with - // the conventional code (0/1/2) and skip the rest of - // the local-run flow. - if let Some(coord_url) = coordinator { - let _ = ( - coord_url, - coord_token, - coord_poll_interval_ms, - coord_max_wait_secs, - ); - return Err("`workflow run --coordinator` is no longer supported — \ - distributed coordinator execution has been removed; \ - run workflows locally" - .into()); - } - let options = RunOptions { policy: Some(policy_obj.clone()), record, @@ -3077,40 +2981,29 @@ fn run_workflow( run_id, step_id, data_dir, - coordinator, - coord_token, - token, } => { - if let Some(url) = coordinator { - let _ = (url, coord_token, token); - return Err("`workflow approve --coordinator` is no longer supported — \ - distributed coordinator execution has been removed" - .into()); - } else { - let _ = token; // local approve is operator-trusted; no gate token - #[cfg(feature = "persist-sqlite")] - { - use boruna_orchestrator::workflow::{record_approval_decision, ApprovalKind}; - let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); - record_approval_decision( - &resolved, - &run_id, - &step_id, - ApprovalKind::Approved, - None, - ) - .map_err(|e| format!("{e}"))?; - println!("approval recorded for step '{step_id}' in run '{run_id}'."); - println!( - "Run `boruna workflow resume {run_id} --data-dir {}` to advance.", - resolved.display() - ); - } - #[cfg(not(feature = "persist-sqlite"))] - { - let _ = (run_id, step_id, data_dir); - return Err("`workflow approve` requires the `persist-sqlite` feature".into()); - } + #[cfg(feature = "persist-sqlite")] + { + use boruna_orchestrator::workflow::{record_approval_decision, ApprovalKind}; + let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); + record_approval_decision( + &resolved, + &run_id, + &step_id, + ApprovalKind::Approved, + None, + ) + .map_err(|e| format!("{e}"))?; + println!("approval recorded for step '{step_id}' in run '{run_id}'."); + println!( + "Run `boruna workflow resume {run_id} --data-dir {}` to advance.", + resolved.display() + ); + } + #[cfg(not(feature = "persist-sqlite"))] + { + let _ = (run_id, step_id, data_dir); + return Err("`workflow approve` requires the `persist-sqlite` feature".into()); } } WorkflowCommand::Reject { @@ -3118,40 +3011,29 @@ fn run_workflow( step_id, reason, data_dir, - coordinator, - coord_token, - token, } => { - if let Some(url) = coordinator { - let _ = (url, coord_token, reason, token); - return Err("`workflow reject --coordinator` is no longer supported — \ - distributed coordinator execution has been removed" - .into()); - } else { - let _ = token; // local reject is operator-trusted; no gate token - #[cfg(feature = "persist-sqlite")] - { - use boruna_orchestrator::workflow::{record_approval_decision, ApprovalKind}; - let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); - record_approval_decision( - &resolved, - &run_id, - &step_id, - ApprovalKind::Rejected, - reason, - ) - .map_err(|e| format!("{e}"))?; - println!("rejection recorded for step '{step_id}' in run '{run_id}'."); - println!( - "Run `boruna workflow resume {run_id} --data-dir {}` to halt the run.", - resolved.display() - ); - } - #[cfg(not(feature = "persist-sqlite"))] - { - let _ = (run_id, step_id, reason, data_dir); - return Err("`workflow reject` requires the `persist-sqlite` feature".into()); - } + #[cfg(feature = "persist-sqlite")] + { + use boruna_orchestrator::workflow::{record_approval_decision, ApprovalKind}; + let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); + record_approval_decision( + &resolved, + &run_id, + &step_id, + ApprovalKind::Rejected, + reason, + ) + .map_err(|e| format!("{e}"))?; + println!("rejection recorded for step '{step_id}' in run '{run_id}'."); + println!( + "Run `boruna workflow resume {run_id} --data-dir {}` to halt the run.", + resolved.display() + ); + } + #[cfg(not(feature = "persist-sqlite"))] + { + let _ = (run_id, step_id, reason, data_dir); + return Err("`workflow reject` requires the `persist-sqlite` feature".into()); } } WorkflowCommand::Trigger { @@ -3161,8 +3043,6 @@ fn run_workflow( payload, payload_file, data_dir, - coordinator, - coord_token, } => { let payload_str = match (payload, payload_file) { (Some(p), None) => p, @@ -3179,34 +3059,25 @@ fn run_workflow( } }; // Defense-in-depth: confirm the payload is well-formed JSON. - // Same posture for both local and remote paths so operators - // get the early failure regardless of mode. serde_json::from_str::(&payload_str) .map_err(|e| format!("--payload is not valid JSON: {e}"))?; - if let Some(url) = coordinator { - let _ = (url, coord_token); - return Err("`workflow trigger --coordinator` is no longer supported — \ - distributed coordinator execution has been removed" - .into()); - } else { - #[cfg(feature = "persist-sqlite")] - { - use boruna_orchestrator::workflow::record_external_trigger; - let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); - record_external_trigger(&resolved, &run_id, &step_id, &token, &payload_str) - .map_err(|e| format!("{e}"))?; - println!("trigger recorded for step '{step_id}' in run '{run_id}'."); - println!( - "Run `boruna workflow resume {run_id} --data-dir {}` to advance.", - resolved.display() - ); - } - #[cfg(not(feature = "persist-sqlite"))] - { - let _ = (run_id, step_id, token, data_dir); - return Err("`workflow trigger` requires the `persist-sqlite` feature".into()); - } + #[cfg(feature = "persist-sqlite")] + { + use boruna_orchestrator::workflow::record_external_trigger; + let resolved = resolve_data_dir(data_dir.as_ref(), env_arg); + record_external_trigger(&resolved, &run_id, &step_id, &token, &payload_str) + .map_err(|e| format!("{e}"))?; + println!("trigger recorded for step '{step_id}' in run '{run_id}'."); + println!( + "Run `boruna workflow resume {run_id} --data-dir {}` to advance.", + resolved.display() + ); + } + #[cfg(not(feature = "persist-sqlite"))] + { + let _ = (run_id, step_id, token, data_dir); + return Err("`workflow trigger` requires the `persist-sqlite` feature".into()); } } WorkflowCommand::Show {