diff --git a/.github/actions/setup-e2e-cli/action.yml b/.github/actions/setup-e2e-cli/action.yml index d7d1c8a296..8b30970d8d 100644 --- a/.github/actions/setup-e2e-cli/action.yml +++ b/.github/actions/setup-e2e-cli/action.yml @@ -1,12 +1,6 @@ name: Setup E2E CLI description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests -inputs: - conformance-artifact-prefix: - description: Optional conformance CLI artifact name prefix; linux- is appended automatically - required: false - default: "" - runs: using: composite steps: @@ -23,27 +17,3 @@ runs: chmod +x "$cli" "$cli" --version echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV" - - - name: Download prebuilt conformance CLI - if: inputs.conformance-artifact-prefix != '' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ format('{0}-{1}', inputs.conformance-artifact-prefix, runner.arch == 'X64' && 'x86_64-unknown-linux-musl' || 'aarch64-unknown-linux-musl') }} - path: .e2e/prebuilt-conformance - - - name: Configure prebuilt conformance CLI - if: inputs.conformance-artifact-prefix != '' - shell: bash - run: | # zizmor: ignore[github-env] validated filename under the trusted workspace path - set -euo pipefail - conformance="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance" - if [[ ! -f "$conformance" ]]; then - echo "downloaded artifact is missing $conformance" >&2 - exit 1 - fi - # TODO: Remove this temporary mode diagnostic after CI confirms artifact permission handling. - echo "conformance binary mode before chmod: $(ls -l "$conformance")" - chmod +x "$conformance" - echo "conformance binary mode after chmod: $(ls -l "$conformance")" - "$conformance" list --output json >/dev/null - echo "OPENSHELL_CONFORMANCE_BIN=$conformance" >> "$GITHUB_ENV" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 85097eea56..404704ce4a 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -321,7 +321,6 @@ jobs: with: image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 - conformance-artifact-prefix: openshell-conformance podman-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -333,7 +332,6 @@ jobs: uses: ./.github/workflows/e2e-podman-test.yml with: image-tag: ${{ github.sha }} - conformance-artifact-prefix: openshell-conformance suite-matrix: >- [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}] @@ -345,8 +343,6 @@ jobs: contents: read packages: read uses: ./.github/workflows/e2e-vm-test.yml - with: - conformance-artifact-prefix: openshell-conformance docker-external-driver-e2e: needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] @@ -361,7 +357,6 @@ jobs: runner: linux-arm64-cpu8 gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu external-driver-binary: openshell-driver-docker - conformance-artifact-prefix: openshell-conformance suite-matrix: >- [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] @@ -377,7 +372,6 @@ jobs: image-tag: ${{ github.sha }} gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu external-driver-binary: openshell-driver-podman - conformance-artifact-prefix: openshell-conformance suite-matrix: >- [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] @@ -393,7 +387,6 @@ jobs: gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu suite-name: external-driver e2e-task: e2e:vm:external-driver - conformance-artifact-prefix: openshell-conformance gpu-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -405,7 +398,6 @@ jobs: uses: ./.github/workflows/e2e-gpu-test.yaml with: image-tag: ${{ github.sha }} - conformance-artifact-prefix: openshell-conformance kubernetes-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -427,7 +419,6 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) agent-sandbox-version: ${{ matrix.agent_sandbox_version }} - conformance-artifact-prefix: openshell-conformance kubernetes-workspace-managed-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -441,7 +432,6 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed - conformance-artifact-prefix: openshell-conformance kubernetes-external-driver-e2e: needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images] @@ -458,7 +448,6 @@ jobs: gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu external-driver-binary: openshell-driver-kubernetes cluster-images: sandbox supervisor - conformance-artifact-prefix: openshell-conformance kubernetes-workspace-operator-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -472,7 +461,6 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator - conformance-artifact-prefix: openshell-conformance kubernetes-ha-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -490,7 +478,6 @@ jobs: test-name: kubernetes_ha_rebalancing kubernetes-features: e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha use-envoy-gateway: true - conformance-artifact-prefix: openshell-conformance kubernetes-credential-drivers-e2e: needs: [pr_metadata, build-binaries, build-images] @@ -504,7 +491,6 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes Credential Drivers E2E e2e-task: e2e:kubernetes:credential-drivers - conformance-artifact-prefix: openshell-conformance core-e2e-result: name: Core E2E result diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 82ef374939..1289968d14 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -62,22 +62,6 @@ jobs: supervisor-image-tag: ${{ inputs.supervisor-image-tag }} extra-cargo-flags: ${{ env.EXTRA_CARGO_FLAGS }} - conformance: - name: openshell-conformance (${{ matrix.triple }}) - env: - PACKAGE: openshell-conformance-cli - BINARY: openshell-conformance - strategy: - matrix: - include: - - triple: x86_64-unknown-linux-musl - runner: linux-amd64-cpu8 - - triple: aarch64-unknown-linux-musl - runner: linux-arm64-cpu8 - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - steps: *build_steps - prover: name: openshell-prover (${{ matrix.triple }}) env: diff --git a/.github/workflows/e2e-docker-test.yml b/.github/workflows/e2e-docker-test.yml index 24eb4a54f7..fa49a1a933 100644 --- a/.github/workflows/e2e-docker-test.yml +++ b/.github/workflows/e2e-docker-test.yml @@ -25,10 +25,6 @@ on: required: false type: string default: "" - conformance-artifact-prefix: - required: false - type: string - default: "" suite-matrix: required: false type: string @@ -71,8 +67,6 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli - with: - conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-gpu-test.yaml b/.github/workflows/e2e-gpu-test.yaml index b363a0509b..1a8275a274 100644 --- a/.github/workflows/e2e-gpu-test.yaml +++ b/.github/workflows/e2e-gpu-test.yaml @@ -7,11 +7,6 @@ on: description: "Image tag to test (typically the commit SHA)" required: true type: string - conformance-artifact-prefix: - description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is )" - required: false - type: string - default: "" permissions: actions: read @@ -66,8 +61,6 @@ jobs: - name: Use prebuilt OpenShell CLI uses: ./.github/actions/setup-e2e-cli - with: - conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - name: Use prebuilt OpenShell gateway uses: ./.github/actions/setup-e2e-gateway diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 0d018187a8..449f3c25f0 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -67,11 +67,6 @@ on: required: false type: string default: "" - conformance-artifact-prefix: - description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is )" - required: false - type: string - default: "" external-driver-binary: description: "Optional standalone compute driver binary used to compose a local test image" required: false @@ -112,8 +107,6 @@ jobs: - name: Use prebuilt OpenShell CLI uses: ./.github/actions/setup-e2e-cli - with: - conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact != '' diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index 4ca40da39c..d70ad6c60c 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -21,10 +21,6 @@ on: required: false type: string default: "" - conformance-artifact-prefix: - required: false - type: string - default: "" suite-matrix: required: true type: string @@ -56,8 +52,6 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli - with: - conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/e2e-vm-test.yml b/.github/workflows/e2e-vm-test.yml index 954de9669e..57e04c32b3 100644 --- a/.github/workflows/e2e-vm-test.yml +++ b/.github/workflows/e2e-vm-test.yml @@ -22,10 +22,6 @@ on: required: false type: string default: e2e:vm - conformance-artifact-prefix: - required: false - type: string - default: "" permissions: actions: read @@ -47,8 +43,6 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-e2e-cli - with: - conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - uses: ./.github/actions/setup-e2e-gateway with: diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 961f61ecde..e3523e45c7 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -159,7 +159,6 @@ jobs: image-tag: ${{ github.sha }} checkout-ref: ${{ github.sha }} runner: linux-arm64-cpu8 - conformance-artifact-prefix: openshell-conformance vm-e2e: needs: [build-binaries, build-vm-driver] @@ -170,7 +169,6 @@ jobs: uses: ./.github/workflows/e2e-vm-test.yml with: checkout-ref: ${{ github.sha }} - conformance-artifact-prefix: openshell-conformance tag-ghcr-dev: name: Tag GHCR Images as Dev diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index da0ef42df2..4c42200d06 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -210,7 +210,6 @@ jobs: image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ needs.compute-versions.outputs.source_sha }} runner: linux-arm64-cpu8 - conformance-artifact-prefix: openshell-conformance vm-e2e: needs: [compute-versions, build-binaries, build-vm-driver] @@ -221,7 +220,6 @@ jobs: uses: ./.github/workflows/e2e-vm-test.yml with: checkout-ref: ${{ needs.compute-versions.outputs.source_sha }} - conformance-artifact-prefix: openshell-conformance protobuf-compatibility: name: Protobuf Compatibility diff --git a/AGENTS.md b/AGENTS.md index b049c28f50..c6af760499 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,6 @@ Do not rely on this file for a full inventory. The detailed public and contribut |------|-----------|---------| | `crates/openshell-cli/` | CLI binary | User-facing command-line interface | | `crates/openshell-conformance/` | CLI conformance library | Reusable driver-agnostic scenarios and command runner | -| `crates/openshell-conformance-cli/` | Conformance CLI | Legacy local `list` and `run` entrypoint pending follow-up cleanup | | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O | | `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking | diff --git a/Cargo.lock b/Cargo.lock index fc0b173702..5b169a43da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4039,17 +4039,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "openshell-conformance-cli" -version = "0.0.0" -dependencies = [ - "clap", - "openshell-conformance", - "serde", - "serde_json", - "tokio", -] - [[package]] name = "openshell-core" version = "0.0.0" diff --git a/TESTING.md b/TESTING.md index a1c2f9476e..570d135f5b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -213,11 +213,12 @@ HTTP `Host`, TLS SNI, and mTLS handling. Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. -- CLI conformance (`openshell-conformance`) - named scenarios for lifecycle, - mechanistic drafts, and the sandbox-local API, including agent-authored - permission requests. Driver E2E wrappers run every scenario. The - installed-artifact conformance suite runs all scenarios and offers a focused - `policy-advisor` testsuite for manual integration runs. +- CLI conformance (`tests/suites/conformance`) - portable Cargo tests for + lifecycle, mechanistic drafts, file transfer, and the sandbox-local API, + including agent-authored permission requests. Driver E2E runs the complete + Cargo test package. The installed-artifact conformance suite runs the same + tests from a nextest archive and offers a focused `policy-advisor` testsuite + for manual integration runs. - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or `e2e-vm`) - CLI conformance plus the common and driver-specific coverage for the selected deployment. @@ -242,17 +243,25 @@ Run the Docker-backed Rust CLI e2e suite: mise run e2e:docker ``` -Run the minimal portable CLI conformance profile against the gateway selected -in your OpenShell CLI configuration: +Run the portable CLI conformance suite against the gateway selected in your +OpenShell CLI configuration: ```shell -mise run e2e:cli-conformance +cargo build --package openshell-cli +OPENSHELL_BIN="$PWD/target/debug/openshell" \ + cargo test \ + --locked \ + --manifest-path tests/suites/conformance/Cargo.toml \ + --package openshell-test-conformance-cli \ + --no-fail-fast \ + -- \ + --test-threads=1 \ + --nocapture ``` -The gateway must already be installed, reachable, and selected before the task -starts. The task does not provision a gateway or select a compute driver. Set -`OPENSHELL_BIN` to test a prebuilt CLI; otherwise, the task builds the CLI from -the current checkout. +The gateway must already be installed, reachable, and selected before the tests +start. The test suite does not provision a gateway or select a compute driver. +Set `OPENSHELL_BIN` to another executable to test a different prebuilt CLI. The phase-1 scenario verifies the complete CLI-to-gateway-to-driver path without depending on how the gateway was installed or which driver is configured. It @@ -278,9 +287,9 @@ openshell sandbox list --output json openshell sandbox delete ``` -Gateway-backed Rust E2E tasks build the standalone conformance CLI, run its -registered scenarios against the configured gateway, then run any lane-specific -Rust tests that still apply. Run the Podman-backed Rust CLI e2e suite: +Gateway-backed Rust E2E tasks run the CLI conformance Cargo tests against the +configured gateway, then run any lane-specific Rust tests that still apply. +Run the Podman-backed Rust CLI e2e suite: ```shell mise run e2e:podman diff --git a/crates/openshell-conformance-cli/Cargo.toml b/crates/openshell-conformance-cli/Cargo.toml deleted file mode 100644 index ce9f89c17c..0000000000 --- a/crates/openshell-conformance-cli/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "openshell-conformance-cli" -description = "Standalone OpenShell CLI conformance test runner" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true - -[[bin]] -name = "openshell-conformance" -path = "src/main.rs" - -[dependencies] -clap.workspace = true -openshell-conformance = { path = "../openshell-conformance" } -serde.workspace = true -serde_json.workspace = true -tokio.workspace = true - -[lints] -workspace = true diff --git a/crates/openshell-conformance-cli/src/main.rs b/crates/openshell-conformance-cli/src/main.rs deleted file mode 100644 index 81ae17257b..0000000000 --- a/crates/openshell-conformance-cli/src/main.rs +++ /dev/null @@ -1,273 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Standalone runner for `OpenShell` CLI conformance scenarios. - -use std::path::PathBuf; -use std::process::ExitCode; - -use clap::{Parser, Subcommand, ValueEnum}; -use openshell_conformance::{OpenShellRunner, Scenario, scenario, scenarios}; -use serde::Serialize; - -#[derive(Debug, Parser)] -#[command( - name = "openshell-conformance", - about = "Run OpenShell CLI conformance scenarios", - version -)] -struct Cli { - #[command(subcommand)] - command: Command, -} - -#[derive(Debug, Subcommand)] -enum Command { - /// List registered scenarios. - List { - #[arg(long, value_enum, default_value_t = OutputFormat::Text)] - output: OutputFormat, - }, - /// Run all registered scenarios, or named scenarios. - Run { - /// Scenario names. Omit to run every registered scenario. - scenarios: Vec, - /// Explicit path to the `OpenShell` CLI. Defaults to `openshell` on PATH. - #[arg(long)] - openshell_bin: Option, - #[arg(long, value_enum, default_value_t = OutputFormat::Text)] - output: OutputFormat, - }, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -enum OutputFormat { - Text, - Json, -} - -#[derive(Serialize)] -struct ScenarioDescription<'a> { - name: &'a str, - description: &'a str, -} - -#[derive(Serialize)] -struct ScenarioResult<'a> { - name: &'a str, - passed: bool, - diagnostic: Option, -} - -#[derive(Serialize)] -struct RunReport<'a> { - scenarios: Vec>, - passed: bool, -} - -#[tokio::main] -async fn main() -> ExitCode { - match execute(Cli::parse()).await { - Ok(()) => ExitCode::SUCCESS, - Err(error) => { - eprintln!("openshell-conformance: {error}"); - ExitCode::FAILURE - } - } -} - -async fn execute(cli: Cli) -> Result<(), String> { - match cli.command { - Command::List { output } => list(output), - Command::Run { - scenarios: requested, - openshell_bin, - output, - } => run(&requested, openshell_bin, output).await, - } -} - -fn list(output: OutputFormat) -> Result<(), String> { - match output { - OutputFormat::Text => { - for candidate in scenarios() { - println!("{:<24} {}", candidate.name, candidate.description); - } - } - OutputFormat::Json => { - let result = scenarios() - .iter() - .map(|candidate| ScenarioDescription { - name: candidate.name, - description: candidate.description, - }) - .collect::>(); - println!( - "{}", - serde_json::to_string_pretty(&result).map_err(|error| error.to_string())? - ); - } - } - Ok(()) -} - -async fn run( - requested: &[String], - binary: Option, - output: OutputFormat, -) -> Result<(), String> { - let selected = select_scenarios(requested)?; - let mut results = Vec::with_capacity(selected.len()); - for candidate in selected { - results.push(run_scenario(candidate, binary.as_ref()).await); - } - - render_results(results, output) -} - -async fn run_scenario( - candidate: &'static Scenario, - binary: Option<&PathBuf>, -) -> ScenarioResult<'static> { - let runner = binary.map_or_else( - || OpenShellRunner::new(candidate.name), - |path| OpenShellRunner::with_binary(path.clone(), candidate.name), - ); - let mut runner = match runner { - Ok(runner) => runner, - Err(error) => { - return ScenarioResult { - name: candidate.name, - passed: false, - diagnostic: Some(error.to_string()), - }; - } - }; - eprintln!("CLI conformance run ID: {}", runner.id()); - let scenario_result = match runner.check_gateway_status().await { - Ok(()) => candidate.run(&mut runner).await, - Err(error) => Err(error), - }; - let outcome = runner.finish(scenario_result).await; - ScenarioResult { - name: candidate.name, - passed: outcome.is_ok(), - diagnostic: outcome.err(), - } -} - -fn render_results( - results: Vec>, - output: OutputFormat, -) -> Result<(), String> { - let passed = results.iter().all(|result| result.passed); - match output { - OutputFormat::Text => { - for result in &results { - if result.passed { - println!("PASS {}", result.name); - } else { - println!( - "FAIL {}\n{}", - result.name, - result.diagnostic.as_deref().unwrap_or("unknown failure") - ); - } - } - } - OutputFormat::Json => println!( - "{}", - serde_json::to_string_pretty(&RunReport { - scenarios: results, - passed - }) - .map_err(|error| error.to_string())? - ), - } - if passed { - Ok(()) - } else { - Err("one or more scenarios failed".to_string()) - } -} - -fn select_scenarios(requested: &[String]) -> Result, String> { - if requested.is_empty() { - return Ok(scenarios().iter().collect()); - } - requested - .iter() - .map(|name| { - scenario(name).ok_or_else(|| { - format!("unknown scenario '{name}'; run `openshell-conformance list`") - }) - }) - .collect() -} - -#[cfg(test)] -mod tests { - use clap::Parser; - - use super::*; - - #[test] - fn selects_all_scenarios_by_default() { - assert_eq!( - select_scenarios(&[]).expect("select all").len(), - scenarios().len() - ); - } - - #[test] - fn selects_named_scenario() { - let selected = select_scenarios(&["smoke".to_string()]).expect("select smoke"); - assert_eq!(selected[0].name, "smoke"); - } - - #[test] - fn unknown_scenario_has_actionable_diagnostic() { - let error = select_scenarios(&["missing".to_string()]).expect_err("unknown scenario"); - assert!(error.contains("openshell-conformance list")); - } - - #[test] - fn selects_named_policy_scenarios() { - let selected = select_scenarios(&[ - "mechanistic-proposal".to_string(), - "policy-local".to_string(), - ]) - .unwrap(); - assert_eq!( - selected - .iter() - .map(|scenario| scenario.name) - .collect::>(), - ["mechanistic-proposal", "policy-local"] - ); - } - - #[test] - fn parses_binary_override_and_json_output() { - let cli = Cli::try_parse_from([ - "openshell-conformance", - "run", - "smoke", - "--openshell-bin", - "/opt/openshell", - "--output", - "json", - ]) - .expect("parse CLI"); - let Command::Run { - openshell_bin, - output, - .. - } = cli.command - else { - panic!("expected run") - }; - assert_eq!(openshell_bin, Some(PathBuf::from("/opt/openshell"))); - assert_eq!(output, OutputFormat::Json); - } -} diff --git a/e2e/policy-advisor/README.md b/e2e/policy-advisor/README.md index a035a9547e..14b383cb44 100644 --- a/e2e/policy-advisor/README.md +++ b/e2e/policy-advisor/README.md @@ -55,20 +55,16 @@ contents write on the repository. The test auto-resolves the token from ## Conformance coverage -The `mechanistic-proposal` and `new-hostname-proposal` conformance scenarios -check draft generation for a denied IP address and for a hostname absent from -policy. The `policy-local` scenario uses `policy.local` to inspect policy, submit -a narrow permission request, and read the resulting proposal. Run them against a configured gateway -with `--openshell-bin` pointing to the CLI under test: - -```bash -openshell-conformance run mechanistic-proposal new-hostname-proposal policy-local --openshell-bin target/debug/openshell -``` - -Run `openshell-conformance list` to see all scenario names. A manual -`Integration Tests` workflow run can select the `policy-advisor` testsuite to -run only these three scenarios against an installed candidate. Set -`artifact-run-id` to the candidate build's workflow run ID and `test-matrix` to: +The portable CLI conformance suite may include policy-related scenarios. The +current coverage checks draft generation for denied IP addresses and hostnames +absent from policy, plus sandbox-local policy inspection and permission +requests through `policy.local`. See [TESTING.md](../../TESTING.md#rust-cli-e2e) +for the supported source and installed-artifact test paths. + +A manual `Integration Tests` workflow run can select the `policy-advisor` +testsuite to run the policy-related subset against an installed candidate. Set +`artifact-run-id` to the candidate build's workflow run ID and `test-matrix` +to: ```json [{"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"policy-advisor"}] diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index dd2cbcc596..46bbb7e243 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -2,29 +2,39 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Shared helpers for running standalone conformance suites against an already +# Shared helpers for running CLI conformance suites against an already # configured e2e gateway. -e2e_run_openshell_conformance() { - local gateway_label=${1:-OpenShell} - +e2e_require_openshell_bin() { if [ -z "${OPENSHELL_BIN:-}" ]; then echo "ERROR: OPENSHELL_BIN must point to the openshell CLI under test" >&2 return 2 fi - if [ -z "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then - echo "ERROR: OPENSHELL_CONFORMANCE_BIN must point to the openshell-conformance CLI under test" >&2 + if [ ! -x "${OPENSHELL_BIN}" ]; then + echo "ERROR: openshell CLI is not executable: ${OPENSHELL_BIN}" >&2 return 2 fi +} - if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then - echo "ERROR: openshell conformance binary is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2 - return 2 - fi +e2e_run_openshell_conformance() { + local gateway_label=${1:-OpenShell} + local root + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - echo "==> Running standalone CLI conformance against the ${gateway_label} gateway" - "${OPENSHELL_CONFORMANCE_BIN}" run \ - --openshell-bin "${OPENSHELL_BIN}" \ - --output json + e2e_require_openshell_bin || return + + echo "==> Running CLI conformance tests against the ${gateway_label} gateway" + cargo test \ + --locked \ + --manifest-path "${root}/tests/suites/conformance/Cargo.toml" \ + --package openshell-test-conformance-cli \ + --no-fail-fast \ + -- \ + --test-threads=1 \ + --nocapture } + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + e2e_run_openshell_conformance +fi diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 9a9378abee..fc7ac075c8 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -175,19 +175,18 @@ prepared guest image to a particular build or driver configuration. Compose it with `gateway-podman` after either Podman configuration. The role uses the recorded mode to select the corresponding service account. It generates configuration only for development artifacts; RPM installations -retain their packaged service and first-start configuration. For example, run -conformance after the rootless provisioners complete: +retain their packaged service and first-start configuration. For example, +check the development gateway after the rootless provisioners complete: ```shell nix run .#test-guest -- \ --distro fedora --with podman-rootless --with selinux \ --copy ./openshell:/usr/local/bin/openshell \ - --copy ./openshell-conformance:/usr/local/bin/openshell-conformance \ --copy ./openshell-gateway:/usr/local/bin/openshell-gateway \ --copy ./openshell-sandbox.tar:/usr/local/lib/openshell-sandbox.tar \ --provision openshell-development \ --provision gateway-podman \ - -- /usr/local/bin/openshell-conformance run smoke + -- /usr/local/bin/openshell status ``` `openshell-rpm` expects OpenShell to have been installed with `--install`. It diff --git a/tasks/test.toml b/tasks/test.toml index 8a90f3c084..28be2d71f3 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -117,26 +117,12 @@ hide = true ["e2e:rust"] description = "Run Rust CLI e2e tests against a Docker-backed gateway" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" +run = "e2e/rust/e2e-docker.sh" ["e2e:workload:build"] description = "Build the Docker E2E workload fixture" run = "CONTAINER_ENGINE=docker bash tasks/scripts/e2e-build-workload.sh" -["e2e:conformance:build"] -description = "Build the standalone CLI conformance binary" -run = "if [ -z \"${OPENSHELL_CONFORMANCE_BIN:-}\" ]; then cargo build -p openshell-conformance-cli; fi" -hide = true - -["e2e:cli-conformance"] -description = "Build and run the standalone CLI conformance suite against the configured gateway" -depends = ["e2e:conformance:build"] -run = [ - "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli; fi", - "\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" run --openshell-bin \"${OPENSHELL_BIN:-$PWD/target/debug/openshell}\"", -] - ["e2e:websocket-conformance"] description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway" run = [ @@ -163,13 +149,11 @@ run = [ ["e2e:podman"] description = "Run Rust CLI e2e tests against a Podman-backed gateway" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh" +run = "e2e/rust/e2e-podman.sh" ["e2e:podman:ci"] description = "Run the Podman Rust e2e targets enabled in required branch CI" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_E2E_PODMAN_TEST_SET=ci OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh" +run = "OPENSHELL_E2E_PODMAN_TEST_SET=ci e2e/rust/e2e-podman.sh" ["e2e:oidc-pkce"] description = "Run Linux browser PKCE and RBAC e2e tests against Keycloak and a Podman gateway" @@ -194,67 +178,56 @@ run = [ ["e2e:podman:gpu"] description = "Run GPU e2e against a standalone gateway with the Podman compute driver" env = { OPENSHELL_E2E_PODMAN_GPU = "1", OPENSHELL_E2E_PODMAN_TEST = "gpu", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-podman-gpu" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh" +run = "e2e/rust/e2e-podman.sh" ["e2e:kubernetes"] description = "Run Rust CLI e2e tests against an OpenShell gateway deployed on Kubernetes via Helm (set OPENSHELL_E2E_KUBE_CONTEXT to reuse a cluster; otherwise creates a local k3d cluster when k3d is installed; set OPENSHELL_E2E_KUBE_TEST= to scope to one test)" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:v1alpha1"] description = "Run Kubernetes e2e against Agent Sandbox v1alpha1" env = { AGENT_SANDBOX_VERSION = "v0.4.6" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:agent-sandbox-versions"] description = "Run Kubernetes e2e against Agent Sandbox v1beta1 and v1alpha1" -depends = ["e2e:conformance:build"] run = [ - "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh", - "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", + "e2e/rust/e2e-kubernetes.sh", + "AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", ] ["e2e:kubernetes:isolation"] description = "Run Kubernetes e2e with the workload network fence and separate supervisor" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:ha-rebalancing"] description = "Run the Kubernetes HA rebalancing suite through Envoy against two gateway replicas and external PostgreSQL" env = { OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET = "openshell-ha-pg", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-high-availability.yaml", OPENSHELL_E2E_KUBE_TEST = "kubernetes_ha_rebalancing", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha", OPENSHELL_E2E_KUBE_USE_ENVOY = "1" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:credential-drivers"] description = "Run Kubernetes e2e for provider credential storage backed by Kubernetes Secrets and Vault" env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-managed"] description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_CREDENTIAL_DRIVER = "kubernetes-secrets", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-operator"] description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:vm"] description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh" +run = "e2e/rust/e2e-vm.sh" ["e2e:gateway:no-compute-drivers"] description = "Build and launch-check openshell-gateway without compiled compute drivers" @@ -263,31 +236,26 @@ run = "bash e2e/no-compute-driver-gateway.sh" ["e2e:docker:external-driver"] description = "Run Docker conformance with a driver-free gateway and external Docker driver binary" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_DOCKER_FEATURES = "" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" +run = "e2e/rust/e2e-docker.sh" ["e2e:podman:external-driver"] description = "Run Podman conformance with a driver-free gateway and external Podman driver binary" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_FEATURES = "" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh" +run = "e2e/rust/e2e-podman.sh" ["e2e:vm:external-driver"] description = "Run VM E2E with a driver-free gateway and external VM driver binary" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh" +run = "e2e/rust/e2e-vm.sh" ["e2e:kubernetes:external-driver"] description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +run = "e2e/rust/e2e-kubernetes.sh" ["e2e:docker"] description = "Run Docker conformance and Rust e2e tests against a standalone gateway" -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" +run = "e2e/rust/e2e-docker.sh" ["e2e:mechanistic-existing-endpoint"] description = "Run #2821 existing inspected-endpoint auto-approval regression" @@ -299,8 +267,7 @@ run = [ ["e2e:docker:gpu"] description = "Run GPU e2e against a standalone gateway with the Docker compute driver" env = { OPENSHELL_E2E_DOCKER_GPU = "1", OPENSHELL_E2E_DOCKER_TEST = "gpu", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-docker-gpu" } -depends = ["e2e:conformance:build"] -run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" +run = "e2e/rust/e2e-docker.sh" ["test:gateway-config"] description = "Test generated local gateway TOML without starting a runtime" diff --git a/tests/artifacts.nix b/tests/artifacts.nix index c86d8f6d30..a077a1ca2a 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -79,7 +79,7 @@ let ''; }; - conformanceCliArchive = mkTestArchive { + conformanceTestArchive = mkTestArchive { name = "openshell-conformance"; workspacePath = "tests/suites/conformance"; manifestPath = "tests/suites/conformance/Cargo.toml"; @@ -196,7 +196,7 @@ let in rec { inherit - conformanceCliArchive + conformanceTestArchive providerRefreshKeycloakArchive podmanDriverArchive podmanE2eArchive @@ -244,7 +244,7 @@ rec { testArchives = pkgs.writeShellApplication { name = "build-artifacts-test-archives"; runtimeInputs = [ - conformanceCliArchive + conformanceTestArchive providerRefreshKeycloakArchive podmanDriverArchive podmanE2eArchive