From 6c30b38b4a9d050287479bcd6815c8147a39b835 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Wed, 7 Oct 2026 14:55:16 +0200 Subject: [PATCH 1/4] Declare direct dependencies flagged by ty missing-direct-dependency Add rich and packaging to project dependencies and move the redteam dependency group to an optional extra, since package code may only import project dependencies or extras. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 70c54f79-d7a0-4cde-baf2-e53dfff3deb4 --- .env.sample | 2 +- .github/actions/setup-python-uv/action.yml | 4 ++-- CONTRIBUTING.md | 2 +- pyproject.toml | 16 ++++++++++------ src/bcbench/cli.py | 14 +++++++------- src/bcbench/commands/redteam.py | 2 +- tests/test_cli_commands.py | 10 +++++----- tests/test_redteam.py | 2 +- uv.lock | 21 +++++++++++++-------- 9 files changed, 41 insertions(+), 32 deletions(-) diff --git a/.env.sample b/.env.sample index 03f481c4d..bf30f78c5 100644 --- a/.env.sample +++ b/.env.sample @@ -20,7 +20,7 @@ BC_SERVER_PASSWORD=your_container_password_here BCAL_LLM_COMMAND=python -m bcbench.agent.bcal.bc_eval_capi_bridge # BCAL_LLM_MODEL=gpt-5 # optional logical model name passed to bc-eval -# AI red teaming ('bcbench redteam scan') -- needs `uv sync --group redteam`, an Azure AI Foundry Hub project, plus `az login`. +# AI red teaming ('bcbench redteam scan') -- needs `uv sync --extra redteam`, an Azure AI Foundry Hub project, plus `az login`. # AZURE_SUBSCRIPTION_ID= # AZURE_RESOURCE_GROUP= # AZURE_PROJECT_NAME= diff --git a/.github/actions/setup-python-uv/action.yml b/.github/actions/setup-python-uv/action.yml index 6da02ef98..9e2b0cc2c 100644 --- a/.github/actions/setup-python-uv/action.yml +++ b/.github/actions/setup-python-uv/action.yml @@ -3,7 +3,7 @@ description: Install UV, setup Python, and install bcbench package inputs: all-groups: - description: "Install all dependency groups (analysis, redteam) on top of the default dev group" + description: "Install all dependency groups (analysis) and extras (redteam) on top of the default dev group" required: false default: "false" @@ -21,5 +21,5 @@ runs: shell: pwsh - name: Install bcbench package - run: uv sync ${{ inputs.all-groups == 'true' && '--all-groups' || '' }} + run: uv sync ${{ inputs.all-groups == 'true' && '--all-groups --all-extras' || '' }} shell: pwsh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44bdfc792..673c63d73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ cd BC-Bench uv python install # Install dependencies -uv sync --all-groups +uv sync --all-groups --all-extras # Install pre-commit hooks uv run pre-commit install diff --git a/pyproject.toml b/pyproject.toml index 69c6c207e..c5fa45764 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,16 @@ dependencies = [ "numpy>=2.3.5", "scipy>=1.16.3", "jinja2>=3.1.6", + "packaging>=25.0", + "rich>=14.2.0", +] + +[project.optional-dependencies] +# `bcbench redteam` only, and not installed by default: azure-ai-evaluation[redteam] pulls in ~76 +# packages (pyrit, transformers, datasets, ...) that no other command needs. +redteam = [ + "azure-ai-evaluation[redteam]>=1.18.2", + "azure-identity>=1.25.3", ] [project.scripts] @@ -98,9 +108,3 @@ dev = [ "pre-commit>=4.3.0", "ty>=0.0.84", ] -# `bcbench redteam` only, and not in default-groups: azure-ai-evaluation[redteam] pulls in ~76 -# packages (pyrit, transformers, datasets, ...) that no other command needs. -redteam = [ - "azure-ai-evaluation[redteam]>=1.18.2", - "azure-identity>=1.25.3", -] diff --git a/src/bcbench/cli.py b/src/bcbench/cli.py index d6e196858..18468878d 100644 --- a/src/bcbench/cli.py +++ b/src/bcbench/cli.py @@ -40,7 +40,7 @@ app.add_typer(contamination_app, name="contamination") -def _redteam_group_installed() -> bool: +def _redteam_extra_installed() -> bool: try: import_module("azure.ai.evaluation.red_team") except ImportError: @@ -49,16 +49,16 @@ def _redteam_group_installed() -> bool: def _add_redteam_app() -> None: - """Register `bcbench redteam`, whose azure-ai-evaluation[redteam] tree ships as an optional dependency group. + """Register `bcbench redteam`, whose azure-ai-evaluation[redteam] tree ships as an optional extra. Importing `bcbench.commands.redteam` pulls that tree in, so probe for it first and otherwise - register a catch-all that names the missing group instead of an opaque "no such command". + register a catch-all that names the missing extra instead of an opaque "no such command". """ - if not _redteam_group_installed(): + if not _redteam_extra_installed(): - @app.command("redteam", context_settings={"ignore_unknown_options": True}, help="Red team BC-Bench agents (needs `uv sync --group redteam`)") - def _missing_group(args: Annotated[list[str] | None, typer.Argument(hidden=True)] = None) -> None: - raise typer.BadParameter("`bcbench redteam` needs the optional redteam dependency group. Install it with `uv sync --group redteam`.") + @app.command("redteam", context_settings={"ignore_unknown_options": True}, help="Red team BC-Bench agents (needs `uv sync --extra redteam`)") + def _missing_extra(args: Annotated[list[str] | None, typer.Argument(hidden=True)] = None) -> None: + raise typer.BadParameter("`bcbench redteam` needs the optional redteam extra. Install it with `uv sync --extra redteam`.") return diff --git a/src/bcbench/commands/redteam.py b/src/bcbench/commands/redteam.py index c97d1dd7f..fb5d1a7e2 100644 --- a/src/bcbench/commands/redteam.py +++ b/src/bcbench/commands/redteam.py @@ -58,7 +58,7 @@ def scan( """ Run an AI red teaming Agent scan against a BC-Bench agent. - Requires the optional redteam dependency group (`uv sync --group redteam`) and a Foundry Hub project via the AZURE_SUBSCRIPTION_ID / AZURE_RESOURCE_GROUP / AZURE_PROJECT_NAME env vars (plus Azure credentials, e.g. `az login`). + Requires the optional redteam extra (`uv sync --extra redteam`) and a Foundry Hub project via the AZURE_SUBSCRIPTION_ID / AZURE_RESOURCE_GROUP / AZURE_PROJECT_NAME env vars (plus Azure credentials, e.g. `az login`). The bcal symbol cache is auto-populated from the BC artifacts cache (run scripts/Download-BCSymbols.ps1 first). Examples: diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 21fa5f2d1..b97d2c856 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -11,7 +11,7 @@ from typer.main import get_command from typer.testing import CliRunner -from bcbench.cli import _redteam_group_installed, app +from bcbench.cli import _redteam_extra_installed, app from bcbench.cli_options import resolve_agent_runtime, resolve_evaluation_runtime from bcbench.commands import evaluate as evaluate_commands from bcbench.commands import run as run_commands @@ -29,14 +29,14 @@ @patch("bcbench.cli.import_module") -def test_redteam_group_installed_when_sdk_module_imports(import_module): - assert _redteam_group_installed() +def test_redteam_extra_installed_when_sdk_module_imports(import_module): + assert _redteam_extra_installed() import_module.assert_called_once_with("azure.ai.evaluation.red_team") @patch("bcbench.cli.import_module", side_effect=ImportError) -def test_redteam_group_not_installed_when_sdk_module_import_fails(import_module): - assert not _redteam_group_installed() +def test_redteam_extra_not_installed_when_sdk_module_import_fails(import_module): + assert not _redteam_extra_installed() import_module.assert_called_once_with("azure.ai.evaluation.red_team") diff --git a/tests/test_redteam.py b/tests/test_redteam.py index 601502777..f18dde237 100644 --- a/tests/test_redteam.py +++ b/tests/test_redteam.py @@ -15,7 +15,7 @@ from rich.console import Console from typer.main import get_command -# Red teaming ships as the optional `redteam` dependency group, so skip when it is not installed. +# Red teaming ships as the optional `redteam` extra, so skip when it is not installed. pytest.importorskip("azure.ai.evaluation.red_team") from bcbench import redteam diff --git a/uv.lock b/uv.lock index f6e18dba0..780573d74 100644 --- a/uv.lock +++ b/uv.lock @@ -258,10 +258,12 @@ dependencies = [ { name = "jinja2" }, { name = "jsonschema" }, { name = "numpy" }, + { name = "packaging" }, { name = "pydantic" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "requests" }, + { name = "rich" }, { name = "scipy" }, { name = "textual" }, { name = "typer" }, @@ -269,6 +271,12 @@ dependencies = [ { name = "unidiff" }, ] +[package.optional-dependencies] +redteam = [ + { name = "azure-ai-evaluation", extra = ["redteam"] }, + { name = "azure-identity" }, +] + [package.dev-dependencies] analysis = [ { name = "ipykernel" }, @@ -283,27 +291,28 @@ dev = [ { name = "ruff" }, { name = "ty" }, ] -redteam = [ - { name = "azure-ai-evaluation", extra = ["redteam"] }, - { name = "azure-identity" }, -] [package.metadata] requires-dist = [ + { name = "azure-ai-evaluation", extras = ["redteam"], marker = "extra == 'redteam'", specifier = ">=1.18.2" }, + { name = "azure-identity", marker = "extra == 'redteam'", specifier = ">=1.25.3" }, { name = "bcbench-core", editable = "packages/bcbench-core" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "jsonschema", specifier = ">=4.0" }, { name = "numpy", specifier = ">=2.3.5" }, + { name = "packaging", specifier = ">=25.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.0" }, + { name = "rich", specifier = ">=14.2.0" }, { name = "scipy", specifier = ">=1.16.3" }, { name = "textual", specifier = ">=7.0" }, { name = "typer", specifier = ">=0.9.0" }, { name = "typing-extensions", specifier = ">=4.0" }, { name = "unidiff", specifier = ">=0.7" }, ] +provides-extras = ["redteam"] [package.metadata.requires-dev] analysis = [ @@ -319,10 +328,6 @@ dev = [ { name = "ruff", specifier = ">=0.16.0" }, { name = "ty", specifier = ">=0.0.84" }, ] -redteam = [ - { name = "azure-ai-evaluation", extras = ["redteam"], specifier = ">=1.18.2" }, - { name = "azure-identity", specifier = ">=1.25.3" }, -] [[package]] name = "bcbench-core" From ae3697c0616111cf95cabb78e2397331fc1dfd69 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Thu, 8 Oct 2026 13:15:41 +0200 Subject: [PATCH 2/4] Rename setup-python-uv input all-groups to full-install Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-python-uv/action.yml | 4 ++-- .github/workflows/CI.yml | 4 ++-- .github/workflows/copilot-setup-steps.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-python-uv/action.yml b/.github/actions/setup-python-uv/action.yml index 9e2b0cc2c..bdea93669 100644 --- a/.github/actions/setup-python-uv/action.yml +++ b/.github/actions/setup-python-uv/action.yml @@ -2,7 +2,7 @@ name: Setup Python with UV description: Install UV, setup Python, and install bcbench package inputs: - all-groups: + full-install: description: "Install all dependency groups (analysis) and extras (redteam) on top of the default dev group" required: false default: "false" @@ -21,5 +21,5 @@ runs: shell: pwsh - name: Install bcbench package - run: uv sync ${{ inputs.all-groups == 'true' && '--all-groups --all-extras' || '' }} + run: uv sync ${{ inputs.full-install == 'true' && '--all-groups --all-extras' || '' }} shell: pwsh diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9ebe2e9c8..39369664a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Python with UV uses: $/.github/actions/setup-python-uv with: - all-groups: true + full-install: true - name: Run pre-commit run: uv run pre-commit run --all-files --show-diff-on-failure --color=always @@ -56,7 +56,7 @@ jobs: - name: Setup Python with UV uses: $/.github/actions/setup-python-uv with: - all-groups: true + full-install: true - name: Run tests with coverage run: uv run pytest --cov --cov-report=term-missing --cov-report=xml diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 5f5c437c7..dbad57e08 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Python with UV uses: ./.github/actions/setup-python-uv with: - all-groups: true + full-install: true - name: Install pre-commit hooks run: uv run pre-commit install From eb525d487dae6f5d5fb4d728bde152b0c7c0aa38 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Thu, 8 Oct 2026 13:26:28 +0200 Subject: [PATCH 3/4] update dev cotainer scripts too --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ae435924c..c7234c5dd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,7 +9,7 @@ "version": "3.3" } }, - "postCreateCommand": "uv sync --all-groups && pre-commit install && cd docs && printf 'source \"https://rubygems.org\"\\ngem \"github-pages\", \"~> 232\", group: :jekyll_plugins\\ngem \"webrick\"\\n' > Gemfile && bundle install", + "postCreateCommand": "uv sync --all-groups --all-extras && pre-commit install && cd docs && printf 'source \"https://rubygems.org\"\\ngem \"github-pages\", \"~> 232\", group: :jekyll_plugins\\ngem \"webrick\"\\n' > Gemfile && bundle install", "customizations": { "vscode": { "extensions": [ From 29e8f9cf665e245d927d1f07b10da8843c4f6183 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Thu, 8 Oct 2026 13:34:56 +0200 Subject: [PATCH 4/4] Declare click as a dev dependency for tests that import it directly Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyproject.toml | 1 + uv.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c5fa45764..b66310961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,4 +107,5 @@ dev = [ "ruff>=0.16.0", "pre-commit>=4.3.0", "ty>=0.0.84", + "click>=8.3.0", ] diff --git a/uv.lock b/uv.lock index 780573d74..d5873a4fa 100644 --- a/uv.lock +++ b/uv.lock @@ -285,6 +285,7 @@ analysis = [ { name = "plotly" }, ] dev = [ + { name = "click" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -322,6 +323,7 @@ analysis = [ { name = "plotly", specifier = ">=6.5.0" }, ] dev = [ + { name = "click", specifier = ">=8.3.0" }, { name = "pre-commit", specifier = ">=4.3.0" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-cov", specifier = ">=7.0" },