Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"version": "3.3"
}
},
"postCreateCommand": "uv sync --all-groups && pre-commit install && cd docs && printf 'source \"https://rubygems.org\"\\ngem \"github-pages\", \"~> 232\", group: :jekyll_plugins\\ngem \"webrick\"\\n' > Gemfile && bundle install",
"postCreateCommand": "uv sync --all-groups --all-extras && pre-commit install && cd docs && printf 'source \"https://rubygems.org\"\\ngem \"github-pages\", \"~> 232\", group: :jekyll_plugins\\ngem \"webrick\"\\n' > Gemfile && bundle install",
"customizations": {
"vscode": {
"extensions": [
Expand Down
2 changes: 1 addition & 1 deletion .env.sample
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ BC_SERVER_PASSWORD=your_container_password_here
BCAL_LLM_COMMAND=python -m bcbench.agent.bcal.bc_eval_capi_bridge
# BCAL_LLM_MODEL=gpt-5 # optional logical model name passed to bc-eval

# AI red teaming ('bcbench redteam scan') -- needs `uv sync --group redteam`, an Azure AI Foundry Hub project, plus `az login`.
# AI red teaming ('bcbench redteam scan') -- needs `uv sync --extra redteam`, an Azure AI Foundry Hub project, plus `az login`.
# AZURE_SUBSCRIPTION_ID=<subscription-guid>
# AZURE_RESOURCE_GROUP=<resource-group>
# AZURE_PROJECT_NAME=<foundry-hub-project>
Expand Down
6 changes: 3 additions & 3 deletions .github/actions/setup-python-uv/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ name: Setup Python with UV
description: Install UV, setup Python, and install bcbench package

inputs:
all-groups:
description: "Install all dependency groups (analysis, redteam) on top of the default dev group"
full-install:
description: "Install all dependency groups (analysis) and extras (redteam) on top of the default dev group"
required: false
default: "false"

Expand All @@ -21,5 +21,5 @@ runs:
shell: pwsh

- name: Install bcbench package
run: uv sync ${{ inputs.all-groups == 'true' && '--all-groups' || '' }}
run: uv sync ${{ inputs.full-install == 'true' && '--all-groups --all-extras' || '' }}
shell: pwsh
4 changes: 2 additions & 2 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
- name: Setup Python with UV
uses: $/.github/actions/setup-python-uv
with:
all-groups: true
full-install: true

- name: Run pre-commit
run: uv run pre-commit run --all-files --show-diff-on-failure --color=always
Expand Down Expand Up @@ -56,7 +56,7 @@ jobs:
- name: Setup Python with UV
uses: $/.github/actions/setup-python-uv
with:
all-groups: true
full-install: true

- name: Run tests with coverage
run: uv run pytest --cov --cov-report=term-missing --cov-report=xml
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/copilot-setup-steps.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- name: Setup Python with UV
uses: ./.github/actions/setup-python-uv
with:
all-groups: true
full-install: true

- name: Install pre-commit hooks
run: uv run pre-commit install
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ cd BC-Bench
uv python install

# Install dependencies
uv sync --all-groups
uv sync --all-groups --all-extras

# Install pre-commit hooks
uv run pre-commit install
Expand Down
17 changes: 11 additions & 6 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@ dependencies = [
"numpy>=2.3.5",
"scipy>=1.16.3",
"jinja2>=3.1.6",
"packaging>=25.0",
"rich>=14.2.0",
]

[project.optional-dependencies]
Comment thread
haoranpb marked this conversation as resolved.
# `bcbench redteam` only, and not installed by default: azure-ai-evaluation[redteam] pulls in ~76
# packages (pyrit, transformers, datasets, ...) that no other command needs.
redteam = [
"azure-ai-evaluation[redteam]>=1.18.2",
"azure-identity>=1.25.3",
]

[project.scripts]
Expand Down Expand Up @@ -97,10 +107,5 @@ dev = [
"ruff>=0.16.0",
"pre-commit>=4.3.0",
"ty>=0.0.84",
]
# `bcbench redteam` only, and not in default-groups: azure-ai-evaluation[redteam] pulls in ~76
# packages (pyrit, transformers, datasets, ...) that no other command needs.
redteam = [
"azure-ai-evaluation[redteam]>=1.18.2",
"azure-identity>=1.25.3",
"click>=8.3.0",
]
14 changes: 7 additions & 7 deletions src/bcbench/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
app.add_typer(contamination_app, name="contamination")


def _redteam_group_installed() -> bool:
def _redteam_extra_installed() -> bool:
try:
import_module("azure.ai.evaluation.red_team")
except ImportError:
Expand All @@ -49,16 +49,16 @@ def _redteam_group_installed() -> bool:


def _add_redteam_app() -> None:
"""Register `bcbench redteam`, whose azure-ai-evaluation[redteam] tree ships as an optional dependency group.
"""Register `bcbench redteam`, whose azure-ai-evaluation[redteam] tree ships as an optional extra.

Importing `bcbench.commands.redteam` pulls that tree in, so probe for it first and otherwise
register a catch-all that names the missing group instead of an opaque "no such command".
register a catch-all that names the missing extra instead of an opaque "no such command".
"""
if not _redteam_group_installed():
if not _redteam_extra_installed():

@app.command("redteam", context_settings={"ignore_unknown_options": True}, help="Red team BC-Bench agents (needs `uv sync --group redteam`)")
def _missing_group(args: Annotated[list[str] | None, typer.Argument(hidden=True)] = None) -> None:
raise typer.BadParameter("`bcbench redteam` needs the optional redteam dependency group. Install it with `uv sync --group redteam`.")
@app.command("redteam", context_settings={"ignore_unknown_options": True}, help="Red team BC-Bench agents (needs `uv sync --extra redteam`)")
def _missing_extra(args: Annotated[list[str] | None, typer.Argument(hidden=True)] = None) -> None:
raise typer.BadParameter("`bcbench redteam` needs the optional redteam extra. Install it with `uv sync --extra redteam`.")

return

Expand Down
2 changes: 1 addition & 1 deletion src/bcbench/commands/redteam.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def scan(
"""
Run an AI red teaming Agent scan against a BC-Bench agent.

Requires the optional redteam dependency group (`uv sync --group redteam`) and a Foundry Hub project via the AZURE_SUBSCRIPTION_ID / AZURE_RESOURCE_GROUP / AZURE_PROJECT_NAME env vars (plus Azure credentials, e.g. `az login`).
Requires the optional redteam extra (`uv sync --extra redteam`) and a Foundry Hub project via the AZURE_SUBSCRIPTION_ID / AZURE_RESOURCE_GROUP / AZURE_PROJECT_NAME env vars (plus Azure credentials, e.g. `az login`).
The bcal symbol cache is auto-populated from the BC artifacts cache (run scripts/Download-BCSymbols.ps1 first).

Examples:
Expand Down
10 changes: 5 additions & 5 deletions tests/test_cli_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from typer.main import get_command
from typer.testing import CliRunner

from bcbench.cli import _redteam_group_installed, app
from bcbench.cli import _redteam_extra_installed, app
from bcbench.cli_options import resolve_agent_runtime, resolve_evaluation_runtime
from bcbench.commands import evaluate as evaluate_commands
from bcbench.commands import run as run_commands
Expand All @@ -29,14 +29,14 @@


@patch("bcbench.cli.import_module")
def test_redteam_group_installed_when_sdk_module_imports(import_module):
assert _redteam_group_installed()
def test_redteam_extra_installed_when_sdk_module_imports(import_module):
assert _redteam_extra_installed()
import_module.assert_called_once_with("azure.ai.evaluation.red_team")


@patch("bcbench.cli.import_module", side_effect=ImportError)
def test_redteam_group_not_installed_when_sdk_module_import_fails(import_module):
assert not _redteam_group_installed()
def test_redteam_extra_not_installed_when_sdk_module_import_fails(import_module):
assert not _redteam_extra_installed()
import_module.assert_called_once_with("azure.ai.evaluation.red_team")


Expand Down
2 changes: 1 addition & 1 deletion tests/test_redteam.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from rich.console import Console
from typer.main import get_command

# Red teaming ships as the optional `redteam` dependency group, so skip when it is not installed.
# Red teaming ships as the optional `redteam` extra, so skip when it is not installed.
pytest.importorskip("azure.ai.evaluation.red_team")

from bcbench import redteam
Expand Down
23 changes: 15 additions & 8 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading