diff --git a/cyberai/bench/agent_engine.py b/cyberai/bench/agent_engine.py index 62d6a1f..30ff8b4 100644 --- a/cyberai/bench/agent_engine.py +++ b/cyberai/bench/agent_engine.py @@ -30,7 +30,7 @@ from __future__ import annotations import logging -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Callable, Optional from cyberai.agents.exploit.agent import ExploitAgent @@ -73,8 +73,14 @@ def agent_attack(base_url: str, config: Optional[CyberAIConfig] = None) -> Attac out of the same knowledge base the recon agent wrote it to. That seam is the thing under measurement: a runner that called the modules directly would still pass while the agents were wired to different keys. + + The config comes from the environment so every other flag reaches the + bench: building it from defaults instead silently pinned each new + capability to off here, and a run would report the pipeline missing what + it was never allowed to try. The two web flags are forced on regardless, + because the web path is the whole measurement. """ - cfg = config or CyberAIConfig(use_web_recon=True, use_web_exploit=True) + cfg = config or replace(CyberAIConfig.from_env(), use_web_recon=True, use_web_exploit=True) session = ScanSession(target=base_url) ReconAgent(cfg, session)._run_web_recon(base_url) diff --git a/tests/unit/test_bench_agent_engine.py b/tests/unit/test_bench_agent_engine.py index 18ad553..8e6402f 100644 --- a/tests/unit/test_bench_agent_engine.py +++ b/tests/unit/test_bench_agent_engine.py @@ -176,3 +176,57 @@ def test_agent_attack_drives_both_agents_through_one_session(live_sqli_app): assert outcome.endpoints_tested >= 1 assert all(f["vuln_class"] == "sqli" for f in outcome.findings) assert all(f["proof"] for f in outcome.findings), "a finding without a proof is a guess" + + +def test_agent_attack_reads_flags_from_the_environment(monkeypatch): + """Env flags must reach the bench: a config built from defaults pinned + every new capability to off, and the run reported the pipeline missing + what it was never allowed to try.""" + monkeypatch.setenv("CYBERAI_USE_API_DISCOVERY", "1") + seen = {} + + class _Recon: + def __init__(self, cfg, session): + seen["cfg"] = cfg + + def _run_web_recon(self, base_url): + return {} + + class _Exploit: + def __init__(self, cfg, session): + pass + + def _run_web_exploit(self, base_url): + return {} + + monkeypatch.setattr("cyberai.bench.agent_engine.ReconAgent", _Recon) + monkeypatch.setattr("cyberai.bench.agent_engine.ExploitAgent", _Exploit) + + agent_attack("http://t") + assert seen["cfg"].use_api_discovery is True + + +def test_agent_attack_forces_the_web_path_on(monkeypatch): + monkeypatch.delenv("CYBERAI_USE_WEB_RECON", raising=False) + monkeypatch.delenv("CYBERAI_USE_WEB_EXPLOIT", raising=False) + seen = {} + + class _Recon: + def __init__(self, cfg, session): + seen["cfg"] = cfg + + def _run_web_recon(self, base_url): + return {} + + class _Exploit: + def __init__(self, cfg, session): + pass + + def _run_web_exploit(self, base_url): + return {} + + monkeypatch.setattr("cyberai.bench.agent_engine.ReconAgent", _Recon) + monkeypatch.setattr("cyberai.bench.agent_engine.ExploitAgent", _Exploit) + + agent_attack("http://t") + assert seen["cfg"].use_web_recon is True and seen["cfg"].use_web_exploit is True