From d864da840c1db52c30b121a09a5a6cfa221e7777 Mon Sep 17 00:00:00 2001 From: XuweiDing Date: Mon, 27 Jul 2026 16:30:29 +0800 Subject: [PATCH] Guard ignored runtime configs with schema versions --- browseruse_bench/cli/__init__.py | 7 +++ browseruse_bench/cli/eval.py | 35 ++++++++----- browseruse_bench/cli/run.py | 11 +++- browseruse_bench/cli/run_eval.py | 14 ++++- browseruse_bench/utils/config_loader.py | 31 +++++++++++ config.example.yaml | 4 ++ docs/en/quickstart.mdx | 9 +++- docs/zh/quickstart.mdx | 6 +++ docs_4_codeagent/imports-runtime-config.md | 3 ++ tests/browseruse_bench/test_cli.py | 28 +++++++++- tests/browseruse_bench/test_config_loader.py | 28 ++++++++++ tests/browseruse_bench/test_eval_cli.py | 43 ++++++++++++++++ tests/browseruse_bench/test_run_eval.py | 20 +++++++- tests/browseruse_bench/test_run_routing.py | 54 +++++++++++++++++++- 14 files changed, 273 insertions(+), 20 deletions(-) diff --git a/browseruse_bench/cli/__init__.py b/browseruse_bench/cli/__init__.py index 05344aa..d479add 100644 --- a/browseruse_bench/cli/__init__.py +++ b/browseruse_bench/cli/__init__.py @@ -22,8 +22,10 @@ load_env_file, setup_logger, ) +from browseruse_bench.utils.config_loader import validate_runtime_config_schema CONFIG_PATH = REPO_ROOT / "config.yaml" +_SCHEMA_GATED_COMMANDS = {"eval", "run"} # Preload .env from root directory for unified configuration reading load_env_file(REPO_ROOT / ".env") @@ -131,6 +133,11 @@ def main(argv: Optional[List[str]] = None) -> int: return run_and_eval(cli_args[1:]) config = load_config_file(CONFIG_PATH) + help_requested = any(arg in {"-h", "--help"} for arg in cli_args[1:]) + if cli_args and cli_args[0] in _SCHEMA_GATED_COMMANDS and not help_requested: + validate_runtime_config_schema(config, CONFIG_PATH) + elif help_requested and not isinstance(config, dict): + config = {} parser = _build_parser(config) args, extra = parser.parse_known_args(argv) if extra: diff --git a/browseruse_bench/cli/eval.py b/browseruse_bench/cli/eval.py index 77e1283..650f51d 100644 --- a/browseruse_bench/cli/eval.py +++ b/browseruse_bench/cli/eval.py @@ -3,6 +3,7 @@ import argparse import json import logging +import sys from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -10,7 +11,6 @@ from browseruse_bench.eval.base import EvaluatorArgs from browseruse_bench.eval.model import TaskIdLogFilter from browseruse_bench.eval.registry import get_evaluator_class -from browseruse_bench.utils.stats import calculate_failure_category_stats from browseruse_bench.utils import ( REPO_ROOT, DataSource, @@ -33,6 +33,8 @@ resolve_split, setup_logger, ) +from browseruse_bench.utils.config_loader import validate_runtime_config_schema +from browseruse_bench.utils.stats import calculate_failure_category_stats CONFIG_PATH = REPO_ROOT / "config.yaml" load_env_file(REPO_ROOT / ".env") @@ -524,6 +526,19 @@ def configure_eval_parser(parser: argparse.ArgumentParser, config: dict[str, Any def eval_command(args: argparse.Namespace, config: dict[str, Any]) -> int: """Entry point for the eval subcommand.""" + validate_runtime_config_schema(config, "root config.yaml") + if args.agent_config is not None: + cfg_path = args.agent_config + if not cfg_path.is_absolute(): + cfg_path = Path.cwd() / cfg_path + if not cfg_path.exists(): + raise SystemExit(f"[FAILED] --agent-config file not found: {cfg_path}") + external_cfg = load_config_file(cfg_path) + validate_runtime_config_schema(external_cfg, cfg_path) + external_eval = external_cfg.get("eval", {}) + if external_eval: + config = {**config, "eval": {**config.get("eval", {}), **external_eval}} + extra_args = getattr(args, "extra_args", []) agent_name = normalize_agent_name(args.agent, config) benchmark_name = normalize_benchmark_name(args.data) @@ -532,23 +547,19 @@ def eval_command(args: argparse.Namespace, config: dict[str, Any]) -> int: @handle_cli_errors def main(argv: list[str] | None = None) -> int: + cli_args = list(argv) if argv is not None else sys.argv[1:] config = load_config_file(CONFIG_PATH) + help_requested = any(arg in {"-h", "--help"} for arg in cli_args) + if help_requested and not isinstance(config, dict): + config = {} + elif not help_requested: + validate_runtime_config_schema(config, CONFIG_PATH) parser = argparse.ArgumentParser(prog="bubench eval") configure_eval_parser(parser, config) - args, extra = parser.parse_known_args(argv) + args, extra = parser.parse_known_args(cli_args) if extra: logger.info("Forwarding extra arguments: %s", " ".join(extra)) args.extra_args = extra - if args.agent_config is not None: - cfg_path = args.agent_config - if not cfg_path.is_absolute(): - cfg_path = Path.cwd() / cfg_path - if not cfg_path.exists(): - raise SystemExit(f"[FAILED] --agent-config file not found: {cfg_path}") - external_cfg = load_config_file(cfg_path) - external_eval = external_cfg.get("eval", {}) - if external_eval: - config = {**config, "eval": {**config.get("eval", {}), **external_eval}} return eval_command(args, config) diff --git a/browseruse_bench/cli/run.py b/browseruse_bench/cli/run.py index 20722af..aa4e157 100644 --- a/browseruse_bench/cli/run.py +++ b/browseruse_bench/cli/run.py @@ -61,6 +61,7 @@ resolve_timeout_value, setup_logger, ) +from browseruse_bench.utils.config_loader import validate_runtime_config_schema from browseruse_bench.utils.run_identity import ( INCLUDE_RAW_MACHINE_IDENTIFIERS_ENV_KEY, MACHINE_ID_ENV_KEY, @@ -1343,6 +1344,7 @@ def run_command(args: argparse.Namespace, config: dict[str, Any]) -> int: """Entry point for the run subcommand.""" add_script_log_handler(logger, REPO_ROOT / "output" / "logs", "run") logger.info("Starting run command") + validate_runtime_config_schema(config, "root config.yaml") args.agent = normalize_agent_name(args.agent, config) source_cfg = config source_label = "root config.yaml" @@ -1354,6 +1356,7 @@ def run_command(args: argparse.Namespace, config: dict[str, Any]) -> int: raise SystemExit(f"[FAILED] --agent-config file not found: {cfg_path}") source_cfg = load_config_file(cfg_path) source_label = str(cfg_path) + validate_runtime_config_schema(source_cfg, source_label) args.browser_id = _canonicalize_cli_browser_id(args.browser_id, source_cfg) @@ -1378,10 +1381,16 @@ def run_command(args: argparse.Namespace, config: dict[str, Any]) -> int: @handle_cli_errors def main(argv: list[str] | None = None) -> int: + cli_args = list(argv) if argv is not None else sys.argv[1:] config = load_config_file(CONFIG_PATH) + help_requested = any(arg in {"-h", "--help"} for arg in cli_args) + if help_requested and not isinstance(config, dict): + config = {} + elif not help_requested: + validate_runtime_config_schema(config, CONFIG_PATH) parser = argparse.ArgumentParser(prog="bubench run") configure_run_parser(parser, config) - args, extra = parser.parse_known_args(argv) + args, extra = parser.parse_known_args(cli_args) if extra: parser.error(f"unrecognized arguments: {' '.join(extra)}") return run_command(args, config) diff --git a/browseruse_bench/cli/run_eval.py b/browseruse_bench/cli/run_eval.py index 1f59e73..556d25f 100644 --- a/browseruse_bench/cli/run_eval.py +++ b/browseruse_bench/cli/run_eval.py @@ -50,6 +50,7 @@ resolve_output_model_id, resolve_split, ) +from browseruse_bench.utils.config_loader import validate_runtime_config_schema logger = logging.getLogger(__name__) @@ -58,7 +59,11 @@ def _shared_parser() -> argparse.ArgumentParser: """Parser for only the flags needed to bridge run -> eval (rest forwarded).""" # allow_abbrev=False: otherwise run's --mode is matched as a prefix of our # --model and consumed here instead of being forwarded to the run stage. - parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + parser = argparse.ArgumentParser( + prog="bubench run-eval", + description="Run a benchmark, then evaluate the produced experiment.", + allow_abbrev=False, + ) parser.add_argument("--agent") parser.add_argument("--data") parser.add_argument("--split", default=None) @@ -152,7 +157,11 @@ def _source_config(root_config: dict, agent_config: str | None) -> dict: cfg_path = Path(agent_config) if not cfg_path.is_absolute(): cfg_path = Path.cwd() / cfg_path - return load_config_file(cfg_path) if cfg_path.exists() else root_config + if not cfg_path.exists(): + return root_config + config = load_config_file(cfg_path) + validate_runtime_config_schema(config, cfg_path) + return config def _run_output_base(agent: str, data: str, split: str | None, model_id: str) -> Path: @@ -276,6 +285,7 @@ def run_and_eval(argv: list[str] | None = None) -> int: known, _ = _shared_parser().parse_known_args(raw_args) root_config = load_config_file(CONFIG_PATH) + validate_runtime_config_schema(root_config, CONFIG_PATH) defaults = root_config.get("default", {}) # Mirror configure_run_parser/eval's default agent so an omitted --agent # with no default.agent resolves to the same path both stages use. diff --git a/browseruse_bench/utils/config_loader.py b/browseruse_bench/utils/config_loader.py index 684bb56..965f94a 100644 --- a/browseruse_bench/utils/config_loader.py +++ b/browseruse_bench/utils/config_loader.py @@ -21,6 +21,8 @@ logger = logging.getLogger(__name__) _EVAL_STRUCTURAL_KEYS = {"api_key", "base_url"} _AGENT_REGISTRY_PATH = REPO_ROOT / "configs" / "agent_registry.yaml" +CONFIG_SCHEMA_VERSION_KEY = "config_schema_version" +CURRENT_CONFIG_SCHEMA_VERSION = 1 def _skyvern_temp_database_string() -> str: @@ -81,6 +83,35 @@ def load_config_file(path: Path) -> dict[str, Any]: return _expand_env_vars(data) +def validate_runtime_config_schema(config: object, source: str | Path) -> None: + """Reject malformed or stale ignored runtime configs before a benchmark launches.""" + if not isinstance(config, dict): + raise SystemExit( + f"[FAILED] Runtime config {source} must contain a top-level YAML mapping; " + f"found {type(config).__name__}." + ) + + version = config.get(CONFIG_SCHEMA_VERSION_KEY) + valid_integer = isinstance(version, int) and not isinstance(version, bool) + if valid_integer and version == CURRENT_CONFIG_SCHEMA_VERSION: + return + + if valid_integer and version > CURRENT_CONFIG_SCHEMA_VERSION: + raise SystemExit( + f"[FAILED] Runtime config {source} requires schema version {version}, but this " + f"checkout supports {CURRENT_CONFIG_SCHEMA_VERSION}. Update the repository first." + ) + + found = "missing" if version is None else repr(version) + raise SystemExit( + f"[FAILED] Runtime config schema is outdated in {source}.\n" + f"Expected `{CONFIG_SCHEMA_VERSION_KEY}: {CURRENT_CONFIG_SCHEMA_VERSION}`; found {found}.\n" + "Runtime config files are normally ignored by Git and are not updated by `git pull`. " + "If this file is missing, create it from the latest config.example.yaml. Otherwise " + "back it up, merge required changes from that template, then update its schema version." + ) + + def load_default_package_config() -> dict[str, Any]: """Load default runtime config bundled with the package.""" if yaml is None: diff --git a/config.example.yaml b/config.example.yaml index efab84b..58feb36 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -4,6 +4,10 @@ # runtime config = agents. + models. + browsers. # Select defaults globally with default.model/default.browser. Override per run # with `--model` and `--browser-id`. +# +# Increment this integer whenever existing ignored config.yaml files require +# manual migration. Model-executing commands validate it before using the config. +config_schema_version: 1 default: agent: browser-use diff --git a/docs/en/quickstart.mdx b/docs/en/quickstart.mdx index b358351..6609889 100644 --- a/docs/en/quickstart.mdx +++ b/docs/en/quickstart.mdx @@ -91,11 +91,18 @@ icon: "rocket" ``` The root `config.yaml` is the canonical runtime config. `$VAR` placeholders are - resolved from `.env` at runtime. Three parts to set up: + resolved from `.env` at runtime. It is ignored by Git, so pulling repository + updates does not replace local credentials or custom settings. Model-executing + commands (`run`, `run-eval`, and `eval`) check its top-level + `config_schema_version` before using it. If the version is stale, back up + `config.yaml`, merge the required changes from the latest `config.example.yaml`, + and then update the version. Three parts to set up: **1. Models** — define shared model profiles under top-level `models` and pick the default in `default.model`: ```yaml + config_schema_version: 1 + default: model: gpt-5.4 diff --git a/docs/zh/quickstart.mdx b/docs/zh/quickstart.mdx index f09ac6f..1e9984d 100644 --- a/docs/zh/quickstart.mdx +++ b/docs/zh/quickstart.mdx @@ -91,11 +91,17 @@ icon: "rocket" ``` 根目录 `config.yaml` 是项目的运行时主配置，`$VAR` 占位符会在运行时从 `.env` 解析。 + 该文件被 Git 忽略，因此拉取仓库更新不会覆盖本地密钥和自定义配置。执行模型的 + `run`、`run-eval` 和 `eval` 命令会在读取配置前校验顶层 + `config_schema_version`；如果版本落后，请先备份 `config.yaml`，再合并最新版 + `config.example.yaml` 中的必需改动，最后更新版本号。 需要配置三部分： **1. 模型** — 在顶层 `models` 定义共享模型配置，并通过 `default.model` 选择默认模型： ```yaml + config_schema_version: 1 + default: model: gpt-5.4 diff --git a/docs_4_codeagent/imports-runtime-config.md b/docs_4_codeagent/imports-runtime-config.md index 1b4ad28..3af4a57 100644 --- a/docs_4_codeagent/imports-runtime-config.md +++ b/docs_4_codeagent/imports-runtime-config.md @@ -44,3 +44,6 @@ export PYTHONPATH=/path/to/browseruse_bench && python script.py - Do not hardcode timeout, URL, API key, model name, or similar runtime values. - Read config from `config.yaml`, environment variables, or passed config objects (for example `AgentConfig`). - Store configured file paths as relative paths and resolve to absolute paths with `REPO_ROOT` when reading. +- When an existing ignored `config.yaml` requires a manual migration, increment both + `CURRENT_CONFIG_SCHEMA_VERSION` and `config.example.yaml`'s `config_schema_version`. + Keep the runtime mismatch error actionable; never overwrite user credentials or custom settings. diff --git a/tests/browseruse_bench/test_cli.py b/tests/browseruse_bench/test_cli.py index 2847c6b..41532cf 100644 --- a/tests/browseruse_bench/test_cli.py +++ b/tests/browseruse_bench/test_cli.py @@ -6,9 +6,9 @@ import pytest +import browseruse_bench.cli as cli_pkg from browseruse_bench.cli.viz import configure_viz_parser -from browseruse_bench.utils import REPO_ROOT -from browseruse_bench.utils import create_eval_parser, create_run_parser +from browseruse_bench.utils import REPO_ROOT, create_eval_parser, create_run_parser class TestCreateRunParser: @@ -26,6 +26,30 @@ def test_parser_has_required_arguments(self): assert hasattr(args, 'count') +@pytest.mark.parametrize("command", ["run", "eval"]) +def test_runtime_commands_reject_non_mapping_config_before_parser( + command: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(cli_pkg, "load_config_file", lambda _: ["not", "a", "mapping"]) + + with pytest.raises(SystemExit, match="top-level YAML mapping"): + cli_pkg.main([command]) + + +@pytest.mark.parametrize("command", ["run", "eval"]) +def test_runtime_command_help_works_with_non_mapping_config( + command: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(cli_pkg, "load_config_file", lambda _: ["not", "a", "mapping"]) + + with pytest.raises(SystemExit) as exc_info: + cli_pkg.main([command, "--help"]) + + assert exc_info.value.code == 0 + + class TestConfigureVizParser: def test_defaults(self): parser = argparse.ArgumentParser() diff --git a/tests/browseruse_bench/test_config_loader.py b/tests/browseruse_bench/test_config_loader.py index 6905999..b819a6a 100644 --- a/tests/browseruse_bench/test_config_loader.py +++ b/tests/browseruse_bench/test_config_loader.py @@ -22,8 +22,10 @@ resolve_split, ) from browseruse_bench.utils.config_loader import ( + CURRENT_CONFIG_SCHEMA_VERSION, load_eval_config, resolve_key_case_insensitive, + validate_runtime_config_schema, ) @@ -60,6 +62,32 @@ def test_load_empty_config_file(self, tmp_path: Path): assert result == {} or result is None +class TestRuntimeConfigSchema: + def test_example_uses_current_schema_version(self) -> None: + config = load_config_file(REPO_ROOT / "config.example.yaml") + + assert config["config_schema_version"] == CURRENT_CONFIG_SCHEMA_VERSION + validate_runtime_config_schema(config, "config.example.yaml") + + @pytest.mark.parametrize("version", [None, 0, "1", True]) + def test_rejects_stale_or_invalid_version(self, version: object) -> None: + config = {} if version is None else {"config_schema_version": version} + + with pytest.raises(SystemExit, match="ignored by Git"): + validate_runtime_config_schema(config, "config.yaml") + + @pytest.mark.parametrize("config", [[], "config", 1, True]) + def test_rejects_non_mapping_yaml(self, config: object) -> None: + with pytest.raises(SystemExit, match="top-level YAML mapping"): + validate_runtime_config_schema(config, "config.yaml") + + def test_rejects_config_newer_than_checkout(self) -> None: + config = {"config_schema_version": CURRENT_CONFIG_SCHEMA_VERSION + 1} + + with pytest.raises(SystemExit, match="Update the repository first"): + validate_runtime_config_schema(config, "config.yaml") + + class TestLoadEvalConfig: """Tests for load_eval_config — returns shared eval settings (structural keys excluded).""" diff --git a/tests/browseruse_bench/test_eval_cli.py b/tests/browseruse_bench/test_eval_cli.py index 790dbe3..7b5b63a 100644 --- a/tests/browseruse_bench/test_eval_cli.py +++ b/tests/browseruse_bench/test_eval_cli.py @@ -94,6 +94,7 @@ def _eval_args(**overrides) -> argparse.Namespace: api_key="k", base_url="", model="judge", score_threshold=None, num_worker=1, dry_run=False, force_reeval=False, data_source="local", force_download=False, eval_strategy=None, + agent_config=None, extra_args=[], ) for key, value in overrides.items(): setattr(ns, key, value) @@ -176,3 +177,45 @@ def test_api_key_default_does_not_shadow_config_eval_key(monkeypatch) -> None: eval_cfg = {"api_key": "sk-config-key"} resolved = args.api_key or eval_cfg.get("api_key") or "sk-stale-env-key" assert resolved == "sk-config-key" + + +def test_eval_command_rejects_stale_root_config_before_evaluation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + eval_cli, + "run_evaluation", + lambda *args: pytest.fail("stale config must fail before evaluation"), + ) + + with pytest.raises(SystemExit, match="ignored by Git"): + eval_cli.eval_command( + _eval_args(agent="browser-use", data="LexBench-Browser"), + {"agents": {"browser-use": {}}}, + ) + + +def test_eval_command_validates_explicit_agent_config( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + config_path = tmp_path / "legacy.yaml" + config_path.write_text("eval: {}\n", encoding="utf-8") + monkeypatch.setattr( + eval_cli, + "run_evaluation", + lambda *args: pytest.fail("stale override must fail before evaluation"), + ) + + with pytest.raises(SystemExit, match=str(config_path)): + eval_cli.eval_command( + _eval_args( + agent="browser-use", + data="LexBench-Browser", + agent_config=config_path, + ), + { + "config_schema_version": 1, + "agents": {"browser-use": {}}, + }, + ) diff --git a/tests/browseruse_bench/test_run_eval.py b/tests/browseruse_bench/test_run_eval.py index 9b97184..7e80412 100644 --- a/tests/browseruse_bench/test_run_eval.py +++ b/tests/browseruse_bench/test_run_eval.py @@ -12,6 +12,7 @@ from browseruse_bench.cli.run_eval import run_and_eval _ROOT_CONFIG = { + "config_schema_version": 1, "default": {"agent": "cursor", "data": "LexBench-Browser", "model": "cursor"}, "models": {"cursor": {"model_id": "gpt-5.2", "api_key": "$CURSOR_API_KEY"}}, "browsers": {"lexmount": {"browser_id": "lexmount"}}, @@ -104,6 +105,19 @@ def test_chains_run_then_eval_with_model_id_and_timestamp(harness: _Harness) -> assert ev[ev.index("--agent") + 1] == "cursor" +def test_help_does_not_require_runtime_config(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + run_eval_mod, + "load_config_file", + lambda _: pytest.fail("--help must exit before loading runtime config"), + ) + + with pytest.raises(SystemExit) as exc_info: + run_and_eval(["--help"]) + + assert exc_info.value.code == 0 + + def test_passthrough_model_flows_to_eval_model_id(harness: _Harness) -> None: run_and_eval(["--agent", "cursor", "--model", "claude-fable-5-thinking-high", "--mode", "single"]) ev = harness.eval_call() @@ -293,7 +307,11 @@ def test_default_agent_falls_back_to_agent_tars_like_run_parser( monkeypatch.setattr(cli_pkg, "main", h.cli_main) monkeypatch.setattr( run_eval_mod, "load_config_file", - lambda _: {"default": {"data": "LexBench-Browser"}, "agents": {}}, + lambda _: { + "config_schema_version": 1, + "default": {"data": "LexBench-Browser"}, + "agents": {}, + }, ) monkeypatch.setattr(run_eval_mod, "_run_output_base", lambda agent, data, split, mid: h.exp_root / mid) monkeypatch.setattr(run_eval_mod, "resolve_output_model_id", lambda *a: "gpt-5.2") diff --git a/tests/browseruse_bench/test_run_routing.py b/tests/browseruse_bench/test_run_routing.py index 44d3385..e2fdcb3 100644 --- a/tests/browseruse_bench/test_run_routing.py +++ b/tests/browseruse_bench/test_run_routing.py @@ -2,17 +2,19 @@ from __future__ import annotations +import argparse import re from pathlib import Path import pytest from browseruse_bench.browsers import login_contexts as lc +from browseruse_bench.cli import run as run_module from browseruse_bench.cli.run import ( _canonicalize_cli_browser_id, - _classify_js_code_for_log, _claim_unique_run_dir, _clarify_agent_stdout_line, + _classify_js_code_for_log, resolve_lexmount_routing_for_task, ) @@ -152,6 +154,56 @@ def test_canonicalize_cli_browser_id_falls_back_to_backend_registry() -> None: assert _canonicalize_cli_browser_id("no-such-backend", {}) == "no-such-backend" +def test_run_command_rejects_stale_root_config_before_launch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(run_module, "add_script_log_handler", lambda *args: None) + monkeypatch.setattr( + run_module, + "run_agent", + lambda *args: pytest.fail("stale config must fail before launching the agent"), + ) + args = argparse.Namespace(agent="browser-use", agent_config=None) + + with pytest.raises(SystemExit, match="ignored by Git"): + run_module.run_command(args, {"agents": {"browser-use": {}}}) + + +def test_run_command_validates_explicit_agent_config( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(run_module, "add_script_log_handler", lambda *args: None) + config_path = tmp_path / "legacy.yaml" + config_path.write_text("agents: {}\n", encoding="utf-8") + args = argparse.Namespace( + agent="browser-use", + agent_config=config_path, + ) + + with pytest.raises(SystemExit, match=str(config_path)): + run_module.run_command( + args, + { + "config_schema_version": 1, + "agents": {"browser-use": {}}, + }, + ) + + +def test_run_command_does_not_allow_agent_config_to_bypass_stale_root( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(run_module, "add_script_log_handler", lambda *args: None) + config_path = tmp_path / "current.yaml" + config_path.write_text("config_schema_version: 1\nagents: {}\n", encoding="utf-8") + args = argparse.Namespace(agent="browser-use", agent_config=config_path) + + with pytest.raises(SystemExit, match="root config.yaml"): + run_module.run_command(args, {"agents": {"browser-use": {}}}) + + def test_claim_unique_run_dir_avoids_collision(tmp_path: Path) -> None: # Concurrent runs with identical params must get distinct dirs, never one # shared (which would interleave their output).