Guard ignored runtime configs with schema versions

browseruse_bench/cli/__init__.py

@@ -22,8 +22,10 @@
     load_env_file,
     setup_logger,
 )
+from browseruse_bench.utils.config_loader import validate_runtime_config_schema
 
 CONFIG_PATH = REPO_ROOT / "config.yaml"
+_SCHEMA_GATED_COMMANDS = {"eval", "run"}
 
 # Preload .env from root directory for unified configuration reading
 load_env_file(REPO_ROOT / ".env")
@@ -131,6 +133,11 @@ def main(argv: Optional[List[str]] = None) -> int:
         return run_and_eval(cli_args[1:])
 
     config = load_config_file(CONFIG_PATH)
+    help_requested = any(arg in {"-h", "--help"} for arg in cli_args[1:])
+    if cli_args and cli_args[0] in _SCHEMA_GATED_COMMANDS and not help_requested:
+        validate_runtime_config_schema(config, CONFIG_PATH)
+    elif help_requested and not isinstance(config, dict):
+        config = {}
     parser = _build_parser(config)
     args, extra = parser.parse_known_args(argv)
     if extra:

browseruse_bench/cli/eval.py

@@ -3,14 +3,14 @@
 import argparse
 import json
 import logging
+import sys
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
 from browseruse_bench.eval.base import EvaluatorArgs
 from browseruse_bench.eval.model import TaskIdLogFilter
 from browseruse_bench.eval.registry import get_evaluator_class
-from browseruse_bench.utils.stats import calculate_failure_category_stats
 from browseruse_bench.utils import (
     REPO_ROOT,
     DataSource,
@@ -33,6 +33,8 @@
     resolve_split,
     setup_logger,
 )
+from browseruse_bench.utils.config_loader import validate_runtime_config_schema
+from browseruse_bench.utils.stats import calculate_failure_category_stats
 
 CONFIG_PATH = REPO_ROOT / "config.yaml"
 load_env_file(REPO_ROOT / ".env")
@@ -524,6 +526,19 @@ def configure_eval_parser(parser: argparse.ArgumentParser, config: dict[str, Any
 
 def eval_command(args: argparse.Namespace, config: dict[str, Any]) -> int:
     """Entry point for the eval subcommand."""
+    validate_runtime_config_schema(config, "root config.yaml")
+    if args.agent_config is not None:
+        cfg_path = args.agent_config
+        if not cfg_path.is_absolute():
+            cfg_path = Path.cwd() / cfg_path
+        if not cfg_path.exists():
+            raise SystemExit(f"[FAILED] --agent-config file not found: {cfg_path}")
+        external_cfg = load_config_file(cfg_path)
+        validate_runtime_config_schema(external_cfg, cfg_path)
+        external_eval = external_cfg.get("eval", {})
+        if external_eval:
+            config = {**config, "eval": {**config.get("eval", {}), **external_eval}}
+
     extra_args = getattr(args, "extra_args", [])
     agent_name = normalize_agent_name(args.agent, config)
     benchmark_name = normalize_benchmark_name(args.data)
@@ -532,23 +547,19 @@ def eval_command(args: argparse.Namespace, config: dict[str, Any]) -> int:
 
 @handle_cli_errors
 def main(argv: list[str] | None = None) -> int:
+    cli_args = list(argv) if argv is not None else sys.argv[1:]
     config = load_config_file(CONFIG_PATH)
+    help_requested = any(arg in {"-h", "--help"} for arg in cli_args)
+    if help_requested and not isinstance(config, dict):
+        config = {}
+    elif not help_requested:
+        validate_runtime_config_schema(config, CONFIG_PATH)
     parser = argparse.ArgumentParser(prog="bubench eval")
     configure_eval_parser(parser, config)
-    args, extra = parser.parse_known_args(argv)
+    args, extra = parser.parse_known_args(cli_args)
     if extra:
         logger.info("Forwarding extra arguments: %s", " ".join(extra))
     args.extra_args = extra
-    if args.agent_config is not None:
-        cfg_path = args.agent_config
-        if not cfg_path.is_absolute():
-            cfg_path = Path.cwd() / cfg_path
-        if not cfg_path.exists():
-            raise SystemExit(f"[FAILED] --agent-config file not found: {cfg_path}")
-        external_cfg = load_config_file(cfg_path)
-        external_eval = external_cfg.get("eval", {})
-        if external_eval:
-            config = {**config, "eval": {**config.get("eval", {}), **external_eval}}
     return eval_command(args, config)
 
 

browseruse_bench/cli/run.py

@@ -61,6 +61,7 @@
     resolve_timeout_value,
     setup_logger,
 )
+from browseruse_bench.utils.config_loader import validate_runtime_config_schema
 from browseruse_bench.utils.run_identity import (
     INCLUDE_RAW_MACHINE_IDENTIFIERS_ENV_KEY,
     MACHINE_ID_ENV_KEY,
@@ -1343,6 +1344,7 @@ def run_command(args: argparse.Namespace, config: dict[str, Any]) -> int:
     """Entry point for the run subcommand."""
     add_script_log_handler(logger, REPO_ROOT / "output" / "logs", "run")
     logger.info("Starting run command")
+    validate_runtime_config_schema(config, "root config.yaml")
     args.agent = normalize_agent_name(args.agent, config)
     source_cfg = config
     source_label = "root config.yaml"
@@ -1354,6 +1356,7 @@ def run_command(args: argparse.Namespace, config: dict[str, Any]) -> int:
             raise SystemExit(f"[FAILED] --agent-config file not found: {cfg_path}")
         source_cfg = load_config_file(cfg_path)
         source_label = str(cfg_path)
+        validate_runtime_config_schema(source_cfg, source_label)
 
     args.browser_id = _canonicalize_cli_browser_id(args.browser_id, source_cfg)
 
@@ -1378,10 +1381,16 @@ def run_command(args: argparse.Namespace, config: dict[str, Any]) -> int:
 
 @handle_cli_errors
 def main(argv: list[str] | None = None) -> int:
+    cli_args = list(argv) if argv is not None else sys.argv[1:]
     config = load_config_file(CONFIG_PATH)
+    help_requested = any(arg in {"-h", "--help"} for arg in cli_args)
+    if help_requested and not isinstance(config, dict):
+        config = {}
+    elif not help_requested:
+        validate_runtime_config_schema(config, CONFIG_PATH)
     parser = argparse.ArgumentParser(prog="bubench run")
     configure_run_parser(parser, config)
-    args, extra = parser.parse_known_args(argv)
+    args, extra = parser.parse_known_args(cli_args)
     if extra:
         parser.error(f"unrecognized arguments: {' '.join(extra)}")
     return run_command(args, config)

browseruse_bench/cli/run_eval.py

@@ -50,6 +50,7 @@
     resolve_output_model_id,
     resolve_split,
 )
+from browseruse_bench.utils.config_loader import validate_runtime_config_schema
 
 logger = logging.getLogger(__name__)
 
@@ -58,7 +59,11 @@ def _shared_parser() -> argparse.ArgumentParser:
     """Parser for only the flags needed to bridge run -> eval (rest forwarded)."""
     # allow_abbrev=False: otherwise run's --mode is matched as a prefix of our
     # --model and consumed here instead of being forwarded to the run stage.
-    parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
+    parser = argparse.ArgumentParser(
+        prog="bubench run-eval",
+        description="Run a benchmark, then evaluate the produced experiment.",
+        allow_abbrev=False,
+    )
     parser.add_argument("--agent")
     parser.add_argument("--data")
     parser.add_argument("--split", default=None)
@@ -152,7 +157,11 @@ def _source_config(root_config: dict, agent_config: str | None) -> dict:
     cfg_path = Path(agent_config)
     if not cfg_path.is_absolute():
         cfg_path = Path.cwd() / cfg_path
-    return load_config_file(cfg_path) if cfg_path.exists() else root_config
+    if not cfg_path.exists():
+        return root_config
+    config = load_config_file(cfg_path)
+    validate_runtime_config_schema(config, cfg_path)
+    return config
 
 
 def _run_output_base(agent: str, data: str, split: str | None, model_id: str) -> Path:
@@ -276,6 +285,7 @@ def run_and_eval(argv: list[str] | None = None) -> int:
     known, _ = _shared_parser().parse_known_args(raw_args)
 
     root_config = load_config_file(CONFIG_PATH)
+    validate_runtime_config_schema(root_config, CONFIG_PATH)
     defaults = root_config.get("default", {})
     # Mirror configure_run_parser/eval's default agent so an omitted --agent
     # with no default.agent resolves to the same path both stages use.

browseruse_bench/utils/config_loader.py

@@ -21,6 +21,8 @@
 logger = logging.getLogger(__name__)
 _EVAL_STRUCTURAL_KEYS = {"api_key", "base_url"}
 _AGENT_REGISTRY_PATH = REPO_ROOT / "configs" / "agent_registry.yaml"
+CONFIG_SCHEMA_VERSION_KEY = "config_schema_version"
+CURRENT_CONFIG_SCHEMA_VERSION = 1
 
 
 def _skyvern_temp_database_string() -> str:
@@ -81,6 +83,35 @@ def load_config_file(path: Path) -> dict[str, Any]:
     return _expand_env_vars(data)
 
 
+def validate_runtime_config_schema(config: object, source: str | Path) -> None:
+    """Reject malformed or stale ignored runtime configs before a benchmark launches."""
+    if not isinstance(config, dict):
+        raise SystemExit(
+            f"[FAILED] Runtime config {source} must contain a top-level YAML mapping; "
+            f"found {type(config).__name__}."
+        )
+
+    version = config.get(CONFIG_SCHEMA_VERSION_KEY)
+    valid_integer = isinstance(version, int) and not isinstance(version, bool)
+    if valid_integer and version == CURRENT_CONFIG_SCHEMA_VERSION:
+        return
+
+    if valid_integer and version > CURRENT_CONFIG_SCHEMA_VERSION:
+        raise SystemExit(
+            f"[FAILED] Runtime config {source} requires schema version {version}, but this "
+            f"checkout supports {CURRENT_CONFIG_SCHEMA_VERSION}. Update the repository first."
+        )
+
+    found = "missing" if version is None else repr(version)
+    raise SystemExit(
+        f"[FAILED] Runtime config schema is outdated in {source}.\n"
+        f"Expected `{CONFIG_SCHEMA_VERSION_KEY}: {CURRENT_CONFIG_SCHEMA_VERSION}`; found {found}.\n"
+        "Runtime config files are normally ignored by Git and are not updated by `git pull`. "
+        "If this file is missing, create it from the latest config.example.yaml. Otherwise "
+        "back it up, merge required changes from that template, then update its schema version."
+    )
+
+
 def load_default_package_config() -> dict[str, Any]:
     """Load default runtime config bundled with the package."""
     if yaml is None:

config.example.yaml

@@ -4,6 +4,10 @@
 #   runtime config = agents.<agent> + models.<model> + browsers.<browser>
 # Select defaults globally with default.model/default.browser. Override per run
 # with `--model` and `--browser-id`.
+#
+# Increment this integer whenever existing ignored config.yaml files require
+# manual migration. Model-executing commands validate it before using the config.
+config_schema_version: 1
 
 default:
   agent: browser-use

docs/en/quickstart.mdx

@@ -91,11 +91,18 @@ icon: "rocket"
     ```
 
     The root `config.yaml` is the canonical runtime config. `$VAR` placeholders are
-    resolved from `.env` at runtime. Three parts to set up:
+    resolved from `.env` at runtime. It is ignored by Git, so pulling repository
+    updates does not replace local credentials or custom settings. Model-executing
+    commands (`run`, `run-eval`, and `eval`) check its top-level
+    `config_schema_version` before using it. If the version is stale, back up
+    `config.yaml`, merge the required changes from the latest `config.example.yaml`,
+    and then update the version. Three parts to set up:
 
     **1. Models** — define shared model profiles under top-level `models` and pick the default in `default.model`:
 
     ```yaml
+    config_schema_version: 1
+
     default:
       model: gpt-5.4
 

docs/zh/quickstart.mdx

@@ -91,11 +91,17 @@ icon: "rocket"
     ```
 
     根目录 `config.yaml` 是项目的运行时主配置，`$VAR` 占位符会在运行时从 `.env` 解析。
+    该文件被 Git 忽略，因此拉取仓库更新不会覆盖本地密钥和自定义配置。执行模型的
+    `run`、`run-eval` 和 `eval` 命令会在读取配置前校验顶层
+    `config_schema_version`；如果版本落后，请先备份 `config.yaml`，再合并最新版
+    `config.example.yaml` 中的必需改动，最后更新版本号。
     需要配置三部分：
 
     **1. 模型** — 在顶层 `models` 定义共享模型配置，并通过 `default.model` 选择默认模型：
 
     ```yaml
+    config_schema_version: 1
+
     default:
       model: gpt-5.4
 

docs_4_codeagent/imports-runtime-config.md

@@ -44,3 +44,6 @@ export PYTHONPATH=/path/to/browseruse_bench && python script.py
 - Do not hardcode timeout, URL, API key, model name, or similar runtime values.
 - Read config from `config.yaml`, environment variables, or passed config objects (for example `AgentConfig`).
 - Store configured file paths as relative paths and resolve to absolute paths with `REPO_ROOT` when reading.
+- When an existing ignored `config.yaml` requires a manual migration, increment both
+  `CURRENT_CONFIG_SCHEMA_VERSION` and `config.example.yaml`'s `config_schema_version`.
+  Keep the runtime mismatch error actionable; never overwrite user credentials or custom settings.

tests/browseruse_bench/test_cli.py

@@ -6,9 +6,9 @@
 
 import pytest
 
+import browseruse_bench.cli as cli_pkg
 from browseruse_bench.cli.viz import configure_viz_parser
-from browseruse_bench.utils import REPO_ROOT
-from browseruse_bench.utils import create_eval_parser, create_run_parser
+from browseruse_bench.utils import REPO_ROOT, create_eval_parser, create_run_parser
 
 
 class TestCreateRunParser:
@@ -26,6 +26,30 @@ def test_parser_has_required_arguments(self):
         assert hasattr(args, 'count')
 
 
+@pytest.mark.parametrize("command", ["run", "eval"])
+def test_runtime_commands_reject_non_mapping_config_before_parser(
+    command: str,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(cli_pkg, "load_config_file", lambda _: ["not", "a", "mapping"])
+
+    with pytest.raises(SystemExit, match="top-level YAML mapping"):
+        cli_pkg.main([command])
+
+
+@pytest.mark.parametrize("command", ["run", "eval"])
+def test_runtime_command_help_works_with_non_mapping_config(
+    command: str,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(cli_pkg, "load_config_file", lambda _: ["not", "a", "mapping"])
+
+    with pytest.raises(SystemExit) as exc_info:
+        cli_pkg.main([command, "--help"])
+
+    assert exc_info.value.code == 0
+
+
 class TestConfigureVizParser:
     def test_defaults(self):
         parser = argparse.ArgumentParser()

tests/browseruse_bench/test_config_loader.py

@@ -22,8 +22,10 @@
     resolve_split,
 )
 from browseruse_bench.utils.config_loader import (
+    CURRENT_CONFIG_SCHEMA_VERSION,
     load_eval_config,
     resolve_key_case_insensitive,
+    validate_runtime_config_schema,
 )
 
 
@@ -60,6 +62,32 @@ def test_load_empty_config_file(self, tmp_path: Path):
         assert result == {} or result is None
 
 
+class TestRuntimeConfigSchema:
+    def test_example_uses_current_schema_version(self) -> None:
+        config = load_config_file(REPO_ROOT / "config.example.yaml")
+
+        assert config["config_schema_version"] == CURRENT_CONFIG_SCHEMA_VERSION
+        validate_runtime_config_schema(config, "config.example.yaml")
+
+    @pytest.mark.parametrize("version", [None, 0, "1", True])
+    def test_rejects_stale_or_invalid_version(self, version: object) -> None:
+        config = {} if version is None else {"config_schema_version": version}
+
+        with pytest.raises(SystemExit, match="ignored by Git"):
+            validate_runtime_config_schema(config, "config.yaml")
+
+    @pytest.mark.parametrize("config", [[], "config", 1, True])
+    def test_rejects_non_mapping_yaml(self, config: object) -> None:
+        with pytest.raises(SystemExit, match="top-level YAML mapping"):
+            validate_runtime_config_schema(config, "config.yaml")
+
+    def test_rejects_config_newer_than_checkout(self) -> None:
+        config = {"config_schema_version": CURRENT_CONFIG_SCHEMA_VERSION + 1}
+
+        with pytest.raises(SystemExit, match="Update the repository first"):
+            validate_runtime_config_schema(config, "config.yaml")
+
+
 class TestLoadEvalConfig:
     """Tests for load_eval_config — returns shared eval settings (structural keys excluded)."""