From 7360b78ce25271ec493830ef14fca631e54ff6fc Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Sat, 21 Nov 2026 02:56:01 +0000 Subject: [PATCH] fix(datasets): limit the variables that a dataset can read on the eval server --- docs/configs/agentic-dataset-config.md | 1 + evalbench/dataset/dataset.py | 45 ++++++++- .../test/dataset_env_placeholders_test.py | 97 +++++++++++++++++++ 3 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 evalbench/test/dataset_env_placeholders_test.py diff --git a/docs/configs/agentic-dataset-config.md b/docs/configs/agentic-dataset-config.md index c1f85ce69..e6e103dc2 100644 --- a/docs/configs/agentic-dataset-config.md +++ b/docs/configs/agentic-dataset-config.md @@ -48,6 +48,7 @@ The file is a single JSON object with a `scenarios` array: ## Important Notes - **Environment placeholders:** `${VAR}` placeholders anywhere in the evalset are expanded from the environment at load time, so values like a GCP project need not be hard-coded. An unresolved placeholder **fails the run immediately** rather than passing a literal `${VAR}` to the agent — export the variable first. + - On the eval server (`eval_server.py`), the dataset comes from the client. There, a dataset can read only the variables in `EVALBENCH_DATASET_ENV_VARS` (comma-separated). The default is `EVAL_GCP_PROJECT_ID,EVAL_GCP_PROJECT_REGION,GOOGLE_CLOUD_PROJECT`. - **Scenario filtering:** You do not need separate evalset files to run a subset. Set `scenarios` (explicit IDs) or `scenario_pattern` (a glob over IDs) in the run config to filter at load time. - **Empty trajectories are valid:** For scenarios scored purely by rubric or goal completion, set `"expected_trajectory": []`. The `trajectory_matcher` scorer can simply be left out of the run config. - **Reset between trials:** If `num_trials` is greater than 1, trials run one after another in the same sandbox home and the same `work_dir`. Before each trial after the first, EvalBench puts both directories back to their state before the first trial, so a trial does not see the files of an earlier one. The download caches in the sandbox home (`.npm`, `.cache`, `.local`) are kept. diff --git a/evalbench/dataset/dataset.py b/evalbench/dataset/dataset.py index fa534dcc5..d2ff0a325 100644 --- a/evalbench/dataset/dataset.py +++ b/evalbench/dataset/dataset.py @@ -13,10 +13,30 @@ from itertools import chain import os import re +import sys _ENV_PLACEHOLDER_RE = re.compile(r"\$\{([^}]+)\}") +# On the eval server, a dataset can read only the variables in this setting. +SERVER_ENV_VARS_SETTING = "EVALBENCH_DATASET_ENV_VARS" +_DEFAULT_SERVER_ENV_VARS = ( + "EVAL_GCP_PROJECT_ID,EVAL_GCP_PROJECT_REGION,GOOGLE_CLOUD_PROJECT" +) + + +def _allowed_env_names() -> Optional[set[str]]: + """Returns the variables that a dataset can read. None means all. + + On the eval server the dataset comes from the client, and ListEvalInputs + sends the expanded dataset to the client. Thus the server expands only + the variables in EVALBENCH_DATASET_ENV_VARS (comma-separated). + """ + if not sys.argv[0].endswith("eval_server.py"): + return None + names = os.environ.get(SERVER_ENV_VARS_SETTING, _DEFAULT_SERVER_ENV_VARS) + return {name.strip() for name in names.split(",") if name.strip()} + def _expand_env_placeholders(text: str, source_path: str) -> str: """Substitutes ``${VAR}`` placeholders in a raw dataset from the @@ -24,15 +44,34 @@ def _expand_env_placeholders(text: str, source_path: str) -> str: of hard-coded. Fails fast on any unresolved placeholder rather than passing a literal ``${VAR}`` to the agent. """ - expanded = os.path.expandvars(text) + allowed = _allowed_env_names() + if allowed is None: + expanded = os.path.expandvars(text) + else: + expanded = _ENV_PLACEHOLDER_RE.sub( + lambda match: os.environ[match.group(1)] + if match.group(1) in allowed and match.group(1) in os.environ + else match.group(0), + text, + ) unresolved = sorted( {m.group(1) for m in _ENV_PLACEHOLDER_RE.finditer(expanded)} ) if unresolved: + if allowed is None: + hint = ( + "Export the corresponding environment variable(s) " + f"before running (e.g. `export {unresolved[0]}=...`)." + ) + else: + hint = ( + "On the eval server, a dataset can read only these " + f"variables: {sorted(allowed)}. The {SERVER_ENV_VARS_SETTING} " + "variable of the server sets this list." + ) raise ValueError( f"Unresolved ${{...}} placeholder(s) in dataset {source_path}: " - f"{unresolved}. Export the corresponding environment variable(s) " - f"before running (e.g. `export {unresolved[0]}=...`)." + f"{unresolved}. {hint}" ) return expanded diff --git a/evalbench/test/dataset_env_placeholders_test.py b/evalbench/test/dataset_env_placeholders_test.py new file mode 100644 index 000000000..3fb8ac046 --- /dev/null +++ b/evalbench/test/dataset_env_placeholders_test.py @@ -0,0 +1,97 @@ +"""Tests for the ${VAR} placeholders of a dataset.""" + +import json +import os +import sys +import tempfile +import unittest +from unittest.mock import patch + +from dataset import dataset +from dataset.dataset import _expand_env_placeholders, load_gemini_cli_json + +_ENV = { + "EVAL_GCP_PROJECT_ID": "my-project", + "SERVER_SECRET": "secret-value", + "TEAM_VAR": "team-value", +} + + +def _expand(text, argv0, setting=None): + env = dict(_ENV) + if setting is not None: + env[dataset.SERVER_ENV_VARS_SETTING] = setting + with patch.dict(os.environ, env), patch.object(sys, "argv", [argv0]): + os.environ.pop("NOT_SET_VAR", None) + if setting is None: + os.environ.pop(dataset.SERVER_ENV_VARS_SETTING, None) + return _expand_env_placeholders(text, "dataset.json") + + +class LocalRunTest(unittest.TestCase): + + def test_expands_each_variable(self): + self.assertEqual( + _expand("${EVAL_GCP_PROJECT_ID} ${SERVER_SECRET}", "evalbench.py"), + "my-project secret-value", + ) + + def test_unresolved_placeholder_fails(self): + with self.assertRaisesRegex(ValueError, "NOT_SET_VAR.*Export"): + _expand("${NOT_SET_VAR}", "evalbench.py") + + +class EvalServerTest(unittest.TestCase): + + ARGV0 = "/evalbench/evalbench/eval_server.py" + + def test_expands_a_variable_in_the_default_list(self): + self.assertEqual( + _expand("p=${EVAL_GCP_PROJECT_ID}", self.ARGV0), "p=my-project") + + def test_rejects_a_variable_that_is_not_in_the_list(self): + with self.assertRaises(ValueError) as raised: + _expand("${EVAL_GCP_PROJECT_ID} ${SERVER_SECRET}", self.ARGV0) + + message = str(raised.exception) + self.assertIn("SERVER_SECRET", message) + self.assertIn(dataset.SERVER_ENV_VARS_SETTING, message) + self.assertNotIn("secret-value", message) + + def test_does_not_expand_a_variable_without_braces(self): + self.assertEqual( + _expand("$SERVER_SECRET $EVAL_GCP_PROJECT_ID", self.ARGV0), + "$SERVER_SECRET $EVAL_GCP_PROJECT_ID", + ) + + def test_setting_replaces_the_default_list(self): + self.assertEqual( + _expand("${TEAM_VAR}", self.ARGV0, setting="TEAM_VAR, OTHER"), + "team-value", + ) + with self.assertRaises(ValueError): + _expand("${EVAL_GCP_PROJECT_ID}", self.ARGV0, setting="TEAM_VAR") + with self.assertRaises(ValueError): + _expand("${EVAL_GCP_PROJECT_ID}", self.ARGV0, setting="") + + def test_unset_variable_in_the_list_fails(self): + with self.assertRaises(ValueError): + _expand("${NOT_SET_VAR}", self.ARGV0, setting="NOT_SET_VAR") + + def test_dataset_load_does_not_return_a_server_variable(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "evalset.json") + with open(path, "w") as f: + json.dump({"scenarios": [{ + "id": "s1", + "starting_prompt": "${SERVER_SECRET}", + }]}, f) + + with patch.dict(os.environ, _ENV), patch.object( + sys, "argv", [self.ARGV0]): + with self.assertRaises(ValueError): + load_gemini_cli_json(path, {}) + + +if __name__ == "__main__": + unittest.main()