Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/configs/agentic-dataset-config.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ The file is a single JSON object with a `scenarios` array:
## Important Notes

- **Environment placeholders:** `${VAR}` placeholders anywhere in the evalset are expanded from the environment at load time, so values like a GCP project need not be hard-coded. An unresolved placeholder **fails the run immediately** rather than passing a literal `${VAR}` to the agent — export the variable first.
- On the eval server (`eval_server.py`), the dataset comes from the client. There, a dataset can read only the variables in `EVALBENCH_DATASET_ENV_VARS` (comma-separated). The default is `EVAL_GCP_PROJECT_ID,EVAL_GCP_PROJECT_REGION,GOOGLE_CLOUD_PROJECT`.
- **Scenario filtering:** You do not need separate evalset files to run a subset. Set `scenarios` (explicit IDs) or `scenario_pattern` (a glob over IDs) in the run config to filter at load time.
- **Empty trajectories are valid:** For scenarios scored purely by rubric or goal completion, set `"expected_trajectory": []`. The `trajectory_matcher` scorer can simply be left out of the run config.
- **Reset between trials:** If `num_trials` is greater than 1, trials run one after another in the same sandbox home and the same `work_dir`. Before each trial after the first, EvalBench puts both directories back to their state before the first trial, so a trial does not see the files of an earlier one. The download caches in the sandbox home (`.npm`, `.cache`, `.local`) are kept.
Expand Down
45 changes: 42 additions & 3 deletions evalbench/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,65 @@
from itertools import chain
import os
import re
import sys


_ENV_PLACEHOLDER_RE = re.compile(r"\$\{([^}]+)\}")

# On the eval server, a dataset can read only the variables in this setting.
SERVER_ENV_VARS_SETTING = "EVALBENCH_DATASET_ENV_VARS"
_DEFAULT_SERVER_ENV_VARS = (
"EVAL_GCP_PROJECT_ID,EVAL_GCP_PROJECT_REGION,GOOGLE_CLOUD_PROJECT"
)


def _allowed_env_names() -> Optional[set[str]]:
"""Returns the variables that a dataset can read. None means all.

On the eval server the dataset comes from the client, and ListEvalInputs
sends the expanded dataset to the client. Thus the server expands only
the variables in EVALBENCH_DATASET_ENV_VARS (comma-separated).
"""
if not sys.argv[0].endswith("eval_server.py"):
return None
names = os.environ.get(SERVER_ENV_VARS_SETTING, _DEFAULT_SERVER_ENV_VARS)
return {name.strip() for name in names.split(",") if name.strip()}


def _expand_env_placeholders(text: str, source_path: str) -> str:
"""Substitutes ``${VAR}`` placeholders in a raw dataset from the
environment, so values like a GCP project are supplied at runtime instead
of hard-coded. Fails fast on any unresolved placeholder rather than
passing a literal ``${VAR}`` to the agent.
"""
expanded = os.path.expandvars(text)
allowed = _allowed_env_names()
if allowed is None:
expanded = os.path.expandvars(text)
else:
expanded = _ENV_PLACEHOLDER_RE.sub(
lambda match: os.environ[match.group(1)]
if match.group(1) in allowed and match.group(1) in os.environ
else match.group(0),
text,
)
unresolved = sorted(
{m.group(1) for m in _ENV_PLACEHOLDER_RE.finditer(expanded)}
)
if unresolved:
if allowed is None:
hint = (
"Export the corresponding environment variable(s) "
f"before running (e.g. `export {unresolved[0]}=...`)."
)
else:
hint = (
"On the eval server, a dataset can read only these "
f"variables: {sorted(allowed)}. The {SERVER_ENV_VARS_SETTING} "
"variable of the server sets this list."
)
raise ValueError(
f"Unresolved ${{...}} placeholder(s) in dataset {source_path}: "
f"{unresolved}. Export the corresponding environment variable(s) "
f"before running (e.g. `export {unresolved[0]}=...`)."
f"{unresolved}. {hint}"
)
return expanded

Expand Down
97 changes: 97 additions & 0 deletions evalbench/test/dataset_env_placeholders_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""Tests for the ${VAR} placeholders of a dataset."""

import json
import os
import sys
import tempfile
import unittest
from unittest.mock import patch

from dataset import dataset
from dataset.dataset import _expand_env_placeholders, load_gemini_cli_json

_ENV = {
"EVAL_GCP_PROJECT_ID": "my-project",
"SERVER_SECRET": "secret-value",
"TEAM_VAR": "team-value",
}


def _expand(text, argv0, setting=None):
env = dict(_ENV)
if setting is not None:
env[dataset.SERVER_ENV_VARS_SETTING] = setting
with patch.dict(os.environ, env), patch.object(sys, "argv", [argv0]):
os.environ.pop("NOT_SET_VAR", None)
if setting is None:
os.environ.pop(dataset.SERVER_ENV_VARS_SETTING, None)
return _expand_env_placeholders(text, "dataset.json")


class LocalRunTest(unittest.TestCase):

def test_expands_each_variable(self):
self.assertEqual(
_expand("${EVAL_GCP_PROJECT_ID} ${SERVER_SECRET}", "evalbench.py"),
"my-project secret-value",
)

def test_unresolved_placeholder_fails(self):
with self.assertRaisesRegex(ValueError, "NOT_SET_VAR.*Export"):
_expand("${NOT_SET_VAR}", "evalbench.py")


class EvalServerTest(unittest.TestCase):

ARGV0 = "/evalbench/evalbench/eval_server.py"

def test_expands_a_variable_in_the_default_list(self):
self.assertEqual(
_expand("p=${EVAL_GCP_PROJECT_ID}", self.ARGV0), "p=my-project")

def test_rejects_a_variable_that_is_not_in_the_list(self):
with self.assertRaises(ValueError) as raised:
_expand("${EVAL_GCP_PROJECT_ID} ${SERVER_SECRET}", self.ARGV0)

message = str(raised.exception)
self.assertIn("SERVER_SECRET", message)
self.assertIn(dataset.SERVER_ENV_VARS_SETTING, message)
self.assertNotIn("secret-value", message)

def test_does_not_expand_a_variable_without_braces(self):
self.assertEqual(
_expand("$SERVER_SECRET $EVAL_GCP_PROJECT_ID", self.ARGV0),
"$SERVER_SECRET $EVAL_GCP_PROJECT_ID",
)

def test_setting_replaces_the_default_list(self):
self.assertEqual(
_expand("${TEAM_VAR}", self.ARGV0, setting="TEAM_VAR, OTHER"),
"team-value",
)
with self.assertRaises(ValueError):
_expand("${EVAL_GCP_PROJECT_ID}", self.ARGV0, setting="TEAM_VAR")
with self.assertRaises(ValueError):
_expand("${EVAL_GCP_PROJECT_ID}", self.ARGV0, setting="")

def test_unset_variable_in_the_list_fails(self):
with self.assertRaises(ValueError):
_expand("${NOT_SET_VAR}", self.ARGV0, setting="NOT_SET_VAR")

def test_dataset_load_does_not_return_a_server_variable(self):
with tempfile.TemporaryDirectory() as tmp:
path = os.path.join(tmp, "evalset.json")
with open(path, "w") as f:
json.dump({"scenarios": [{
"id": "s1",
"starting_prompt": "${SERVER_SECRET}",
}]}, f)

with patch.dict(os.environ, _ENV), patch.object(
sys, "argv", [self.ARGV0]):
with self.assertRaises(ValueError):
load_gemini_cli_json(path, {})


if __name__ == "__main__":
unittest.main()
Loading