Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 123 additions & 0 deletions evalbench/test/config_redaction_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
"""Tests for the removal of secret values from the reported config."""

import copy
import unittest

from util.config import REDACTED, config_to_df, redact_secrets


def _reported(experiment_config=None, model_config=None, db_configs=None):
"""Returns the reported config as a {key: value} dict."""
df = config_to_df(
"job", None, experiment_config or {}, model_config or {},
db_configs or [])
return dict(zip(df["config"].tolist(), df["value"].tolist()))


class RedactSecretsTest(unittest.TestCase):

def test_redacts_secret_keys(self):
config = {
"password": "pw",
"CLOUD_SQL_POSTGRES_PASSWORD": "pw",
"client_secret": "s",
"GITHUB_TOKEN": "t",
"accessToken": "t",
"GOOGLE_API_KEY": "k",
"x-api-key": "k",
"apiKey": "k",
"private_key": "k",
"Authorization": "Bearer t",
"credentials": {"user": "u", "value": "v"},
}

redacted = redact_secrets(config)

self.assertEqual(set(redacted.values()), {REDACTED})

def test_keeps_keys_that_are_not_secrets(self):
config = {
"token_consumption": {},
"tokens_processed": {},
"effective_billed_tokens": {},
"max_tokens": 1024,
"token_budget": 5,
"secret_manager_path": "projects/p/secrets/s",
"key": "primary",
"partition_key": "id",
"user_name": "evalbench",
}

self.assertEqual(redact_secrets(config), config)

def test_keeps_empty_values_and_secret_references(self):
config = {
"password": "",
"api_key": None,
"use_token": False,
"openai_api_key_secret": "projects/p/secrets/s/versions/1",
}

self.assertEqual(redact_secrets(config), config)

def test_redacts_inside_lists_and_nested_dicts(self):
config = {
"db_configs": [{"user_name": "u", "password": "pw"}],
"setup": {"mcp_servers": {"s": {"headers": {"Authorization": "t"}}}},
}

self.assertEqual(redact_secrets(config), {
"db_configs": [{"user_name": "u", "password": REDACTED}],
"setup": {
"mcp_servers": {"s": {"headers": {"Authorization": REDACTED}}}
},
})

def test_redacts_the_password_in_a_url(self):
config = {
"connection_string": "mongodb://user:pw@host:27017/db",
"url": "https://example.com/a:b@c",
"image": "repo/image:tag",
}

self.assertEqual(redact_secrets(config), {
"connection_string": f"mongodb://user:{REDACTED}@host:27017/db",
"url": "https://example.com/a:b@c",
"image": "repo/image:tag",
})

def test_does_not_change_the_input(self):
config = {"db_configs": [{"password": "pw"}], "env": {"API_KEY": "k"}}
original = copy.deepcopy(config)

redact_secrets(config)

self.assertEqual(config, original)


class ConfigToDfRedactionTest(unittest.TestCase):

def test_report_has_no_secret_values(self):
secrets = ["db-password", "api-key-value", "bearer-token"]
reported = _reported(
experiment_config={"orchestrator": "agent"},
model_config={
"generator": "gemini_cli",
"env": {"GOOGLE_API_KEY": secrets[1]},
"setup": {"mcp_servers": {
"s": {"headers": {"Authorization": secrets[2]}}}},
},
db_configs=[{"user_name": "u", "password": secrets[0]}],
)

for value in reported.values():
for secret in secrets:
self.assertNotIn(secret, str(value))
self.assertEqual(reported["experiment_config.orchestrator"], "agent")
self.assertEqual(reported["model_config.generator"], "gemini_cli")
self.assertEqual(reported["model_config.env.GOOGLE_API_KEY"], REDACTED)
self.assertIn("'user_name': 'u'", reported["db_configs"])


if __name__ == "__main__":
unittest.main()
64 changes: 62 additions & 2 deletions evalbench/util/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,66 @@ def _load_setup_sql(sql_file_path: str):
return []


REDACTED = "[REDACTED]"
_SECRET_KEY_WORDS = frozenset({
"password", "passwd", "secret", "token", "credential", "credentials",
"authorization", "apikey",
})
# "key" is a secret only after one of these words, as in api_key.
_SECRET_KEY_QUALIFIERS = frozenset({"api", "private", "access", "secret"})
_CAMEL_CASE_RE = re.compile(r"([a-z0-9])([A-Z])")
_URL_PASSWORD_RE = re.compile(r"(://[^/\s:@]+:)[^@\s/]+(@)")
# A Secret Manager resource name is a reference to a secret, not a secret.
_SECRET_REFERENCE_RE = re.compile(
r"^projects/[^/\s]+/secrets/[^/\s]+(/versions/[^/\s]+)?$")


def _is_secret_key(key) -> bool:
"""Returns True if the last words of a config key name a secret."""
words = [
word
for word in re.split(
r"[^a-z0-9]+", _CAMEL_CASE_RE.sub(r"\1_\2", str(key)).lower())
if word
]
if not words:
return False
if words[-1] in _SECRET_KEY_WORDS:
return True
return (
len(words) > 1
and words[-1] == "key"
and words[-2] in _SECRET_KEY_QUALIFIERS
)


def _is_secret_value(value) -> bool:
if value is None or value == "" or isinstance(value, bool):
return False
return not (isinstance(value, str) and _SECRET_REFERENCE_RE.match(value))


def redact_secrets(config):
"""Returns a copy of a config without the values of secret keys.

The copy is for reports (configs.csv, BigQuery). It also removes the
password from a URL of the form scheme://user:password@host.
"""
if isinstance(config, dict):
return {
key: REDACTED
if _is_secret_key(key) and _is_secret_value(value)
else redact_secrets(value)
for key, value in config.items()
}
if isinstance(config, (list, tuple)):
return [redact_secrets(item) for item in config]
if isinstance(config, str):
return _URL_PASSWORD_RE.sub(
lambda match: match.group(1) + REDACTED + match.group(2), config)
return config


def config_to_df(
job_id: str,
run_time: datetime.datetime,
Expand All @@ -114,11 +174,11 @@ def config_to_df(
db_configs: list[dict],
):
configs = []
config = {
config = redact_secrets({
"experiment_config": experiment_config,
"model_config": model_config,
"db_configs": db_configs,
}
})
df = pd.json_normalize(config, sep=".")
d_flat = df.to_dict(orient="records")[0]
for key in d_flat:
Expand Down
Loading