diff --git a/evalbench/test/config_redaction_test.py b/evalbench/test/config_redaction_test.py new file mode 100644 index 00000000..0943cdfa --- /dev/null +++ b/evalbench/test/config_redaction_test.py @@ -0,0 +1,123 @@ +"""Tests for the removal of secret values from the reported config.""" + +import copy +import unittest + +from util.config import REDACTED, config_to_df, redact_secrets + + +def _reported(experiment_config=None, model_config=None, db_configs=None): + """Returns the reported config as a {key: value} dict.""" + df = config_to_df( + "job", None, experiment_config or {}, model_config or {}, + db_configs or []) + return dict(zip(df["config"].tolist(), df["value"].tolist())) + + +class RedactSecretsTest(unittest.TestCase): + + def test_redacts_secret_keys(self): + config = { + "password": "pw", + "CLOUD_SQL_POSTGRES_PASSWORD": "pw", + "client_secret": "s", + "GITHUB_TOKEN": "t", + "accessToken": "t", + "GOOGLE_API_KEY": "k", + "x-api-key": "k", + "apiKey": "k", + "private_key": "k", + "Authorization": "Bearer t", + "credentials": {"user": "u", "value": "v"}, + } + + redacted = redact_secrets(config) + + self.assertEqual(set(redacted.values()), {REDACTED}) + + def test_keeps_keys_that_are_not_secrets(self): + config = { + "token_consumption": {}, + "tokens_processed": {}, + "effective_billed_tokens": {}, + "max_tokens": 1024, + "token_budget": 5, + "secret_manager_path": "projects/p/secrets/s", + "key": "primary", + "partition_key": "id", + "user_name": "evalbench", + } + + self.assertEqual(redact_secrets(config), config) + + def test_keeps_empty_values_and_secret_references(self): + config = { + "password": "", + "api_key": None, + "use_token": False, + "openai_api_key_secret": "projects/p/secrets/s/versions/1", + } + + self.assertEqual(redact_secrets(config), config) + + def test_redacts_inside_lists_and_nested_dicts(self): + config = { + "db_configs": [{"user_name": "u", "password": "pw"}], + "setup": {"mcp_servers": {"s": {"headers": {"Authorization": "t"}}}}, + } + + self.assertEqual(redact_secrets(config), { + "db_configs": [{"user_name": "u", "password": REDACTED}], + "setup": { + "mcp_servers": {"s": {"headers": {"Authorization": REDACTED}}} + }, + }) + + def test_redacts_the_password_in_a_url(self): + config = { + "connection_string": "mongodb://user:pw@host:27017/db", + "url": "https://example.com/a:b@c", + "image": "repo/image:tag", + } + + self.assertEqual(redact_secrets(config), { + "connection_string": f"mongodb://user:{REDACTED}@host:27017/db", + "url": "https://example.com/a:b@c", + "image": "repo/image:tag", + }) + + def test_does_not_change_the_input(self): + config = {"db_configs": [{"password": "pw"}], "env": {"API_KEY": "k"}} + original = copy.deepcopy(config) + + redact_secrets(config) + + self.assertEqual(config, original) + + +class ConfigToDfRedactionTest(unittest.TestCase): + + def test_report_has_no_secret_values(self): + secrets = ["db-password", "api-key-value", "bearer-token"] + reported = _reported( + experiment_config={"orchestrator": "agent"}, + model_config={ + "generator": "gemini_cli", + "env": {"GOOGLE_API_KEY": secrets[1]}, + "setup": {"mcp_servers": { + "s": {"headers": {"Authorization": secrets[2]}}}}, + }, + db_configs=[{"user_name": "u", "password": secrets[0]}], + ) + + for value in reported.values(): + for secret in secrets: + self.assertNotIn(secret, str(value)) + self.assertEqual(reported["experiment_config.orchestrator"], "agent") + self.assertEqual(reported["model_config.generator"], "gemini_cli") + self.assertEqual(reported["model_config.env.GOOGLE_API_KEY"], REDACTED) + self.assertIn("'user_name': 'u'", reported["db_configs"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/evalbench/util/config.py b/evalbench/util/config.py index 90f97a3e..40aaa3e1 100644 --- a/evalbench/util/config.py +++ b/evalbench/util/config.py @@ -106,6 +106,66 @@ def _load_setup_sql(sql_file_path: str): return [] +REDACTED = "[REDACTED]" +_SECRET_KEY_WORDS = frozenset({ + "password", "passwd", "secret", "token", "credential", "credentials", + "authorization", "apikey", +}) +# "key" is a secret only after one of these words, as in api_key. +_SECRET_KEY_QUALIFIERS = frozenset({"api", "private", "access", "secret"}) +_CAMEL_CASE_RE = re.compile(r"([a-z0-9])([A-Z])") +_URL_PASSWORD_RE = re.compile(r"(://[^/\s:@]+:)[^@\s/]+(@)") +# A Secret Manager resource name is a reference to a secret, not a secret. +_SECRET_REFERENCE_RE = re.compile( + r"^projects/[^/\s]+/secrets/[^/\s]+(/versions/[^/\s]+)?$") + + +def _is_secret_key(key) -> bool: + """Returns True if the last words of a config key name a secret.""" + words = [ + word + for word in re.split( + r"[^a-z0-9]+", _CAMEL_CASE_RE.sub(r"\1_\2", str(key)).lower()) + if word + ] + if not words: + return False + if words[-1] in _SECRET_KEY_WORDS: + return True + return ( + len(words) > 1 + and words[-1] == "key" + and words[-2] in _SECRET_KEY_QUALIFIERS + ) + + +def _is_secret_value(value) -> bool: + if value is None or value == "" or isinstance(value, bool): + return False + return not (isinstance(value, str) and _SECRET_REFERENCE_RE.match(value)) + + +def redact_secrets(config): + """Returns a copy of a config without the values of secret keys. + + The copy is for reports (configs.csv, BigQuery). It also removes the + password from a URL of the form scheme://user:password@host. + """ + if isinstance(config, dict): + return { + key: REDACTED + if _is_secret_key(key) and _is_secret_value(value) + else redact_secrets(value) + for key, value in config.items() + } + if isinstance(config, (list, tuple)): + return [redact_secrets(item) for item in config] + if isinstance(config, str): + return _URL_PASSWORD_RE.sub( + lambda match: match.group(1) + REDACTED + match.group(2), config) + return config + + def config_to_df( job_id: str, run_time: datetime.datetime, @@ -114,11 +174,11 @@ def config_to_df( db_configs: list[dict], ): configs = [] - config = { + config = redact_secrets({ "experiment_config": experiment_config, "model_config": model_config, "db_configs": db_configs, - } + }) df = pd.json_normalize(config, sep=".") d_flat = df.to_dict(orient="records")[0] for key in d_flat: