diff --git a/evalbench/test/class_loader_server_test.py b/evalbench/test/class_loader_server_test.py new file mode 100644 index 00000000..24ac1235 --- /dev/null +++ b/evalbench/test/class_loader_server_test.py @@ -0,0 +1,97 @@ +"""Tests for the custom classes that the eval server can load.""" + +import collections +import collections.abc +import os +import sys +import tempfile +import threading +import unittest +from unittest.mock import patch + +from databases import get_database +from generators.models import get_generator +from util import class_loader +from util.class_loader import load_custom_class + +_SERVER = "/evalbench/evalbench/eval_server.py" + + +def _load(class_path, argv0, modules=None): + env = {} if modules is None else { + class_loader.SERVER_MODULES_SETTING: modules} + with patch.dict(os.environ, env), patch.object(sys, "argv", [argv0]): + if modules is None: + os.environ.pop(class_loader.SERVER_MODULES_SETTING, None) + return load_custom_class(class_path) + + +class LocalRunTest(unittest.TestCase): + + def test_loads_each_class(self): + self.assertIs( + _load("collections:OrderedDict", "evalbench.py"), + collections.OrderedDict) + + +class EvalServerTest(unittest.TestCase): + + def test_default_list_permits_no_class(self): + for class_path in ("subprocess:Popen", "collections.OrderedDict"): + with self.assertRaises(PermissionError) as raised: + _load(class_path, _SERVER) + self.assertIn( + class_loader.SERVER_MODULES_SETTING, str(raised.exception)) + + def test_loads_a_class_from_a_permitted_module(self): + modules = "my_package, collections" + + self.assertIs( + _load("collections:OrderedDict", _SERVER, modules), + collections.OrderedDict) + self.assertIs( + _load("collections.abc:Mapping", _SERVER, modules), + collections.abc.Mapping) + + def test_rejects_a_module_that_is_not_permitted(self): + for class_path in ("subprocess:Popen", "collectionsx:OrderedDict", + "os.path:join"): + with self.assertRaises(PermissionError): + _load(class_path, _SERVER, "collections") + + def test_does_not_import_a_rejected_module(self): + with patch.object(class_loader.importlib, "import_module") as imported: + with self.assertRaises(PermissionError): + _load("subprocess:Popen", _SERVER) + + imported.assert_not_called() + + def test_generator_class_does_not_run_on_the_server(self): + with tempfile.TemporaryDirectory() as tmp: + marker = os.path.join(tmp, "MARKER") + config_path = os.path.join(tmp, "model.yaml") + with open(config_path, "w") as f: + # subprocess.Popen(config) uses the keys as the command. + f.write(f'"/usr/bin/touch": 1\n"{marker}": 2\n' + 'generator_class: "subprocess:Popen"\n') + global_models = { + "lock": threading.Lock(), "registered_models": {}} + + with patch.dict(os.environ), patch.object(sys, "argv", [_SERVER]): + os.environ.pop(class_loader.SERVER_MODULES_SETTING, None) + with self.assertRaises(PermissionError): + get_generator(global_models, config_path) + + self.assertFalse(os.path.exists(marker)) + + def test_connector_class_does_not_load_on_the_server(self): + db_config = {"connector_class": "subprocess:Popen"} + + with patch.dict(os.environ), patch.object(sys, "argv", [_SERVER]): + os.environ.pop(class_loader.SERVER_MODULES_SETTING, None) + with self.assertRaises(PermissionError): + get_database(db_config, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/evalbench/util/class_loader.py b/evalbench/util/class_loader.py index 8c9c1757..5831b4c3 100644 --- a/evalbench/util/class_loader.py +++ b/evalbench/util/class_loader.py @@ -7,6 +7,34 @@ _SYS_PATH_LOCK = threading.Lock() +# On the eval server, a custom class must be in a module of this setting. +SERVER_MODULES_SETTING = "EVALBENCH_CUSTOM_CLASS_MODULES" + + +def _check_server_modules(class_path: str, mod_name: str) -> None: + """Raises PermissionError if the eval server must not load mod_name. + + On the eval server the config comes from the client, so a class path can + name any importable class. Thus the server loads a custom class only from + the modules in EVALBENCH_CUSTOM_CLASS_MODULES (comma-separated). A module + in the list also permits its submodules. The default list is empty. + """ + if not sys.argv[0].endswith("eval_server.py"): + return + allowed = [ + name.strip() + for name in os.environ.get(SERVER_MODULES_SETTING, "").split(",") + if name.strip() + ] + if any(mod_name == name or mod_name.startswith(name + ".") + for name in allowed): + return + raise PermissionError( + f"The eval server does not load the custom class '{class_path}'. " + f"The {SERVER_MODULES_SETTING} variable of the server sets the " + "permitted modules." + ) + def _import_with_cwd_fallback(mod_name: str): """Imports mod_name, falling back to appending os.getcwd() to sys.path.""" @@ -43,6 +71,7 @@ def load_custom_class(class_path: str): Raises: ValueError: If class_path format is invalid. + PermissionError: If the eval server does not permit the module. ImportError: If the module cannot be imported. AttributeError: If the class does not exist in the module. @@ -60,6 +89,8 @@ def load_custom_class(class_path: str): " 'module.submodule.ClassName' or 'module:ClassName'." ) + _check_server_modules(class_path, mod_name) + try: mod = _import_with_cwd_fallback(mod_name) except ImportError as e: