From 10181aa619240f487fbb5b251011989f13875e2f Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Sat, 21 Nov 2026 02:52:55 +0000 Subject: [PATCH] fix: keep dataset file names inside their base directory --- evalbench/container/case_runner.py | 7 +- evalbench/container/spec.py | 12 ++- evalbench/databases/sqlite.py | 7 +- evalbench/scorers/sqlite_bridge.py | 8 +- evalbench/test/safepath_test.py | 136 +++++++++++++++++++++++++++++ evalbench/util/safepath.py | 25 ++++++ 6 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 evalbench/test/safepath_test.py create mode 100644 evalbench/util/safepath.py diff --git a/evalbench/container/case_runner.py b/evalbench/container/case_runner.py index 818aacd5..b80fa4e3 100644 --- a/evalbench/container/case_runner.py +++ b/evalbench/container/case_runner.py @@ -30,6 +30,7 @@ from container.backend import RESULT_BEGIN, RESULT_END # noqa: E402 from container.spec import EvalCaseSpec # noqa: E402 +from util.safepath import contained_path # noqa: E402 def _configure_logging() -> None: @@ -79,7 +80,11 @@ def _materialize_env_files(spec: EvalCaseSpec, session_dir: str) -> None: env_dir = os.path.join(session_dir, "env_files") os.makedirs(env_dir, exist_ok=True) for name, content in spec.env_files.items(): - path = os.path.join(env_dir, name) + try: + path = contained_path(env_dir, name) + except ValueError as e: + logging.error("case: skipped an env file: %s", e) + continue os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write(content) diff --git a/evalbench/container/spec.py b/evalbench/container/spec.py index 79d0353a..cac4dbc4 100644 --- a/evalbench/container/spec.py +++ b/evalbench/container/spec.py @@ -16,6 +16,8 @@ import re import yaml +from util.safepath import contained_path + # Where the spec is mounted inside the case container. CASE_DIR = "/etc/evalbench-case" @@ -415,8 +417,16 @@ def _collect_env_files(scenario: dict, session_dir: Optional[str]) -> dict[str, return {} collected: dict[str, str] = {} + env_dir = os.path.join(session_dir, "env_files") for name in declared: - src = os.path.join(session_dir, "env_files", name) + # The scenario comes from the dataset, so the name is untrusted. + try: + src = contained_path(env_dir, name) + except ValueError as e: + logging.warning( + "Skipped an env file of containerized case %s: %s", + scenario.get("id"), e) + continue if not os.path.isfile(src): logging.warning( "Declared env file not found for containerized case %s: %s", diff --git a/evalbench/databases/sqlite.py b/evalbench/databases/sqlite.py index b6524295..bc6f4690 100644 --- a/evalbench/databases/sqlite.py +++ b/evalbench/databases/sqlite.py @@ -14,6 +14,7 @@ DatabaseSchema, ) from util.rate_limit import rate_limit, ResourceExhaustedError +from util.safepath import contained_path from typing import Any, List, Optional, Tuple DROP_TABLE_SQL = "DROP TABLE {TABLE};" @@ -229,7 +230,7 @@ def drop_tmp_database(self, database_name: str): logging.error(f"Could not delete database: {error}") def ensure_database_exists(self, database_name: str) -> None: - filename = f"{self.db_path}/{database_name}{self.extension}" + filename = self._get_connection_path(self.db_path, database_name) os.makedirs(self.db_path, exist_ok=True) conn = sqlite3.connect(filename) conn.close() @@ -294,5 +295,5 @@ def _get_connection_path(self, db_path, db_name): if not db_name.endswith(extension): db_name = db_name + extension - connection_path = os.path.join(db_path, f"{db_name}") - return connection_path + # The name can come from the dataset. Keep the file in db_path. + return contained_path(db_path, db_name) diff --git a/evalbench/scorers/sqlite_bridge.py b/evalbench/scorers/sqlite_bridge.py index 592001c2..d6c5832d 100644 --- a/evalbench/scorers/sqlite_bridge.py +++ b/evalbench/scorers/sqlite_bridge.py @@ -1,11 +1,13 @@ """SQLite Ground Truth Resolution Adapter for EvalBench.""" +import logging import os import sqlite3 import pandas as pd from databases.util import unique_column_names +from util.safepath import contained_path def get_sqlite_ground_truth( @@ -15,7 +17,11 @@ def get_sqlite_ground_truth( ) -> list: """Resolves candidate SQLite database files and executes query.""" - sqlite_path = os.path.join(db_dir, f"{database}.sqlite") + try: + sqlite_path = contained_path(db_dir, f"{database}.sqlite") + except ValueError as e: + logging.warning("Skipped the SQLite ground truth: %s", e) + return [] if not os.path.exists(sqlite_path): return [] conn = sqlite3.connect(sqlite_path) diff --git a/evalbench/test/safepath_test.py b/evalbench/test/safepath_test.py new file mode 100644 index 00000000..7b2d712e --- /dev/null +++ b/evalbench/test/safepath_test.py @@ -0,0 +1,136 @@ +"""Tests for file names that come from a dataset or a client.""" + +import os +import sqlite3 +import tempfile +import unittest +from types import SimpleNamespace + +from container import case_runner +from container.spec import _collect_env_files +from databases.sqlite import SQLiteDB +from scorers.sqlite_bridge import get_sqlite_ground_truth +from util.safepath import contained_path + + +class _TempDirTestCase(unittest.TestCase): + + def setUp(self): + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + self.tmp = temp_dir.name + self.base = os.path.join(self.tmp, "base") + self.outside = os.path.join(self.tmp, "outside") + os.makedirs(self.base) + os.makedirs(self.outside) + + def _files_outside(self): + return sorted(os.listdir(self.outside)) + + +class ContainedPathTest(_TempDirTestCase): + + def test_returns_the_joined_path(self): + for name in ("a.txt", "sub/a.txt", "sub/../a.txt"): + self.assertEqual( + contained_path(self.base, name), os.path.join(self.base, name)) + + def test_rejects_names_that_leave_the_base(self): + names = [ + "", ".", "..", "../outside/a.txt", "sub/../../outside/a.txt", + os.path.join(self.outside, "a.txt"), "a\x00.txt", None, 5, + ] + for name in names: + with self.assertRaises(ValueError, msg=repr(name)): + contained_path(self.base, name) + + def test_rejects_a_symlink_that_leaves_the_base(self): + os.symlink(self.outside, os.path.join(self.base, "link")) + + with self.assertRaises(ValueError): + contained_path(self.base, "link/a.txt") + + +class ContainerEnvFilesTest(_TempDirTestCase): + + def setUp(self): + super().setUp() + self.env_dir = os.path.join(self.base, "env_files") + os.makedirs(os.path.join(self.env_dir, "env")) + with open(os.path.join(self.env_dir, "env", "ok.py"), "w") as f: + f.write("ok") + self.host_file = os.path.join(self.outside, "host.txt") + with open(self.host_file, "w") as f: + f.write("HOST_FILE") + + def test_collect_skips_files_outside_the_env_files_directory(self): + scenario = { + "id": "s", + "env_files": ["../../outside/host.txt", self.host_file, "env/ok.py"], + } + + self.assertEqual( + _collect_env_files(scenario, self.base), {"env/ok.py": "ok"}) + + def test_materialize_skips_files_outside_the_env_files_directory(self): + spec = SimpleNamespace(env_files={ + "../../outside/written.txt": "x", + os.path.join(self.outside, "abs.txt"): "x", + "env/new.py": "new", + }) + + case_runner._materialize_env_files(spec, self.base) + + self.assertEqual(self._files_outside(), ["host.txt"]) + with open(os.path.join(self.env_dir, "env", "new.py")) as f: + self.assertEqual(f.read(), "new") + + +class SqliteDatabaseNameTest(_TempDirTestCase): + + def _db(self, database_name): + return SQLiteDB({ + "db_type": "sqlite", + "database_path": self.base, + "database_name": database_name, + "extension": ".sqlite", + "max_executions_per_minute": 10, + }) + + def test_connects_to_a_database_in_the_database_path(self): + db = self._db("good") + + with db.engine.connect(): + pass + + self.assertEqual(os.listdir(self.base), ["good.sqlite"]) + + def test_rejects_a_database_name_outside_the_database_path(self): + for name in ("../outside/evil", os.path.join(self.outside, "evil")): + db = self._db(name) + + with self.assertRaises(ValueError): + with db.engine.connect(): + pass + with self.assertRaises(ValueError): + db.ensure_database_exists(name) + with self.assertRaises(RuntimeError): + db.create_tmp_database(name) + + self.assertEqual(self._files_outside(), []) + + def test_ground_truth_skips_a_database_outside_the_directory(self): + path = os.path.join(self.outside, "other.sqlite") + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE t (id INT)") + conn.execute("INSERT INTO t VALUES (1)") + conn.commit() + conn.close() + + for name in ("../outside/other", os.path.join(self.outside, "other")): + self.assertEqual( + get_sqlite_ground_truth("SELECT * FROM t", name, self.base), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/evalbench/util/safepath.py b/evalbench/util/safepath.py new file mode 100644 index 00000000..ef39526c --- /dev/null +++ b/evalbench/util/safepath.py @@ -0,0 +1,25 @@ +"""Path checks for file names that come from a dataset or a client.""" + +import os + + +def contained_path(base: str, name: str) -> str: + """Returns base/name. Raises ValueError if the path is outside base. + + realpath resolves "../" and symlinks before the containment check. + """ + if ( + not isinstance(name, str) + or not name + or "\x00" in name + or os.path.isabs(name) + ): + raise ValueError(f"Invalid file name: {name!r}") + real_base = os.path.realpath(base) + real_path = os.path.realpath(os.path.join(real_base, name)) + if ( + real_path == real_base + or os.path.commonpath([real_base, real_path]) != real_base + ): + raise ValueError(f"File name is outside {base!r}: {name!r}") + return os.path.join(base, name)