Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion evalbench/container/case_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

from container.backend import RESULT_BEGIN, RESULT_END # noqa: E402
from container.spec import EvalCaseSpec # noqa: E402
from util.safepath import contained_path # noqa: E402


def _configure_logging() -> None:
Expand Down Expand Up @@ -79,7 +80,11 @@ def _materialize_env_files(spec: EvalCaseSpec, session_dir: str) -> None:
env_dir = os.path.join(session_dir, "env_files")
os.makedirs(env_dir, exist_ok=True)
for name, content in spec.env_files.items():
path = os.path.join(env_dir, name)
try:
path = contained_path(env_dir, name)
except ValueError as e:
logging.error("case: skipped an env file: %s", e)
continue
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(content)
Expand Down
12 changes: 11 additions & 1 deletion evalbench/container/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
import re
import yaml

from util.safepath import contained_path

# Where the spec is mounted inside the case container.
CASE_DIR = "/etc/evalbench-case"

Expand Down Expand Up @@ -415,8 +417,16 @@ def _collect_env_files(scenario: dict, session_dir: Optional[str]) -> dict[str,
return {}

collected: dict[str, str] = {}
env_dir = os.path.join(session_dir, "env_files")
for name in declared:
src = os.path.join(session_dir, "env_files", name)
# The scenario comes from the dataset, so the name is untrusted.
try:
src = contained_path(env_dir, name)
except ValueError as e:
logging.warning(
"Skipped an env file of containerized case %s: %s",
scenario.get("id"), e)
continue
if not os.path.isfile(src):
logging.warning(
"Declared env file not found for containerized case %s: %s",
Expand Down
7 changes: 4 additions & 3 deletions evalbench/databases/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
DatabaseSchema,
)
from util.rate_limit import rate_limit, ResourceExhaustedError
from util.safepath import contained_path
from typing import Any, List, Optional, Tuple

DROP_TABLE_SQL = "DROP TABLE {TABLE};"
Expand Down Expand Up @@ -229,7 +230,7 @@ def drop_tmp_database(self, database_name: str):
logging.error(f"Could not delete database: {error}")

def ensure_database_exists(self, database_name: str) -> None:
filename = f"{self.db_path}/{database_name}{self.extension}"
filename = self._get_connection_path(self.db_path, database_name)
os.makedirs(self.db_path, exist_ok=True)
conn = sqlite3.connect(filename)
conn.close()
Expand Down Expand Up @@ -294,5 +295,5 @@ def _get_connection_path(self, db_path, db_name):
if not db_name.endswith(extension):
db_name = db_name + extension

connection_path = os.path.join(db_path, f"{db_name}")
return connection_path
# The name can come from the dataset. Keep the file in db_path.
return contained_path(db_path, db_name)
8 changes: 7 additions & 1 deletion evalbench/scorers/sqlite_bridge.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
"""SQLite Ground Truth Resolution Adapter for EvalBench."""

import logging
import os
import sqlite3

import pandas as pd

from databases.util import unique_column_names
from util.safepath import contained_path


def get_sqlite_ground_truth(
Expand All @@ -15,7 +17,11 @@ def get_sqlite_ground_truth(
) -> list:
"""Resolves candidate SQLite database files and executes query."""

sqlite_path = os.path.join(db_dir, f"{database}.sqlite")
try:
sqlite_path = contained_path(db_dir, f"{database}.sqlite")
except ValueError as e:
logging.warning("Skipped the SQLite ground truth: %s", e)
return []
if not os.path.exists(sqlite_path):
return []
conn = sqlite3.connect(sqlite_path)
Expand Down
136 changes: 136 additions & 0 deletions evalbench/test/safepath_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
"""Tests for file names that come from a dataset or a client."""

import os
import sqlite3
import tempfile
import unittest
from types import SimpleNamespace

from container import case_runner
from container.spec import _collect_env_files
from databases.sqlite import SQLiteDB
from scorers.sqlite_bridge import get_sqlite_ground_truth
from util.safepath import contained_path


class _TempDirTestCase(unittest.TestCase):

def setUp(self):
temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(temp_dir.cleanup)
self.tmp = temp_dir.name
self.base = os.path.join(self.tmp, "base")
self.outside = os.path.join(self.tmp, "outside")
os.makedirs(self.base)
os.makedirs(self.outside)

def _files_outside(self):
return sorted(os.listdir(self.outside))


class ContainedPathTest(_TempDirTestCase):

def test_returns_the_joined_path(self):
for name in ("a.txt", "sub/a.txt", "sub/../a.txt"):
self.assertEqual(
contained_path(self.base, name), os.path.join(self.base, name))

def test_rejects_names_that_leave_the_base(self):
names = [
"", ".", "..", "../outside/a.txt", "sub/../../outside/a.txt",
os.path.join(self.outside, "a.txt"), "a\x00.txt", None, 5,
]
for name in names:
with self.assertRaises(ValueError, msg=repr(name)):
contained_path(self.base, name)

def test_rejects_a_symlink_that_leaves_the_base(self):
os.symlink(self.outside, os.path.join(self.base, "link"))

with self.assertRaises(ValueError):
contained_path(self.base, "link/a.txt")


class ContainerEnvFilesTest(_TempDirTestCase):

def setUp(self):
super().setUp()
self.env_dir = os.path.join(self.base, "env_files")
os.makedirs(os.path.join(self.env_dir, "env"))
with open(os.path.join(self.env_dir, "env", "ok.py"), "w") as f:
f.write("ok")
self.host_file = os.path.join(self.outside, "host.txt")
with open(self.host_file, "w") as f:
f.write("HOST_FILE")

def test_collect_skips_files_outside_the_env_files_directory(self):
scenario = {
"id": "s",
"env_files": ["../../outside/host.txt", self.host_file, "env/ok.py"],
}

self.assertEqual(
_collect_env_files(scenario, self.base), {"env/ok.py": "ok"})

def test_materialize_skips_files_outside_the_env_files_directory(self):
spec = SimpleNamespace(env_files={
"../../outside/written.txt": "x",
os.path.join(self.outside, "abs.txt"): "x",
"env/new.py": "new",
})

case_runner._materialize_env_files(spec, self.base)

self.assertEqual(self._files_outside(), ["host.txt"])
with open(os.path.join(self.env_dir, "env", "new.py")) as f:
self.assertEqual(f.read(), "new")


class SqliteDatabaseNameTest(_TempDirTestCase):

def _db(self, database_name):
return SQLiteDB({
"db_type": "sqlite",
"database_path": self.base,
"database_name": database_name,
"extension": ".sqlite",
"max_executions_per_minute": 10,
})

def test_connects_to_a_database_in_the_database_path(self):
db = self._db("good")

with db.engine.connect():
pass

self.assertEqual(os.listdir(self.base), ["good.sqlite"])

def test_rejects_a_database_name_outside_the_database_path(self):
for name in ("../outside/evil", os.path.join(self.outside, "evil")):
db = self._db(name)

with self.assertRaises(ValueError):
with db.engine.connect():
pass
with self.assertRaises(ValueError):
db.ensure_database_exists(name)
with self.assertRaises(RuntimeError):
db.create_tmp_database(name)

self.assertEqual(self._files_outside(), [])

def test_ground_truth_skips_a_database_outside_the_directory(self):
path = os.path.join(self.outside, "other.sqlite")
conn = sqlite3.connect(path)
conn.execute("CREATE TABLE t (id INT)")
conn.execute("INSERT INTO t VALUES (1)")
conn.commit()
conn.close()

for name in ("../outside/other", os.path.join(self.outside, "other")):
self.assertEqual(
get_sqlite_ground_truth("SELECT * FROM t", name, self.base), [])


if __name__ == "__main__":
unittest.main()
25 changes: 25 additions & 0 deletions evalbench/util/safepath.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
"""Path checks for file names that come from a dataset or a client."""

import os


def contained_path(base: str, name: str) -> str:
"""Returns base/name. Raises ValueError if the path is outside base.

realpath resolves "../" and symlinks before the containment check.
"""
if (
not isinstance(name, str)
or not name
or "\x00" in name
or os.path.isabs(name)
):
raise ValueError(f"Invalid file name: {name!r}")
real_base = os.path.realpath(base)
real_path = os.path.realpath(os.path.join(real_base, name))
if (
real_path == real_base
or os.path.commonpath([real_base, real_path]) != real_base
):
raise ValueError(f"File name is outside {base!r}: {name!r}")
return os.path.join(base, name)
Loading