Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion evalbench/reporting/gcs_artifact.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import stat
import tempfile
import zipfile
import logging
Expand Down Expand Up @@ -102,6 +103,18 @@ def store(self, results: pd.DataFrame, type: STORETYPE) -> None:
eval_id = rows["eval_id"].iloc[0]
self._zip_and_upload(fake_home, eval_id, bucket)

@staticmethod
def _is_regular_file(path: str) -> bool:
"""Returns True for a regular file. A symlink is not a regular file.

The agent under test writes the sandbox. A symlink there can point to
a host file, and zipfile copies the target. A FIFO blocks the read.
"""
try:
return stat.S_ISREG(os.lstat(path).st_mode)
except OSError:
return False

def _zip_and_upload(
self, src_dir: str, eval_id: str, bucket: storage.Bucket
) -> None:
Expand All @@ -128,8 +141,12 @@ def _zip_and_upload(
) as tmp_file:
zip_path = tmp_file.name

skipped = 0
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf:
for root, dirs, files in os.walk(src_dir):
# os.walk does not enter a symlinked directory, and
# _is_regular_file rejects a symlinked file. Thus each file
# in the zip is inside src_dir.
for root, dirs, files in os.walk(src_dir, followlinks=False):
# Exclude hidden directories and common heavy/cache directories
dirs[:] = [
d
Expand All @@ -142,9 +159,20 @@ def _zip_and_upload(
if file.startswith("."):
continue
file_path = os.path.join(root, file)
if not self._is_regular_file(file_path):
skipped += 1
continue
arcname = os.path.relpath(file_path, src_dir)
zipf.write(file_path, arcname)

if skipped:
logging.warning(
"GcsReporter._zip_and_upload: skipped %d symlinks or "
"special files in %s.",
skipped,
src_dir,
)

blob_name = f"{self.path_prefix}/{self.job_id}/{eval_id}.zip"
logging.info(
"GcsReporter._zip_and_upload: Zip created. Size=%d bytes. Uploading to gs://%s/%s ...",
Expand Down
59 changes: 59 additions & 0 deletions evalbench/test/gcs_artifact_test.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import os
import tempfile
import unittest
import zipfile
from unittest.mock import patch, MagicMock
import pandas as pd
import datetime
Expand Down Expand Up @@ -111,6 +114,62 @@ def test_store_shared(self, mock_remove, mock_tempfile, mock_zipfile, mock_exist
mock_bucket.blob.assert_called_once_with("results/test-job-id/fake_home.zip")
mock_blob.upload_from_filename.assert_called_once_with("/tmp/temp.zip")

def _zip_names_and_data(self, src_dir):
"""Runs _zip_and_upload and returns the names and bytes in the zip."""
captured = {}

def read_zip(zip_path):
with zipfile.ZipFile(zip_path) as zipf:
captured["names"] = sorted(zipf.namelist())
captured["data"] = b"".join(
zipf.read(name) for name in zipf.namelist())

bucket = MagicMock()
bucket.blob.return_value.upload_from_filename.side_effect = read_zip
reporter = GcsReporter(self.reporting_config, self.job_id, self.run_time)
reporter._zip_and_upload(src_dir, "eval1", bucket)
return captured["names"], captured["data"]

def _make_sandbox(self):
temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(temp_dir.cleanup)
fake_home = os.path.join(temp_dir.name, "fake_home")
os.makedirs(os.path.join(fake_home, "sub"))
for name in ("note.txt", os.path.join("sub", "kept.txt")):
with open(os.path.join(fake_home, name), "w") as f:
f.write("sandbox file")
return temp_dir.name, fake_home

@patch('reporting.gcs_artifact.storage.Client')
def test_zip_does_not_follow_symlinks(self, mock_storage_client):
"""A symlink in the sandbox must not copy a host file into the zip."""
host_dir, fake_home = self._make_sandbox()
host_file = os.path.join(host_dir, "host_secret.txt")
with open(host_file, "w") as f:
f.write("HOST_SECRET")
host_subdir = os.path.join(host_dir, "host_subdir")
os.makedirs(host_subdir)
with open(os.path.join(host_subdir, "inner.txt"), "w") as f:
f.write("HOST_SECRET")
os.symlink(host_file, os.path.join(fake_home, "link_to_file.txt"))
os.symlink(host_file, os.path.join(fake_home, "sub", "nested_link.txt"))
os.symlink(host_subdir, os.path.join(fake_home, "link_to_dir"))

names, data = self._zip_names_and_data(fake_home)

self.assertEqual(names, ["note.txt", "sub/kept.txt"])
self.assertNotIn(b"HOST_SECRET", data)

@patch('reporting.gcs_artifact.storage.Client')
def test_zip_skips_special_files(self, mock_storage_client):
"""A FIFO in the sandbox must not block the reporter."""
_, fake_home = self._make_sandbox()
os.mkfifo(os.path.join(fake_home, "pipe"))

names, _ = self._zip_names_and_data(fake_home)

self.assertEqual(names, ["note.txt", "sub/kept.txt"])


if __name__ == '__main__':
unittest.main()
Loading