Skip to content

MAINT: Restore unit test type check target #2090

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
biefan wants to merge 1 commit into
base: main
Choose a base branch
from
+56 −39
Open

MAINT: Restore unit test type check target #2090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,23 @@ include = ["pyrit/prompt_target/hugging_face/**"]
[tool.ty.overrides.rules]
invalid-argument-type = "ignore"

# Unit tests intentionally exercise runtime validation failures, mutable test doubles,
# and optional fields after fixtures have populated them. Keep library code strict while
# avoiding hundreds of per-assertion ignores in tests.
[[tool.ty.overrides]]
include = ["tests/unit/**"]
[tool.ty.overrides.rules]
call-non-callable = "ignore"
invalid-assignment = "ignore"
invalid-await = "ignore"
invalid-parameter-default = "ignore"
invalid-type-arguments = "ignore"
invalid-yield = "ignore"
missing-argument = "ignore"
not-subscriptable = "ignore"
unknown-argument = "ignore"
unsupported-operator = "ignore"

[tool.uv]
constraint-dependencies = [
"aiohttp>=3.13.4",
Expand Down
78 changes: 39 additions & 39 deletions tests/unit/score/test_scorer_eval_csv_schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,28 @@
VALID_DATA_TYPES = set(PromptDataType.__args__) # type: ignore[attr-defined]

# Collect all CSV paths once for cross-cutting tests
ALL_CSV_FILES = (
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))
+ list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))
+ list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv"))
)
OBJECTIVE_CSV_FILES = list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))
HARM_CSV_FILES = list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))
REFUSAL_CSV_FILES = list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv"))
ALL_CSV_FILES = OBJECTIVE_CSV_FILES + HARM_CSV_FILES + REFUSAL_CSV_FILES


@pytest.fixture(scope="class")
def objective_csv_files() -> list[Path]:
"""Get all CSV files in the objective scorer evals directory."""
return OBJECTIVE_CSV_FILES


@pytest.fixture(scope="class")
def harm_csv_files() -> list[Path]:
"""Get all CSV files in the harm scorer evals directory."""
return HARM_CSV_FILES


@pytest.fixture(scope="class")
def refusal_csv_files() -> list[Path]:
"""Get all CSV files in the refusal scorer evals directory."""
return REFUSAL_CSV_FILES


def _skip_comment_lines(f) -> None:
Expand Down Expand Up @@ -89,18 +106,13 @@ def _read_csv_as_dataframe(csv_file: Path) -> pd.DataFrame:
class TestObjectiveScorerEvalCSVSchema:
"""Test that all objective scorer evaluation CSVs have the correct schema."""

@pytest.fixture(scope="class")
def objective_csv_files(self) -> list[Path]:
"""Get all CSV files in the objective scorer evals directory."""
return list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))

def test_objective_csv_files_exist(self, objective_csv_files: list[Path]) -> None:
"""Verify that objective CSV files exist."""
assert len(objective_csv_files) > 0, "No objective CSV files found"

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
OBJECTIVE_CSV_FILES,
ids=lambda p: p.name,
)
def test_objective_csv_has_required_columns(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -133,7 +145,7 @@ def test_objective_csv_has_required_columns(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
OBJECTIVE_CSV_FILES,
ids=lambda p: p.name,
)
def test_objective_csv_column_names_exact(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -164,7 +176,7 @@ def test_objective_csv_column_names_exact(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
OBJECTIVE_CSV_FILES,
ids=lambda p: p.name,
)
def test_objective_csv_scores_are_binary(self, csv_file: Path) -> None:
Expand All @@ -187,7 +199,7 @@ def test_objective_csv_scores_are_binary(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
OBJECTIVE_CSV_FILES,
ids=lambda p: p.name,
)
def test_objective_csv_loads_via_from_csv(self, csv_file: Path) -> None:
Expand All @@ -213,18 +225,13 @@ def test_objective_csv_loads_via_from_csv(self, csv_file: Path) -> None:
class TestHarmScorerEvalCSVSchema:
"""Test that all harm scorer evaluation CSVs have the correct schema."""

@pytest.fixture(scope="class")
def harm_csv_files(self) -> list[Path]:
"""Get all CSV files in the harm scorer evals directory."""
return list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))

def test_harm_csv_files_exist(self, harm_csv_files: list[Path]) -> None:
"""Verify that harm CSV files exist."""
assert len(harm_csv_files) > 0, "No harm CSV files found"

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_csv_has_required_columns(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -269,7 +276,7 @@ def test_harm_csv_has_required_columns(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_csv_has_human_score_columns(self, csv_file: Path) -> None:
Expand All @@ -296,7 +303,7 @@ def test_harm_csv_has_human_score_columns(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_csv_has_harm_definition(self, csv_file: Path) -> None:
Expand All @@ -317,7 +324,7 @@ def test_harm_csv_has_harm_definition(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_definition_file_exists_and_is_valid(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -365,7 +372,7 @@ def test_harm_definition_file_exists_and_is_valid(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_csv_scores_in_valid_range(self, csv_file: Path) -> None:
Expand All @@ -387,7 +394,7 @@ def test_harm_csv_scores_in_valid_range(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_csv_single_harm_category(self, csv_file: Path) -> None:
Expand All @@ -409,7 +416,7 @@ def test_harm_csv_single_harm_category(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_csv_definition_version_matches_yaml(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -459,7 +466,7 @@ def test_harm_csv_definition_version_matches_yaml(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
HARM_CSV_FILES,
ids=lambda p: p.name,
)
def test_harm_csv_loads_via_from_csv(self, csv_file: Path) -> None:
Expand All @@ -485,18 +492,13 @@ def test_harm_csv_loads_via_from_csv(self, csv_file: Path) -> None:
class TestRefusalScorerEvalCSVSchema:
"""Test that all refusal scorer evaluation CSVs have the correct schema."""

@pytest.fixture(scope="class")
def refusal_csv_files(self) -> list[Path]:
"""Get all CSV files in the refusal scorer evals directory."""
return list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv"))

def test_refusal_csv_files_exist(self, refusal_csv_files: list[Path]) -> None:
"""Verify that refusal CSV files exist."""
assert len(refusal_csv_files) > 0, "No refusal CSV files found"

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
REFUSAL_CSV_FILES,
ids=lambda p: p.name,
)
def test_refusal_csv_has_required_columns(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -529,7 +531,7 @@ def test_refusal_csv_has_required_columns(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
REFUSAL_CSV_FILES,
ids=lambda p: p.name,
)
def test_refusal_csv_column_names_exact(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -558,7 +560,7 @@ def test_refusal_csv_column_names_exact(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
REFUSAL_CSV_FILES,
ids=lambda p: p.name,
)
def test_refusal_csv_scores_are_binary(self, csv_file: Path) -> None:
Expand All @@ -581,7 +583,7 @@ def test_refusal_csv_scores_are_binary(self, csv_file: Path) -> None:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
REFUSAL_CSV_FILES,
ids=lambda p: p.name,
)
def test_refusal_csv_loads_via_from_csv(self, csv_file: Path) -> None:
Expand Down Expand Up @@ -614,9 +616,7 @@ class TestCSVVersionMetadata:

@pytest.mark.parametrize(
"csv_file",
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))
+ list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))
+ list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
ALL_CSV_FILES,
ids=lambda p: f"{p.parent.name}/{p.name}",
)
def test_csv_has_dataset_version_line(self, csv_file: Path) -> None:
Expand Down