From 569f6493f72cf64484e8b39c19b41da7e0ac6434 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Fri, 2 Oct 2026 13:31:26 +0200 Subject: [PATCH 1/2] Move pure filesystem, project, and scoring helpers into bcbench-core First code to move into the library. These helpers have no dependency on BC-Bench configuration or policy, so they move with minimal changes and the app imports them directly from bcbench_core (no re-export shims). - bcbench_core.filesystem: remove_tree, clear_directory, prepare_run_dir - bcbench_core.projects: categorize_projects; the AL test-project identifiers become DEFAULT_TEST_PROJECT_IDENTIFIERS instead of a constant app config field, and collection uses the same constant - bcbench_core.scoring: precision/recall, F-beta, pass@k, pass^k; bootstrap_ci stays in the app because it needs numpy and scipy - Core tests live in packages/bcbench-core/tests with their own pytest configuration (importlib import mode, strict) and run as a separate CI step Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3fa47b3c-f5d6-4c36-9be3-c4fdf2bebe47 --- .github/workflows/CI.yml | 3 + CONTRIBUTING.md | 3 +- notebooks/archive/multi-run.ipynb | 3 +- notebooks/bug-fix/altool-comparison.ipynb | 3 +- .../test-generation/altest-comparison.ipynb | 3 +- notebooks/utils.py | 3 +- packages/bcbench-core/README.md | 1 + packages/bcbench-core/pyproject.toml | 11 ++ .../src/bcbench_core/filesystem.py | 2 +- .../bcbench-core/src/bcbench_core/projects.py | 14 +- .../bcbench-core/src/bcbench_core/scoring.py | 78 +++++++++++ .../bcbench-core/tests/test_filesystem.py | 55 ++++++++ .../bcbench-core/tests/test_projects.py | 2 +- packages/bcbench-core/tests/test_scoring.py | 130 ++++++++++++++++++ src/bcbench/collection/collect_gh.py | 5 +- src/bcbench/collection/patch_utils.py | 3 +- src/bcbench/commands/contamination.py | 2 +- src/bcbench/commands/evaluate.py | 2 +- src/bcbench/config.py | 2 - src/bcbench/evaluate/bugfix.py | 3 +- .../evaluate/codereview_judge_calibration.py | 2 +- src/bcbench/evaluate/dataquery.py | 3 +- src/bcbench/evaluate/nl2al.py | 4 +- src/bcbench/evaluate/testgeneration.py | 2 +- src/bcbench/operations/__init__.py | 6 - src/bcbench/operations/bc_operations.py | 2 +- src/bcbench/operations/git_operations.py | 3 +- src/bcbench/results/__init__.py | 5 +- src/bcbench/results/codereview.py | 2 +- src/bcbench/results/leaderboard.py | 3 +- src/bcbench/results/metrics.py | 77 ----------- tests/test_filesystem_operations.py | 25 ---- tests/test_metrics.py | 107 +------------- uv.lock | 10 ++ 34 files changed, 329 insertions(+), 250 deletions(-) rename src/bcbench/operations/filesystem_operations.py => packages/bcbench-core/src/bcbench_core/filesystem.py (94%) rename src/bcbench/operations/project_operations.py => packages/bcbench-core/src/bcbench_core/projects.py (80%) create mode 100644 packages/bcbench-core/src/bcbench_core/scoring.py create mode 100644 packages/bcbench-core/tests/test_filesystem.py rename tests/test_project_categorization.py => packages/bcbench-core/tests/test_projects.py (98%) create mode 100644 packages/bcbench-core/tests/test_scoring.py delete mode 100644 tests/test_filesystem_operations.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6fbb1d22c..fcb45b6f4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -39,6 +39,9 @@ jobs: - name: Run tests with coverage run: uv run pytest --cov=src/bcbench --cov-report=term-missing + - name: Run bcbench-core tests with coverage + run: uv run pytest packages/bcbench-core --cov=bcbench_core --cov-report=term-missing + e2e: strategy: fail-fast: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5507736d9..e50bb4e64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,8 +61,9 @@ uv run bcbench --help # This is very fast, give it a go and see it live! uv run bcbench run copilot microsoft__BCApps-5633 --category bug-fix --repo-path /path/to/BCApps -# Run tests +# Run tests (bcbench, then bcbench-core with its own pytest configuration) uv run pytest --cov=src/bcbench --cov-report=term-missing +uv run pytest packages/bcbench-core --cov=bcbench_core --cov-report=term-missing # Lint and format uv run pre-commit run --all-files diff --git a/notebooks/archive/multi-run.ipynb b/notebooks/archive/multi-run.ipynb index bde6d78a6..3614ad87b 100644 --- a/notebooks/archive/multi-run.ipynb +++ b/notebooks/archive/multi-run.ipynb @@ -1126,10 +1126,9 @@ ], "source": [ "import plotly.graph_objects as go\n", + "from bcbench_core.scoring import pass_at_k, pass_hat_k\n", "from plotly.subplots import make_subplots\n", "\n", - "from bcbench.results.metrics import pass_at_k, pass_hat_k\n", - "\n", "# Visualize: pass^k vs pass@k comparison using correct formulas\n", "run_ids_sorted = sorted(analysis_model_df[\"run_id\"].unique())\n", "pivot = analysis_model_df.pivot_table(index=\"instance_id\", columns=\"run_id\", values=\"resolved\", aggfunc=lambda x: x.iloc[0])\n", diff --git a/notebooks/bug-fix/altool-comparison.ipynb b/notebooks/bug-fix/altool-comparison.ipynb index 235656f9b..455c319bd 100644 --- a/notebooks/bug-fix/altool-comparison.ipynb +++ b/notebooks/bug-fix/altool-comparison.ipynb @@ -46,8 +46,9 @@ "source": [ "import numpy as np\n", "import plotly.graph_objects as go\n", + "from bcbench_core.scoring import pass_at_k, pass_hat_k\n", "\n", - "from bcbench.results.metrics import bootstrap_ci, pass_at_k, pass_hat_k" + "from bcbench.results.metrics import bootstrap_ci" ] }, { diff --git a/notebooks/test-generation/altest-comparison.ipynb b/notebooks/test-generation/altest-comparison.ipynb index e1ced1c63..244f3fbfe 100644 --- a/notebooks/test-generation/altest-comparison.ipynb +++ b/notebooks/test-generation/altest-comparison.ipynb @@ -37,8 +37,9 @@ "source": [ "import numpy as np\n", "import plotly.graph_objects as go\n", + "from bcbench_core.scoring import pass_at_k, pass_hat_k\n", "\n", - "from bcbench.results.metrics import bootstrap_ci, pass_at_k, pass_hat_k" + "from bcbench.results.metrics import bootstrap_ci" ] }, { diff --git a/notebooks/utils.py b/notebooks/utils.py index 9866fdd6d..870fcc84c 100644 --- a/notebooks/utils.py +++ b/notebooks/utils.py @@ -3,10 +3,9 @@ from typing import Literal, TypedDict import pandas as pd +from bcbench_core.scoring import pass_at_k, pass_hat_k from unidiff import PatchSet -from bcbench.results.metrics import pass_at_k, pass_hat_k - # Root paths - all notebooks should use these NOTEBOOKS_ROOT = Path(__file__).parent DATASET_PATH = NOTEBOOKS_ROOT.parent / "dataset" / "bcbench.jsonl" diff --git a/packages/bcbench-core/README.md b/packages/bcbench-core/README.md index 770ec7672..75fd84817 100644 --- a/packages/bcbench-core/README.md +++ b/packages/bcbench-core/README.md @@ -23,5 +23,6 @@ The package is a [uv workspace](https://docs.astral.sh/uv/concepts/projects/work uv sync --all-groups uv run ruff check packages/bcbench-core uv check --package bcbench-core --preview-features check-command +uv run pytest packages/bcbench-core uv build --package bcbench-core ``` diff --git a/packages/bcbench-core/pyproject.toml b/packages/bcbench-core/pyproject.toml index 6b60943aa..d4733387c 100644 --- a/packages/bcbench-core/pyproject.toml +++ b/packages/bcbench-core/pyproject.toml @@ -20,6 +20,14 @@ classifiers = [ ] dependencies = [] +[dependency-groups] +dev = ["pytest>=9.0.3"] + +[tool.pytest] +testpaths = ["tests"] +addopts = ["--import-mode=importlib"] +strict = true + [tool.ruff] target-version = "py313" line-length = 200 @@ -73,6 +81,9 @@ ignore = [ [tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["ANN"] + [tool.ruff.lint.flake8-tidy-imports.banned-api] "bcbench".msg = "bcbench-core must not depend on the BC-Bench application." "os.environ".msg = "bcbench-core must not read the environment; accept values as parameters." diff --git a/src/bcbench/operations/filesystem_operations.py b/packages/bcbench-core/src/bcbench_core/filesystem.py similarity index 94% rename from src/bcbench/operations/filesystem_operations.py rename to packages/bcbench-core/src/bcbench_core/filesystem.py index 661f4ab50..b2acc58a3 100644 --- a/src/bcbench/operations/filesystem_operations.py +++ b/packages/bcbench-core/src/bcbench_core/filesystem.py @@ -1,4 +1,4 @@ -"""Filesystem operations.""" +"""Filesystem helpers that tolerate read-only files on Windows.""" import shutil import stat diff --git a/src/bcbench/operations/project_operations.py b/packages/bcbench-core/src/bcbench_core/projects.py similarity index 80% rename from src/bcbench/operations/project_operations.py rename to packages/bcbench-core/src/bcbench_core/projects.py index 5977ce4b7..2f438be0c 100644 --- a/src/bcbench/operations/project_operations.py +++ b/packages/bcbench-core/src/bcbench_core/projects.py @@ -1,10 +1,10 @@ -"""Project path categorization and management operations.""" +"""Project path categorization for AL repositories.""" -from bcbench.config import get_config -from bcbench.logger import get_logger +import logging -logger = get_logger(__name__) -_config = get_config() +logger = logging.getLogger(__name__) + +DEFAULT_TEST_PROJECT_IDENTIFIERS: tuple[str, ...] = ("test", "tests") def _is_test_project(project_path: str, test_identifiers: tuple[str, ...]) -> bool: @@ -25,11 +25,12 @@ def _is_test_project(project_path: str, test_identifiers: tuple[str, ...]) -> bo return any(f"/{identifier}" in project_lower or f"\\{identifier}" in project_lower for identifier in test_identifiers) -def categorize_projects(project_paths: list[str]) -> tuple[list[str], list[str]]: +def categorize_projects(project_paths: list[str], test_identifiers: tuple[str, ...] = DEFAULT_TEST_PROJECT_IDENTIFIERS) -> tuple[list[str], list[str]]: """Categorize project paths into test projects and application projects. Args: project_paths: List of project paths to categorize + test_identifiers: Path components that mark a test project Returns: Tuple of (test_projects, app_projects) @@ -37,7 +38,6 @@ def categorize_projects(project_paths: list[str]) -> tuple[list[str], list[str]] Raises: RuntimeError: If project categorization fails (no test or app projects found) """ - test_identifiers = _config.file_patterns.test_project_identifiers test_projects: list[str] = [project for project in project_paths if _is_test_project(project, test_identifiers)] app_projects: list[str] = [project for project in project_paths if project not in test_projects] diff --git a/packages/bcbench-core/src/bcbench_core/scoring.py b/packages/bcbench-core/src/bcbench_core/scoring.py new file mode 100644 index 000000000..0b4d1db7f --- /dev/null +++ b/packages/bcbench-core/src/bcbench_core/scoring.py @@ -0,0 +1,78 @@ +"""Scoring functions for evaluation results.""" + +import math + + +def precision_recall(matched_count: int, generated_count: int, expected_count: int) -> tuple[float, float]: + """Precision and recall for a set-matching task. + + An empty generated or expected set yields perfect precision or recall respectively, + so a model that correctly produces no output is not penalized. + """ + precision = matched_count / generated_count if generated_count else 1.0 + recall = matched_count / expected_count if expected_count else 1.0 + return precision, recall + + +def f_beta_score(precision: float, recall: float, beta: float = 1.0) -> float: + if precision + recall == 0: + return 0.0 + beta_sq = beta**2 + return (1 + beta_sq) * precision * recall / (beta_sq * precision + recall) + + +def f1_score(precision: float, recall: float) -> float: + return f_beta_score(precision, recall, beta=1.0) + + +def pass_hat_k(num_trials: int, success_count: int, k: int) -> float: + """Measures the probability that all k trials succeed + + Formula: C(success_count, k) / C(num_trials, k) + + Reference: https://arxiv.org/pdf/2406.12045 + + Args: + num_trials: The number of trials (n). + success_count: The number of successful trials. + k: The number of trials to consider. + + Returns: + The pass^k metric (0.0 to 1.0). + + Raises: + ValueError: If num_trials < k. + """ + if num_trials < k: + raise ValueError(f"Number of trials {num_trials} is less than k {k}.") + return math.comb(success_count, k) / math.comb(num_trials, k) + + +def pass_at_k(num_samples: int, num_correct: int, k: int) -> float: + """Measures the likelihood that an agent gets at least one correct solution in k attempts + + Formula: 1 - C(n-c, k) / C(n, k) + + Reference: https://github.com/openai/human-eval-infilling/blob/main/human_eval_infilling/evaluation.py + + Args: + num_samples: Total number of samples (n). + num_correct: Number of correct samples (c). + k: Number of samples to draw. + + Returns: + The pass@k probability (0.0 to 1.0). + + Raises: + ValueError: If num_samples < k. + """ + if num_samples < k: + raise ValueError(f"Number of samples {num_samples} is less than k {k}.") + if num_samples - num_correct < k: + return 1.0 + # Use product formulation to avoid large combinatorial numbers + # 1 - prod_{i=n-c+1}^{n} (1 - k/i) + result = 1.0 + for i in range(num_samples - num_correct + 1, num_samples + 1): + result *= 1.0 - k / i + return 1.0 - result diff --git a/packages/bcbench-core/tests/test_filesystem.py b/packages/bcbench-core/tests/test_filesystem.py new file mode 100644 index 000000000..345fcb540 --- /dev/null +++ b/packages/bcbench-core/tests/test_filesystem.py @@ -0,0 +1,55 @@ +import stat + +from bcbench_core.filesystem import clear_directory, prepare_run_dir, remove_tree + + +def test_remove_tree_removes_read_only_files(tmp_path): + tree = tmp_path / "tree" + (tree / "nested").mkdir(parents=True) + read_only_file = tree / "nested" / "read-only.txt" + read_only_file.write_text("content", encoding="utf-8") + read_only_file.chmod(stat.S_IREAD) + + remove_tree(tree) + + assert not tree.exists() + + +def test_prepare_run_dir_replaces_existing_run(tmp_path): + stale_file = tmp_path / "run-1" / "stale.jsonl" + stale_file.parent.mkdir() + stale_file.write_text("{}", encoding="utf-8") + + run_dir = prepare_run_dir(tmp_path, "run-1") + + assert run_dir == tmp_path / "run-1" + assert run_dir.is_dir() + assert list(run_dir.iterdir()) == [] + + +def test_prepare_run_dir_creates_missing_parents(tmp_path): + run_dir = prepare_run_dir(tmp_path / "results", "run-2") + + assert run_dir.is_dir() + + +def test_clear_directory_removes_contents_and_preserves_directory(tmp_path): + nested_directory = tmp_path / "nested" + nested_directory.mkdir() + (nested_directory / "nested.txt").write_text("nested", encoding="utf-8") + read_only_file = tmp_path / "read-only.txt" + read_only_file.write_text("content", encoding="utf-8") + read_only_file.chmod(stat.S_IREAD) + + clear_directory(tmp_path) + + assert tmp_path.is_dir() + assert list(tmp_path.iterdir()) == [] + + +def test_clear_directory_creates_missing_directory(tmp_path): + missing_directory = tmp_path / "missing" + + clear_directory(missing_directory) + + assert missing_directory.is_dir() diff --git a/tests/test_project_categorization.py b/packages/bcbench-core/tests/test_projects.py similarity index 98% rename from tests/test_project_categorization.py rename to packages/bcbench-core/tests/test_projects.py index 04f0a7564..a943719b6 100644 --- a/tests/test_project_categorization.py +++ b/packages/bcbench-core/tests/test_projects.py @@ -2,7 +2,7 @@ import pytest -from bcbench.operations.project_operations import _is_test_project, categorize_projects +from bcbench_core.projects import _is_test_project, categorize_projects class TestIsTestProject: diff --git a/packages/bcbench-core/tests/test_scoring.py b/packages/bcbench-core/tests/test_scoring.py new file mode 100644 index 000000000..0855d88cc --- /dev/null +++ b/packages/bcbench-core/tests/test_scoring.py @@ -0,0 +1,130 @@ +import pytest + +from bcbench_core.scoring import f1_score, f_beta_score, pass_at_k, pass_hat_k, precision_recall + + +class TestPrecisionRecall: + def test_partial_match(self): + assert precision_recall(matched_count=1, generated_count=4, expected_count=2) == (0.25, 0.5) + + def test_empty_generated_set_has_perfect_precision(self): + assert precision_recall(matched_count=0, generated_count=0, expected_count=3) == (1.0, 0.0) + + def test_empty_expected_set_has_perfect_recall(self): + assert precision_recall(matched_count=0, generated_count=2, expected_count=0) == (0.0, 1.0) + + +class TestFBetaScore: + def test_zero_precision_and_recall(self): + assert f_beta_score(0.0, 0.0, beta=2.0) == 0.0 + + def test_f1_is_harmonic_mean(self): + assert f1_score(0.5, 1.0) == pytest.approx(2 / 3) + + def test_beta_weights_recall(self): + assert f_beta_score(0.5, 1.0, beta=2.0) > f1_score(0.5, 1.0) > f_beta_score(0.5, 1.0, beta=0.5) + + +class TestPassHatK: + def test_all_successes(self): + # C(3,3)/C(3,3) = 1.0 + assert pass_hat_k(num_trials=3, success_count=3, k=3) == 1.0 + + def test_no_successes(self): + # C(0,3)/C(3,3) = 0.0 + assert pass_hat_k(num_trials=3, success_count=0, k=3) == 0.0 + + def test_partial_success_k1(self): + # C(2,1)/C(3,1) = 2/3 + result = pass_hat_k(num_trials=3, success_count=2, k=1) + assert abs(result - 2 / 3) < 1e-9 + + def test_partial_success_k2(self): + # C(2,2)/C(3,2) = 1/3 + result = pass_hat_k(num_trials=3, success_count=2, k=2) + assert abs(result - 1 / 3) < 1e-9 + + def test_insufficient_successes_for_k(self): + # C(1,3)/C(3,3) = 0 (can't choose 3 from 1) + assert pass_hat_k(num_trials=3, success_count=1, k=3) == 0.0 + + def test_raises_when_trials_less_than_k(self): + with pytest.raises(ValueError, match="Number of trials 2 is less than k 3"): + pass_hat_k(num_trials=2, success_count=2, k=3) + + def test_k_equals_1_is_success_rate(self): + # pass^1 should equal success_count / num_trials + assert pass_hat_k(num_trials=10, success_count=7, k=1) == 0.7 + + def test_large_numbers(self): + # With 10 trials and 8 successes, k=5 + # C(8,5)/C(10,5) = 56/252 = 2/9 + result = pass_hat_k(num_trials=10, success_count=8, k=5) + assert abs(result - 56 / 252) < 1e-9 + + +class TestPassAtK: + def test_all_correct(self): + # If all samples are correct, pass@k = 1.0 + assert pass_at_k(num_samples=5, num_correct=5, k=1) == 1.0 + assert pass_at_k(num_samples=5, num_correct=5, k=5) == 1.0 + + def test_none_correct(self): + # If no samples are correct, pass@k = 0.0 + assert pass_at_k(num_samples=5, num_correct=0, k=1) == 0.0 + assert pass_at_k(num_samples=5, num_correct=0, k=3) == 0.0 + + def test_one_correct_k1(self): + # With 1 correct out of 5, pass@1 = 1 - C(4,1)/C(5,1) = 1 - 4/5 = 0.2 + result = pass_at_k(num_samples=5, num_correct=1, k=1) + assert abs(result - 0.2) < 1e-9 + + def test_one_correct_k5(self): + # With 1 correct out of 5, pass@5 = 1 - C(4,5)/C(5,5) = 1 - 0 = 1.0 + result = pass_at_k(num_samples=5, num_correct=1, k=5) + assert result == 1.0 + + def test_half_correct(self): + # With 3 correct out of 6, pass@1 = 1 - C(3,1)/C(6,1) = 1 - 3/6 = 0.5 + result = pass_at_k(num_samples=6, num_correct=3, k=1) + assert abs(result - 0.5) < 1e-9 + + def test_raises_when_samples_less_than_k(self): + with pytest.raises(ValueError, match="Number of samples 2 is less than k 3"): + pass_at_k(num_samples=2, num_correct=1, k=3) + + def test_returns_1_when_enough_correct(self): + # If n - c < k, return 1.0 (guaranteed success) + # 5 samples, 4 correct, k=2: n-c=1 < k=2, so return 1.0 + assert pass_at_k(num_samples=5, num_correct=4, k=2) == 1.0 + + def test_openai_example(self): + # Test case from OpenAI's human-eval + # n=10, c=3, k=1: 1 - C(7,1)/C(10,1) = 1 - 7/10 = 0.3 + result = pass_at_k(num_samples=10, num_correct=3, k=1) + assert abs(result - 0.3) < 1e-9 + + def test_pass_at_k_increases_with_k(self): + # pass@k should increase (or stay same) as k increases + results = [pass_at_k(num_samples=10, num_correct=3, k=k) for k in range(1, 11)] + for i in range(len(results) - 1): + assert results[i] <= results[i + 1] + + +class TestMetricsRelationship: + def test_pass_at_1_equals_pass_hat_1_for_single_trial(self): + # With k=1, both metrics equal success_count/num_trials + for n in range(1, 10): + for c in range(n + 1): + pass_at = pass_at_k(num_samples=n, num_correct=c, k=1) + pass_hat = pass_hat_k(num_trials=n, success_count=c, k=1) + assert abs(pass_at - pass_hat) < 1e-9 + + def test_pass_at_k_geq_pass_hat_k(self): + # pass@k >= pass^k always (optimistic vs pessimistic) + for n in range(1, 8): + for c in range(n + 1): + for k in range(1, n + 1): + pass_at = pass_at_k(num_samples=n, num_correct=c, k=k) + pass_hat = pass_hat_k(num_trials=n, success_count=c, k=k) + assert pass_at >= pass_hat - 1e-9, f"n={n}, c={c}, k={k}: pass@k={pass_at} < pass^k={pass_hat}" diff --git a/src/bcbench/collection/collect_gh.py b/src/bcbench/collection/collect_gh.py index cbc8377b7..003bacda0 100644 --- a/src/bcbench/collection/collect_gh.py +++ b/src/bcbench/collection/collect_gh.py @@ -5,6 +5,7 @@ from typing import Any import typer +from bcbench_core.projects import DEFAULT_TEST_PROJECT_IDENTIFIERS from bcbench.collection.gh_client import GHClient from bcbench.collection.patch_utils import extract_file_paths_from_patch, find_project_paths_from_diff, separate_patches @@ -60,7 +61,7 @@ def fail(reason: str) -> ScreeningResult: raise CollectionError(f"Failed to fetch PR #{pr_number} from {repo}: {exc}") from exc try: - patch, patch_fix, patch_test = separate_patches(diff, _config.file_patterns.test_project_identifiers) + patch, patch_fix, patch_test = separate_patches(diff, DEFAULT_TEST_PROJECT_IDENTIFIERS) project_paths = find_project_paths_from_diff(patch) except CollectionError as exc: raise CollectionError(f"Failed to parse PR diff: {exc}") from exc @@ -103,7 +104,7 @@ def _build_bugfix_entry( diff = gh_client.get_pr_diff(pr_number) - patch, patch_fix, patch_test = separate_patches(diff, _config.file_patterns.test_project_identifiers) + patch, patch_fix, patch_test = separate_patches(diff, DEFAULT_TEST_PROJECT_IDENTIFIERS) # Extract problem statement from PR title = pr_data.get("title", "") diff --git a/src/bcbench/collection/patch_utils.py b/src/bcbench/collection/patch_utils.py index ba384beda..231a2d4d4 100644 --- a/src/bcbench/collection/patch_utils.py +++ b/src/bcbench/collection/patch_utils.py @@ -3,6 +3,7 @@ import subprocess from pathlib import Path +from bcbench_core.projects import DEFAULT_TEST_PROJECT_IDENTIFIERS from unidiff import PatchSet from unidiff.errors import UnidiffParseError @@ -76,7 +77,7 @@ def extract_patches(repo_path: Path, base_commit_id: str, commit_id: str, diff_p if not patch: raise CollectionError("No patch data found between the specified commits") - return separate_patches(patch, _config.file_patterns.test_project_identifiers) + return separate_patches(patch, DEFAULT_TEST_PROJECT_IDENTIFIERS) def find_project_paths_from_diff(patch: str) -> list[str]: diff --git a/src/bcbench/commands/contamination.py b/src/bcbench/commands/contamination.py index a54512e8b..0900cd659 100644 --- a/src/bcbench/commands/contamination.py +++ b/src/bcbench/commands/contamination.py @@ -7,6 +7,7 @@ from typing import Annotated import typer +from bcbench_core.filesystem import prepare_run_dir from bcbench.cli_options import CopilotModel, EvaluationCategoryOption, OutputDir, RunId from bcbench.config import get_config @@ -14,7 +15,6 @@ from bcbench.contamination.runner import load_identification_results, run_filepath_identification from bcbench.dataset import BugFixEntry from bcbench.logger import get_logger -from bcbench.operations import prepare_run_dir from bcbench.types import EvaluationCategory logger = get_logger(__name__) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 2710dfcfb..1be08d95f 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -3,6 +3,7 @@ from typing import Annotated, cast import typer +from bcbench_core.filesystem import prepare_run_dir from bcbench.agent import BCalBackendConfig, get_claude_version, get_copilot_version, get_pr_review_version, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent from bcbench.cli_options import ( @@ -27,7 +28,6 @@ from bcbench.evaluate import AgentRunner, EvaluationPipeline from bcbench.evaluate.codereview_judge_calibration import run_calibration from bcbench.logger import get_logger -from bcbench.operations import prepare_run_dir from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.types import AgentHarness, AgentMetrics, BCalLLMBackend, EvaluationCategory, EvaluationContext, ExperimentConfiguration diff --git a/src/bcbench/config.py b/src/bcbench/config.py index 291a646dc..fc9cdf0a2 100644 --- a/src/bcbench/config.py +++ b/src/bcbench/config.py @@ -110,7 +110,6 @@ class FilePatternConfig: result_pattern: str instruction_source_naming: str instructions_dirname: str - test_project_identifiers: tuple[str, ...] problem_statement_readme: str problem_statement_dest_dir: str alpackages_dirname: str @@ -127,7 +126,6 @@ def default(cls) -> FilePatternConfig: result_pattern=".jsonl", instruction_source_naming="AGENTS.md", instructions_dirname="instructions", - test_project_identifiers=("test", "tests"), problem_statement_readme="README.md", problem_statement_dest_dir="problem", alpackages_dirname=".alpackages", diff --git a/src/bcbench/evaluate/bugfix.py b/src/bcbench/evaluate/bugfix.py index acc1c9838..2c03c4950 100644 --- a/src/bcbench/evaluate/bugfix.py +++ b/src/bcbench/evaluate/bugfix.py @@ -1,5 +1,7 @@ from pathlib import Path +from bcbench_core.projects import categorize_projects + from bcbench.dataset import BugFixEntry from bcbench.evaluate.base import AgentRunner, EvaluationPipeline from bcbench.exceptions import BuildError, TestExecutionError @@ -8,7 +10,6 @@ from bcbench.operations import ( apply_patch, build_and_publish_projects, - categorize_projects, clean_project_paths, copy_problem_statement_folder, run_tests, diff --git a/src/bcbench/evaluate/codereview_judge_calibration.py b/src/bcbench/evaluate/codereview_judge_calibration.py index dbfbf13de..4e2312e60 100644 --- a/src/bcbench/evaluate/codereview_judge_calibration.py +++ b/src/bcbench/evaluate/codereview_judge_calibration.py @@ -14,12 +14,12 @@ from pathlib import Path +from bcbench_core.scoring import precision_recall from pydantic import BaseModel, ConfigDict from bcbench.config import get_config from bcbench.dataset.codereview import ReviewComment from bcbench.evaluate.codereview_judge import judge_verdicts -from bcbench.results.metrics import precision_recall from bcbench.types import JudgeCalibrationReport _config = get_config() diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index a455dbd2d..2a8423d15 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -3,12 +3,13 @@ from decimal import Decimal, InvalidOperation from pathlib import Path +from bcbench_core.filesystem import clear_directory + from bcbench.dataset import DataQueryEntry from bcbench.evaluate.base import EvaluationPipeline from bcbench.exceptions import EmptyGoldResultError from bcbench.github_actions import github_log_group from bcbench.logger import get_logger -from bcbench.operations import clear_directory from bcbench.results.base import ExecutionBasedEvaluationResult from bcbench.types import EvaluationContext diff --git a/src/bcbench/evaluate/nl2al.py b/src/bcbench/evaluate/nl2al.py index ff330a0c2..8288ab8a2 100644 --- a/src/bcbench/evaluate/nl2al.py +++ b/src/bcbench/evaluate/nl2al.py @@ -2,12 +2,14 @@ import subprocess from pathlib import Path +from bcbench_core.filesystem import remove_tree + from bcbench.dataset import NL2ALEntry from bcbench.evaluate.base import AgentRunner, EvaluationPipeline from bcbench.exceptions import EmptyDiffError from bcbench.github_actions import github_log_group from bcbench.logger import get_logger -from bcbench.operations import copy_symbol_apps, remove_tree, stage_and_get_diff +from bcbench.operations import copy_symbol_apps, stage_and_get_diff from bcbench.results.base import JudgeBasedEvaluationResult from bcbench.types import EvaluationContext diff --git a/src/bcbench/evaluate/testgeneration.py b/src/bcbench/evaluate/testgeneration.py index 966db099a..2e129e4bd 100644 --- a/src/bcbench/evaluate/testgeneration.py +++ b/src/bcbench/evaluate/testgeneration.py @@ -1,6 +1,7 @@ from pathlib import Path import yaml +from bcbench_core.projects import categorize_projects from bcbench.collection.patch_utils import extract_file_paths_from_patch from bcbench.config import get_config @@ -12,7 +13,6 @@ from bcbench.operations import ( apply_patch, build_and_publish_projects, - categorize_projects, clean_project_paths, copy_problem_statement_folder, extract_tests_from_patch, diff --git a/src/bcbench/operations/__init__.py b/src/bcbench/operations/__init__.py index 00a67989a..e82f8ce28 100644 --- a/src/bcbench/operations/__init__.py +++ b/src/bcbench/operations/__init__.py @@ -11,7 +11,6 @@ run_tests, wrap_query_as_api, ) -from bcbench.operations.filesystem_operations import clear_directory, prepare_run_dir, remove_tree from bcbench.operations.git_operations import ( apply_patch, checkout_commit, @@ -25,7 +24,6 @@ stage_and_get_diff, ) from bcbench.operations.instruction_operations import copy_problem_statement_folder, setup_custom_agent, setup_instructions_from_config -from bcbench.operations.project_operations import categorize_projects from bcbench.operations.setup_operations import bootstrap_app_json, set_runtime_version, setup_repo_prebuild from bcbench.operations.skills_operations import setup_agent_skills from bcbench.operations.test_operations import extract_tests_from_patch @@ -37,11 +35,9 @@ "build_ps_app_build_and_publish", "build_ps_dataset_tests_script", "build_ps_test_script", - "categorize_projects", "checkout_commit", "clean_project_paths", "clean_repo", - "clear_directory", "clone_repo_at_revision", "commit_changes", "copy_problem_statement_folder", @@ -51,8 +47,6 @@ "fetch_commit_if_missing", "has_changes", "init_repo", - "prepare_run_dir", - "remove_tree", "resolve_artifact_version_root", "run_tests", "set_runtime_version", diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index 74f608d5d..d8253c179 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -6,6 +6,7 @@ from string import Template from typing import Literal +from bcbench_core.filesystem import remove_tree from pydantic import TypeAdapter from bcbench.config import get_config @@ -13,7 +14,6 @@ from bcbench.dataset.dataset_entry import _BugFixTestGenBase from bcbench.exceptions import BuildError, BuildTimeoutExpired, TestExecutionError, TestExecutionTimeoutExpired from bcbench.logger import get_logger -from bcbench.operations.filesystem_operations import remove_tree from bcbench.operations.setup_operations import bootstrap_app_json from bcbench.types import ContainerConfig diff --git a/src/bcbench/operations/git_operations.py b/src/bcbench/operations/git_operations.py index ab5accf09..82687c881 100644 --- a/src/bcbench/operations/git_operations.py +++ b/src/bcbench/operations/git_operations.py @@ -4,10 +4,11 @@ import tempfile from pathlib import Path +from bcbench_core.filesystem import remove_tree + from bcbench.config import get_config from bcbench.exceptions import EmptyDiffError, PatchApplicationError from bcbench.logger import get_logger -from bcbench.operations.filesystem_operations import remove_tree logger = get_logger(__name__) _config = get_config() diff --git a/src/bcbench/results/__init__.py b/src/bcbench/results/__init__.py index 11de8f933..bb2f86515 100644 --- a/src/bcbench/results/__init__.py +++ b/src/bcbench/results/__init__.py @@ -8,7 +8,7 @@ Leaderboard, LeaderboardAggregate, ) -from bcbench.results.metrics import bootstrap_ci, f_beta_score, pass_at_k, pass_hat_k +from bcbench.results.metrics import bootstrap_ci from bcbench.results.summary import ( BaseEvaluationResult, EvaluationResultSummary, @@ -32,8 +32,5 @@ "bootstrap_ci", "create_console_summary", "create_github_job_summary", - "f_beta_score", - "pass_at_k", - "pass_hat_k", "write_bceval_results", ] diff --git a/src/bcbench/results/codereview.py b/src/bcbench/results/codereview.py index 04ca5f917..89f8bb60a 100644 --- a/src/bcbench/results/codereview.py +++ b/src/bcbench/results/codereview.py @@ -2,6 +2,7 @@ from typing import NamedTuple, Self import numpy as np +from bcbench_core.scoring import f1_score, f_beta_score, precision_recall from pydantic import Field from rich.console import Group, RenderableType from rich.panel import Panel @@ -10,7 +11,6 @@ from bcbench.dataset import ReviewComment from bcbench.results.base import BaseEvaluationResult, JudgeScoredEvaluationResult -from bcbench.results.metrics import f1_score, f_beta_score, precision_recall from bcbench.results.summary import JudgeBasedEvaluationResultSummary from bcbench.types import EvaluationContext diff --git a/src/bcbench/results/leaderboard.py b/src/bcbench/results/leaderboard.py index d7596eb67..10c4fa31a 100644 --- a/src/bcbench/results/leaderboard.py +++ b/src/bcbench/results/leaderboard.py @@ -5,10 +5,11 @@ from pathlib import Path from typing import Any +from bcbench_core.scoring import pass_hat_k from pydantic import BaseModel, field_validator from bcbench.logger import get_logger -from bcbench.results.metrics import bootstrap_ci, pass_hat_k +from bcbench.results.metrics import bootstrap_ci from bcbench.results.summary import EvaluationResultSummary, ExecutionBasedEvaluationResultSummary from bcbench.types import EvaluationCategory, ExperimentConfiguration diff --git a/src/bcbench/results/metrics.py b/src/bcbench/results/metrics.py index 5f08a3b22..c06254048 100644 --- a/src/bcbench/results/metrics.py +++ b/src/bcbench/results/metrics.py @@ -1,5 +1,3 @@ -import math - import numpy as np from scipy.stats import bootstrap as scipy_bootstrap @@ -25,78 +23,3 @@ def bootstrap_ci(values: list[float] | np.ndarray, n_bootstrap: int = 10000, ci_ "ci_low": float(result.confidence_interval.low), "ci_high": float(result.confidence_interval.high), } - - -def precision_recall(matched_count: int, generated_count: int, expected_count: int) -> tuple[float, float]: - """Precision and recall for a set-matching task. - - An empty generated or expected set yields perfect precision or recall respectively, - so a model that correctly produces no output is not penalized. - """ - precision = matched_count / generated_count if generated_count else 1.0 - recall = matched_count / expected_count if expected_count else 1.0 - return precision, recall - - -def f_beta_score(precision: float, recall: float, beta: float = 1.0) -> float: - if precision + recall == 0: - return 0.0 - beta_sq = beta**2 - return (1 + beta_sq) * precision * recall / (beta_sq * precision + recall) - - -def f1_score(precision: float, recall: float) -> float: - return f_beta_score(precision, recall, beta=1.0) - - -def pass_hat_k(num_trials: int, success_count: int, k: int) -> float: - """Measures the probability that all k trials succeed - - Formula: C(success_count, k) / C(num_trials, k) - - Reference: https://arxiv.org/pdf/2406.12045 - - Args: - num_trials: The number of trials (n). - success_count: The number of successful trials. - k: The number of trials to consider. - - Returns: - The pass^k metric (0.0 to 1.0). - - Raises: - ValueError: If num_trials < k. - """ - if num_trials < k: - raise ValueError(f"Number of trials {num_trials} is less than k {k}.") - return math.comb(success_count, k) / math.comb(num_trials, k) - - -def pass_at_k(num_samples: int, num_correct: int, k: int) -> float: - """Measures the likelihood that an agent gets at least one correct solution in k attempts - - Formula: 1 - C(n-c, k) / C(n, k) - - Reference: https://github.com/openai/human-eval-infilling/blob/main/human_eval_infilling/evaluation.py - - Args: - num_samples: Total number of samples (n). - num_correct: Number of correct samples (c). - k: Number of samples to draw. - - Returns: - The pass@k probability (0.0 to 1.0). - - Raises: - ValueError: If num_samples < k. - """ - if num_samples < k: - raise ValueError(f"Number of samples {num_samples} is less than k {k}.") - if num_samples - num_correct < k: - return 1.0 - # Use product formulation to avoid large combinatorial numbers - # 1 - prod_{i=n-c+1}^{n} (1 - k/i) - result = 1.0 - for i in range(num_samples - num_correct + 1, num_samples + 1): - result *= 1.0 - k / i - return 1.0 - result diff --git a/tests/test_filesystem_operations.py b/tests/test_filesystem_operations.py deleted file mode 100644 index 75542b847..000000000 --- a/tests/test_filesystem_operations.py +++ /dev/null @@ -1,25 +0,0 @@ -import stat - -from bcbench.operations import clear_directory - - -def test_clear_directory_removes_contents_and_preserves_directory(tmp_path): - nested_directory = tmp_path / "nested" - nested_directory.mkdir() - (nested_directory / "nested.txt").write_text("nested", encoding="utf-8") - read_only_file = tmp_path / "read-only.txt" - read_only_file.write_text("content", encoding="utf-8") - read_only_file.chmod(stat.S_IREAD) - - clear_directory(tmp_path) - - assert tmp_path.is_dir() - assert list(tmp_path.iterdir()) == [] - - -def test_clear_directory_creates_missing_directory(tmp_path): - missing_directory = tmp_path / "missing" - - clear_directory(missing_directory) - - assert missing_directory.is_dir() diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 890e8a59c..3b04d6805 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -2,7 +2,7 @@ import pytest -from bcbench.results.metrics import bootstrap_ci, pass_at_k, pass_hat_k +from bcbench.results.metrics import bootstrap_ci class TestBootstrapCI: @@ -80,108 +80,3 @@ def test_ci_contains_mean(self): assert ci_high is not None assert mean is not None assert ci_low <= mean <= ci_high - - -class TestPassHatK: - def test_all_successes(self): - # C(3,3)/C(3,3) = 1.0 - assert pass_hat_k(num_trials=3, success_count=3, k=3) == 1.0 - - def test_no_successes(self): - # C(0,3)/C(3,3) = 0.0 - assert pass_hat_k(num_trials=3, success_count=0, k=3) == 0.0 - - def test_partial_success_k1(self): - # C(2,1)/C(3,1) = 2/3 - result = pass_hat_k(num_trials=3, success_count=2, k=1) - assert abs(result - 2 / 3) < 1e-9 - - def test_partial_success_k2(self): - # C(2,2)/C(3,2) = 1/3 - result = pass_hat_k(num_trials=3, success_count=2, k=2) - assert abs(result - 1 / 3) < 1e-9 - - def test_insufficient_successes_for_k(self): - # C(1,3)/C(3,3) = 0 (can't choose 3 from 1) - assert pass_hat_k(num_trials=3, success_count=1, k=3) == 0.0 - - def test_raises_when_trials_less_than_k(self): - with pytest.raises(ValueError, match="Number of trials 2 is less than k 3"): - pass_hat_k(num_trials=2, success_count=2, k=3) - - def test_k_equals_1_is_success_rate(self): - # pass^1 should equal success_count / num_trials - assert pass_hat_k(num_trials=10, success_count=7, k=1) == 0.7 - - def test_large_numbers(self): - # With 10 trials and 8 successes, k=5 - # C(8,5)/C(10,5) = 56/252 = 2/9 - result = pass_hat_k(num_trials=10, success_count=8, k=5) - assert abs(result - 56 / 252) < 1e-9 - - -class TestPassAtK: - def test_all_correct(self): - # If all samples are correct, pass@k = 1.0 - assert pass_at_k(num_samples=5, num_correct=5, k=1) == 1.0 - assert pass_at_k(num_samples=5, num_correct=5, k=5) == 1.0 - - def test_none_correct(self): - # If no samples are correct, pass@k = 0.0 - assert pass_at_k(num_samples=5, num_correct=0, k=1) == 0.0 - assert pass_at_k(num_samples=5, num_correct=0, k=3) == 0.0 - - def test_one_correct_k1(self): - # With 1 correct out of 5, pass@1 = 1 - C(4,1)/C(5,1) = 1 - 4/5 = 0.2 - result = pass_at_k(num_samples=5, num_correct=1, k=1) - assert abs(result - 0.2) < 1e-9 - - def test_one_correct_k5(self): - # With 1 correct out of 5, pass@5 = 1 - C(4,5)/C(5,5) = 1 - 0 = 1.0 - result = pass_at_k(num_samples=5, num_correct=1, k=5) - assert result == 1.0 - - def test_half_correct(self): - # With 3 correct out of 6, pass@1 = 1 - C(3,1)/C(6,1) = 1 - 3/6 = 0.5 - result = pass_at_k(num_samples=6, num_correct=3, k=1) - assert abs(result - 0.5) < 1e-9 - - def test_raises_when_samples_less_than_k(self): - with pytest.raises(ValueError, match="Number of samples 2 is less than k 3"): - pass_at_k(num_samples=2, num_correct=1, k=3) - - def test_returns_1_when_enough_correct(self): - # If n - c < k, return 1.0 (guaranteed success) - # 5 samples, 4 correct, k=2: n-c=1 < k=2, so return 1.0 - assert pass_at_k(num_samples=5, num_correct=4, k=2) == 1.0 - - def test_openai_example(self): - # Test case from OpenAI's human-eval - # n=10, c=3, k=1: 1 - C(7,1)/C(10,1) = 1 - 7/10 = 0.3 - result = pass_at_k(num_samples=10, num_correct=3, k=1) - assert abs(result - 0.3) < 1e-9 - - def test_pass_at_k_increases_with_k(self): - # pass@k should increase (or stay same) as k increases - results = [pass_at_k(num_samples=10, num_correct=3, k=k) for k in range(1, 11)] - for i in range(len(results) - 1): - assert results[i] <= results[i + 1] - - -class TestMetricsRelationship: - def test_pass_at_1_equals_pass_hat_1_for_single_trial(self): - # With k=1, both metrics equal success_count/num_trials - for n in range(1, 10): - for c in range(n + 1): - pass_at = pass_at_k(num_samples=n, num_correct=c, k=1) - pass_hat = pass_hat_k(num_trials=n, success_count=c, k=1) - assert abs(pass_at - pass_hat) < 1e-9 - - def test_pass_at_k_geq_pass_hat_k(self): - # pass@k >= pass^k always (optimistic vs pessimistic) - for n in range(1, 8): - for c in range(n + 1): - for k in range(1, n + 1): - pass_at = pass_at_k(num_samples=n, num_correct=c, k=k) - pass_hat = pass_hat_k(num_trials=n, success_count=c, k=k) - assert pass_at >= pass_hat - 1e-9, f"n={n}, c={c}, k={k}: pass@k={pass_at} < pass^k={pass_hat}" diff --git a/uv.lock b/uv.lock index 616874c7e..dbd317ea5 100644 --- a/uv.lock +++ b/uv.lock @@ -329,6 +329,16 @@ name = "bcbench-core" version = "0.1.0" source = { editable = "packages/bcbench-core" } +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.0.3" }] + [[package]] name = "certifi" version = "2025.10.5" From b04b4f211e782c54030faf74f4c0e18db0c602a5 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Fri, 2 Oct 2026 14:03:36 +0200 Subject: [PATCH 2/2] Address review: unify pytest run, move bootstrap_ci, public is_test_project - Run app and core tests in one pytest invocation from the root config; uv workspaces manage environments, not test runners, and a single run yields one coverage report for both packages - Move bootstrap_ci into bcbench_core.stats; core now depends on numpy and scipy, kept out of bcbench_core.scoring so scoring stays dependency-free - Make is_test_project public and inline the ("test", "tests") default; collection keeps its own substring-based identifiers in app config Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3fa47b3c-f5d6-4c36-9be3-c4fdf2bebe47 --- .github/workflows/CI.yml | 5 +---- CONTRIBUTING.md | 5 ++--- notebooks/bug-fix/altool-comparison.ipynb | 3 +-- notebooks/bug-fix/claude-vs-copilot.ipynb | 3 +-- .../test-generation/altest-comparison.ipynb | 3 +-- packages/bcbench-core/pyproject.toml | 10 ++++----- .../bcbench-core/src/bcbench_core/projects.py | 21 ++++++++----------- .../bcbench-core/src/bcbench_core/stats.py | 2 ++ packages/bcbench-core/tests/test_projects.py | 16 +++++++------- .../bcbench-core/tests/test_stats.py | 2 +- pyproject.toml | 2 +- src/bcbench/collection/collect_gh.py | 5 ++--- src/bcbench/collection/patch_utils.py | 3 +-- src/bcbench/config.py | 2 ++ src/bcbench/results/__init__.py | 2 -- src/bcbench/results/leaderboard.py | 2 +- uv.lock | 8 +++++++ 17 files changed, 45 insertions(+), 49 deletions(-) rename src/bcbench/results/metrics.py => packages/bcbench-core/src/bcbench_core/stats.py (93%) rename tests/test_metrics.py => packages/bcbench-core/tests/test_stats.py (98%) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index fcb45b6f4..0bce9ac66 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -37,10 +37,7 @@ jobs: RUFF_OUTPUT_FORMAT: github - name: Run tests with coverage - run: uv run pytest --cov=src/bcbench --cov-report=term-missing - - - name: Run bcbench-core tests with coverage - run: uv run pytest packages/bcbench-core --cov=bcbench_core --cov-report=term-missing + run: uv run pytest --cov=src/bcbench --cov=packages/bcbench-core/src/bcbench_core --cov-report=term-missing e2e: strategy: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e50bb4e64..98ea84f11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,9 +61,8 @@ uv run bcbench --help # This is very fast, give it a go and see it live! uv run bcbench run copilot microsoft__BCApps-5633 --category bug-fix --repo-path /path/to/BCApps -# Run tests (bcbench, then bcbench-core with its own pytest configuration) -uv run pytest --cov=src/bcbench --cov-report=term-missing -uv run pytest packages/bcbench-core --cov=bcbench_core --cov-report=term-missing +# Run tests (bcbench and bcbench-core) +uv run pytest --cov=src/bcbench --cov=packages/bcbench-core/src/bcbench_core --cov-report=term-missing # Lint and format uv run pre-commit run --all-files diff --git a/notebooks/bug-fix/altool-comparison.ipynb b/notebooks/bug-fix/altool-comparison.ipynb index 455c319bd..e1c95824d 100644 --- a/notebooks/bug-fix/altool-comparison.ipynb +++ b/notebooks/bug-fix/altool-comparison.ipynb @@ -47,8 +47,7 @@ "import numpy as np\n", "import plotly.graph_objects as go\n", "from bcbench_core.scoring import pass_at_k, pass_hat_k\n", - "\n", - "from bcbench.results.metrics import bootstrap_ci" + "from bcbench_core.stats import bootstrap_ci" ] }, { diff --git a/notebooks/bug-fix/claude-vs-copilot.ipynb b/notebooks/bug-fix/claude-vs-copilot.ipynb index 49d1421c2..a26fc6463 100644 --- a/notebooks/bug-fix/claude-vs-copilot.ipynb +++ b/notebooks/bug-fix/claude-vs-copilot.ipynb @@ -55,10 +55,9 @@ "outputs": [], "source": [ "import plotly.graph_objects as go\n", + "from bcbench_core.stats import bootstrap_ci\n", "from IPython.display import Markdown, display\n", "\n", - "from bcbench.results.metrics import bootstrap_ci\n", - "\n", "\n", "def build_mean_ci_table(model: str) -> pd.DataFrame:\n", " rows = []\n", diff --git a/notebooks/test-generation/altest-comparison.ipynb b/notebooks/test-generation/altest-comparison.ipynb index 244f3fbfe..6bc7349cd 100644 --- a/notebooks/test-generation/altest-comparison.ipynb +++ b/notebooks/test-generation/altest-comparison.ipynb @@ -38,8 +38,7 @@ "import numpy as np\n", "import plotly.graph_objects as go\n", "from bcbench_core.scoring import pass_at_k, pass_hat_k\n", - "\n", - "from bcbench.results.metrics import bootstrap_ci" + "from bcbench_core.stats import bootstrap_ci" ] }, { diff --git a/packages/bcbench-core/pyproject.toml b/packages/bcbench-core/pyproject.toml index d4733387c..fc8b5064d 100644 --- a/packages/bcbench-core/pyproject.toml +++ b/packages/bcbench-core/pyproject.toml @@ -18,16 +18,14 @@ classifiers = [ "Programming Language :: Python :: 3.13", "Typing :: Typed", ] -dependencies = [] +dependencies = [ + "numpy>=2.3.5", + "scipy>=1.16.3", +] [dependency-groups] dev = ["pytest>=9.0.3"] -[tool.pytest] -testpaths = ["tests"] -addopts = ["--import-mode=importlib"] -strict = true - [tool.ruff] target-version = "py313" line-length = 200 diff --git a/packages/bcbench-core/src/bcbench_core/projects.py b/packages/bcbench-core/src/bcbench_core/projects.py index 2f438be0c..4543aa682 100644 --- a/packages/bcbench-core/src/bcbench_core/projects.py +++ b/packages/bcbench-core/src/bcbench_core/projects.py @@ -4,33 +4,30 @@ logger = logging.getLogger(__name__) -DEFAULT_TEST_PROJECT_IDENTIFIERS: tuple[str, ...] = ("test", "tests") +def is_test_project(project_path: str, test_identifiers: tuple[str, ...] = ("test", "tests")) -> bool: + r"""Check if a project path is a test project. -def _is_test_project(project_path: str, test_identifiers: tuple[str, ...]) -> bool: - r"""Check if a project path is a test project based on configured identifiers. - - The function checks if any test identifier appears as a complete path component - by looking for the identifier preceded by a path separator (/ or \). - This ensures that 'src/contest' does not match 'test', but 'src/test' does. + A path is a test project when one of its components, after a path separator (/ or \), + starts with a test identifier: 'src/test' and 'src/test1' match 'test', 'src/contest' does not. Args: project_path: The project path to check - test_identifiers: Tuple of test identifier strings (e.g., 'test', 'tests') + test_identifiers: Case-insensitive prefixes that mark a test project component Returns: - True if the project path contains a test identifier as a path component + True if a path component starts with a test identifier """ project_lower = project_path.lower() return any(f"/{identifier}" in project_lower or f"\\{identifier}" in project_lower for identifier in test_identifiers) -def categorize_projects(project_paths: list[str], test_identifiers: tuple[str, ...] = DEFAULT_TEST_PROJECT_IDENTIFIERS) -> tuple[list[str], list[str]]: +def categorize_projects(project_paths: list[str], test_identifiers: tuple[str, ...] = ("test", "tests")) -> tuple[list[str], list[str]]: """Categorize project paths into test projects and application projects. Args: project_paths: List of project paths to categorize - test_identifiers: Path components that mark a test project + test_identifiers: Case-insensitive prefixes that mark a test project component Returns: Tuple of (test_projects, app_projects) @@ -38,7 +35,7 @@ def categorize_projects(project_paths: list[str], test_identifiers: tuple[str, . Raises: RuntimeError: If project categorization fails (no test or app projects found) """ - test_projects: list[str] = [project for project in project_paths if _is_test_project(project, test_identifiers)] + test_projects: list[str] = [project for project in project_paths if is_test_project(project, test_identifiers)] app_projects: list[str] = [project for project in project_paths if project not in test_projects] if not test_projects or not app_projects: diff --git a/src/bcbench/results/metrics.py b/packages/bcbench-core/src/bcbench_core/stats.py similarity index 93% rename from src/bcbench/results/metrics.py rename to packages/bcbench-core/src/bcbench_core/stats.py index c06254048..893e8178f 100644 --- a/src/bcbench/results/metrics.py +++ b/packages/bcbench-core/src/bcbench_core/stats.py @@ -1,3 +1,5 @@ +"""Statistics for aggregating evaluation results across runs.""" + import numpy as np from scipy.stats import bootstrap as scipy_bootstrap diff --git a/packages/bcbench-core/tests/test_projects.py b/packages/bcbench-core/tests/test_projects.py index a943719b6..cb798c229 100644 --- a/packages/bcbench-core/tests/test_projects.py +++ b/packages/bcbench-core/tests/test_projects.py @@ -2,29 +2,29 @@ import pytest -from bcbench_core.projects import _is_test_project, categorize_projects +from bcbench_core.projects import categorize_projects, is_test_project class TestIsTestProject: - """Test suite for _is_test_project helper function.""" + """Test suite for is_test_project.""" def test_is_test_project_with_test_identifier(self): - assert _is_test_project("src/test", ("test", "tests")) is True + assert is_test_project("src/test", ("test", "tests")) is True def test_is_test_project_with_tests_identifier(self): - assert _is_test_project("src/tests", ("test", "tests")) is True + assert is_test_project("src/tests", ("test", "tests")) is True def test_is_test_project_with_windows_separator(self): - assert _is_test_project("src\\test", ("test", "tests")) is True + assert is_test_project("src\\test", ("test", "tests")) is True def test_is_test_project_case_insensitive(self): - assert _is_test_project("src/Test", ("test", "tests")) is True + assert is_test_project("src/Test", ("test", "tests")) is True def test_is_test_project_substring_not_path_component(self): - assert _is_test_project("src/contest", ("test", "tests")) is False + assert is_test_project("src/contest", ("test", "tests")) is False def test_is_test_project_without_identifier(self): - assert _is_test_project("src/app", ("test", "tests")) is False + assert is_test_project("src/app", ("test", "tests")) is False class TestCategorizeProjects: diff --git a/tests/test_metrics.py b/packages/bcbench-core/tests/test_stats.py similarity index 98% rename from tests/test_metrics.py rename to packages/bcbench-core/tests/test_stats.py index 3b04d6805..296997599 100644 --- a/tests/test_metrics.py +++ b/packages/bcbench-core/tests/test_stats.py @@ -2,7 +2,7 @@ import pytest -from bcbench.results.metrics import bootstrap_ci +from bcbench_core.stats import bootstrap_ci class TestBootstrapCI: diff --git a/pyproject.toml b/pyproject.toml index 288d9ed8c..6331ce995 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ url = "https://packagefeedproxy.microsoft.io/pypi/simple" default = true [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["tests", "packages/bcbench-core/tests"] addopts = ["-v", "--strict-markers", "-m", "not e2e"] markers = [ "integration", diff --git a/src/bcbench/collection/collect_gh.py b/src/bcbench/collection/collect_gh.py index 003bacda0..cbc8377b7 100644 --- a/src/bcbench/collection/collect_gh.py +++ b/src/bcbench/collection/collect_gh.py @@ -5,7 +5,6 @@ from typing import Any import typer -from bcbench_core.projects import DEFAULT_TEST_PROJECT_IDENTIFIERS from bcbench.collection.gh_client import GHClient from bcbench.collection.patch_utils import extract_file_paths_from_patch, find_project_paths_from_diff, separate_patches @@ -61,7 +60,7 @@ def fail(reason: str) -> ScreeningResult: raise CollectionError(f"Failed to fetch PR #{pr_number} from {repo}: {exc}") from exc try: - patch, patch_fix, patch_test = separate_patches(diff, DEFAULT_TEST_PROJECT_IDENTIFIERS) + patch, patch_fix, patch_test = separate_patches(diff, _config.file_patterns.test_project_identifiers) project_paths = find_project_paths_from_diff(patch) except CollectionError as exc: raise CollectionError(f"Failed to parse PR diff: {exc}") from exc @@ -104,7 +103,7 @@ def _build_bugfix_entry( diff = gh_client.get_pr_diff(pr_number) - patch, patch_fix, patch_test = separate_patches(diff, DEFAULT_TEST_PROJECT_IDENTIFIERS) + patch, patch_fix, patch_test = separate_patches(diff, _config.file_patterns.test_project_identifiers) # Extract problem statement from PR title = pr_data.get("title", "") diff --git a/src/bcbench/collection/patch_utils.py b/src/bcbench/collection/patch_utils.py index 231a2d4d4..ba384beda 100644 --- a/src/bcbench/collection/patch_utils.py +++ b/src/bcbench/collection/patch_utils.py @@ -3,7 +3,6 @@ import subprocess from pathlib import Path -from bcbench_core.projects import DEFAULT_TEST_PROJECT_IDENTIFIERS from unidiff import PatchSet from unidiff.errors import UnidiffParseError @@ -77,7 +76,7 @@ def extract_patches(repo_path: Path, base_commit_id: str, commit_id: str, diff_p if not patch: raise CollectionError("No patch data found between the specified commits") - return separate_patches(patch, DEFAULT_TEST_PROJECT_IDENTIFIERS) + return separate_patches(patch, _config.file_patterns.test_project_identifiers) def find_project_paths_from_diff(patch: str) -> list[str]: diff --git a/src/bcbench/config.py b/src/bcbench/config.py index fc9cdf0a2..291a646dc 100644 --- a/src/bcbench/config.py +++ b/src/bcbench/config.py @@ -110,6 +110,7 @@ class FilePatternConfig: result_pattern: str instruction_source_naming: str instructions_dirname: str + test_project_identifiers: tuple[str, ...] problem_statement_readme: str problem_statement_dest_dir: str alpackages_dirname: str @@ -126,6 +127,7 @@ def default(cls) -> FilePatternConfig: result_pattern=".jsonl", instruction_source_naming="AGENTS.md", instructions_dirname="instructions", + test_project_identifiers=("test", "tests"), problem_statement_readme="README.md", problem_statement_dest_dir="problem", alpackages_dirname=".alpackages", diff --git a/src/bcbench/results/__init__.py b/src/bcbench/results/__init__.py index bb2f86515..1d8112d0d 100644 --- a/src/bcbench/results/__init__.py +++ b/src/bcbench/results/__init__.py @@ -8,7 +8,6 @@ Leaderboard, LeaderboardAggregate, ) -from bcbench.results.metrics import bootstrap_ci from bcbench.results.summary import ( BaseEvaluationResult, EvaluationResultSummary, @@ -29,7 +28,6 @@ "JudgeBasedEvaluationResultSummary", "Leaderboard", "LeaderboardAggregate", - "bootstrap_ci", "create_console_summary", "create_github_job_summary", "write_bceval_results", diff --git a/src/bcbench/results/leaderboard.py b/src/bcbench/results/leaderboard.py index 10c4fa31a..d761e050d 100644 --- a/src/bcbench/results/leaderboard.py +++ b/src/bcbench/results/leaderboard.py @@ -6,10 +6,10 @@ from typing import Any from bcbench_core.scoring import pass_hat_k +from bcbench_core.stats import bootstrap_ci from pydantic import BaseModel, field_validator from bcbench.logger import get_logger -from bcbench.results.metrics import bootstrap_ci from bcbench.results.summary import EvaluationResultSummary, ExecutionBasedEvaluationResultSummary from bcbench.types import EvaluationCategory, ExperimentConfiguration diff --git a/uv.lock b/uv.lock index dbd317ea5..18a978da5 100644 --- a/uv.lock +++ b/uv.lock @@ -328,6 +328,10 @@ redteam = [ name = "bcbench-core" version = "0.1.0" source = { editable = "packages/bcbench-core" } +dependencies = [ + { name = "numpy" }, + { name = "scipy" }, +] [package.dev-dependencies] dev = [ @@ -335,6 +339,10 @@ dev = [ ] [package.metadata] +requires-dist = [ + { name = "numpy", specifier = ">=2.3.5" }, + { name = "scipy", specifier = ">=1.16.3" }, +] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.3" }]