diff --git a/.github/workflows/result-server-tests.yml b/.github/workflows/result-server-tests.yml index 8d9c21b..525199c 100644 --- a/.github/workflows/result-server-tests.yml +++ b/.github/workflows/result-server-tests.yml @@ -40,53 +40,35 @@ jobs: - name: Run result_server pytest suite run: python result_server/tests/run_result_server_tests.py - profile-data-shell: + profile-data-shell-group: + name: profile-data-shell / ${{ matrix.shell-group }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shell-group: + - common + - result-sender + - estimation steps: - name: Check out repository uses: actions/checkout@v4 - - name: Check shell test dependencies - run: | - set -euo pipefail - command -v jq - jq --version - command -v python3 - python3 --version - estimation_python="" - for candidate in python3.12 python3.11 python3; do - if command -v "$candidate" >/dev/null 2>&1; then - estimation_python=$(command -v "$candidate") - break - fi - done - test -n "$estimation_python" - "$estimation_python" - <<'PY' - import sys - raise SystemExit(0 if sys.version_info >= (3, 11) else 1) - PY + - name: Run ${{ matrix.shell-group }} shell tests + run: bash scripts/tests/run_profile_data_shell_tests.sh "${{ matrix.shell-group }}" - - name: Run profiler, profile-data, and estimation shell tests + profile-data-shell: + runs-on: ubuntu-latest + needs: profile-data-shell-group + if: ${{ always() }} + + steps: + - name: Check shell test groups run: | - bash scripts/tests/test_bk_profiler.sh - bash scripts/tests/test_bk_fetch_source.sh - bash scripts/tests/test_bk_input_info.sh - bash scripts/tests/test_bk_timing_observations.sh - bash scripts/tests/test_qws_timing_artifact.sh - bash scripts/tests/test_build_cache.sh - bash scripts/tests/test_build_environment_snapshot.sh - bash scripts/tests/test_ci_timing_context.sh - bash scripts/tests/test_ncu_plan_generation.sh - bash scripts/tests/test_result_profile_data.sh - bash scripts/tests/test_process_and_send_results.sh - bash scripts/tests/test_result_common_json_contract.sh - bash scripts/tests/test_scheduler_extra_args.sh - bash scripts/tests/test_send_results_profile_data.sh - bash scripts/tests/test_send_estimate_artifacts.sh - bash scripts/tests/test_estimation_common_json_contract.sh - bash scripts/tests/test_estimation_run_timing.sh - bash scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh - bash scripts/tests/test_estimation_gpu_kernel_lightgbm_v10.sh - bash scripts/tests/test_estimation_gpu_kernel_mlp_v15.sh - bash scripts/tests/test_genesis_gpu_mlp_estimation.sh + set -euo pipefail + if [ "${{ needs.profile-data-shell-group.result }}" != "success" ]; then + echo "One or more shell test groups failed" + exit 1 + fi + echo "All shell test groups passed" diff --git a/result_server/tests/test_profile_data_shell_runner.py b/result_server/tests/test_profile_data_shell_runner.py new file mode 100644 index 0000000..21ea72f --- /dev/null +++ b/result_server/tests/test_profile_data_shell_runner.py @@ -0,0 +1,37 @@ +import subprocess +from pathlib import Path + + +REPO_DIR = Path(__file__).resolve().parents[2] +RUNNER = REPO_DIR / "scripts/tests/run_profile_data_shell_tests.sh" + + +def _runner_lines(*args): + output = subprocess.check_output( + ["bash", str(RUNNER), *args], + cwd=REPO_DIR, + text=True, + ) + return [line for line in output.splitlines() if line] + + +def test_profile_data_shell_runner_covers_shell_tests_once(): + groups = _runner_lines("list") + assert groups == ["common", "result-sender", "estimation"] + + grouped_tests = [] + for group in groups: + grouped_tests.extend(_runner_lines("list-tests", group)) + + all_tests = _runner_lines("list-tests") + assert grouped_tests == all_tests + assert len(grouped_tests) == len(set(grouped_tests)) + + discovered_tests = sorted( + str(path.relative_to(REPO_DIR)) + for path in (REPO_DIR / "scripts/tests").glob("test_*.sh") + ) + assert sorted(grouped_tests) == discovered_tests + + for test_path in grouped_tests: + assert (REPO_DIR / test_path).is_file() diff --git a/scripts/tests/run_profile_data_shell_tests.sh b/scripts/tests/run_profile_data_shell_tests.sh new file mode 100644 index 0000000..4dd9a27 --- /dev/null +++ b/scripts/tests/run_profile_data_shell_tests.sh @@ -0,0 +1,141 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +ALL_GROUPS=(common result-sender estimation) + +usage() { + cat <<'EOF' +Usage: bash scripts/tests/run_profile_data_shell_tests.sh [all|common|result-sender|estimation|list|list-tests [GROUP]] + +Runs the shell tests covered by the Result Server Tests workflow. +Use "all" locally for the full suite, or a group name in CI matrix jobs. +EOF +} + +check_dependencies() { + command -v jq + jq --version + command -v python3 + python3 --version + + local estimation_python="" + local candidate + for candidate in python3.12 python3.11 python3; do + if command -v "$candidate" >/dev/null 2>&1; then + estimation_python=$(command -v "$candidate") + break + fi + done + test -n "$estimation_python" + "$estimation_python" - <<'PY' +import sys +raise SystemExit(0 if sys.version_info >= (3, 11) else 1) +PY +} + +tests_for_group() { + case "$1" in + common) + cat <<'EOF' +scripts/tests/test_bk_profiler.sh +scripts/tests/test_bk_fetch_source.sh +scripts/tests/test_bk_input_info.sh +scripts/tests/test_bk_timing_observations.sh +scripts/tests/test_qws_timing_artifact.sh +scripts/tests/test_build_environment_snapshot.sh +scripts/tests/test_ci_timing_context.sh +scripts/tests/test_ncu_plan_generation.sh +scripts/tests/test_scheduler_extra_args.sh +EOF + ;; + result-sender) + cat <<'EOF' +scripts/tests/test_build_cache.sh +scripts/tests/test_result_profile_data.sh +scripts/tests/test_process_and_send_results.sh +scripts/tests/test_result_common_json_contract.sh +scripts/tests/test_send_results_profile_data.sh +scripts/tests/test_send_estimate_artifacts.sh +EOF + ;; + estimation) + cat <<'EOF' +scripts/tests/test_estimation_common_json_contract.sh +scripts/tests/test_estimation_run_timing.sh +scripts/tests/test_estimation_gpu_kernel_ensemble_average.sh +scripts/tests/test_estimation_gpu_kernel_lightgbm_v10.sh +scripts/tests/test_estimation_gpu_kernel_mlp_v15.sh +scripts/tests/test_genesis_gpu_mlp_estimation.sh +EOF + ;; + all) + local group + for group in "${ALL_GROUPS[@]}"; do + tests_for_group "$group" + done + ;; + *) + echo "Unknown shell test group: $1" >&2 + usage >&2 + return 2 + ;; + esac +} + +run_one_test() { + local test_rel="$1" + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::group::${test_rel}" + else + echo "==> ${test_rel}" + fi + bash "${REPO_DIR}/${test_rel}" + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::endgroup::" + fi +} + +run_group() { + local group="$1" + local count=0 + local test_rel + + check_dependencies + while IFS= read -r test_rel; do + [ -n "$test_rel" ] || continue + if [ ! -f "${REPO_DIR}/${test_rel}" ]; then + echo "Missing shell test: ${test_rel}" >&2 + return 1 + fi + run_one_test "$test_rel" + count=$((count + 1)) + done < <(tests_for_group "$group") + + if [ "$count" -eq 0 ]; then + echo "No shell tests selected for group: ${group}" >&2 + return 1 + fi + echo "Ran ${count} shell test(s) for group: ${group}" +} + +main() { + local group="${1:-all}" + case "$group" in + -h|--help) + usage + ;; + list|--list) + printf '%s\n' "${ALL_GROUPS[@]}" + ;; + list-tests|--list-tests) + tests_for_group "${2:-all}" + ;; + *) + run_group "$group" + ;; + esac +} + +main "$@"