diff --git a/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml b/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml index a72e6761df238..0ea4fc7f14655 100644 --- a/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml +++ b/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml @@ -7,6 +7,10 @@ base_images: name: nested-podman namespace: ci tag: latest + prow-agent-eval: + name: prow-agent-eval + namespace: ci + tag: prow-agent-eval sippy-devcontainer: name: sippy namespace: ci @@ -21,6 +25,7 @@ images: - dockerfile_literal: | FROM claude-ai-helpers AS ai-helpers FROM nested-podman AS podman + FROM prow-agent-eval AS proweval FROM sippy-devcontainer USER root RUN dnf install -y podman fuse-overlayfs shadow-utils openssl && dnf clean all @@ -36,6 +41,7 @@ images: /home/vscode/.claude/plugins/known_marketplaces.json RUN chgrp -R 0 /opt/ai-helpers /home/vscode/.claude && \ chmod -R g=u /opt/ai-helpers /home/vscode/.claude + COPY --from=proweval /usr/local/bin/prow-agent-eval /usr/local/bin/prow-agent-eval COPY ci-operator/step-registry/openshift/agentic/trt/jira-solver/openshift-agentic-trt-jira-solver-commands.sh /opt/scripts/solve.sh RUN chmod +x /opt/scripts/solve.sh RUN mkdir -p /workspace && chgrp 0 /workspace && chmod g=u /workspace @@ -52,6 +58,9 @@ images: nested-podman: as: - nested-podman + prow-agent-eval: + as: + - prow-agent-eval to: agentic-dev run_if_changed: ^ci-operator/step-registry/openshift/agentic/trt/ resources: diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh index 0373b3824354b..79cf0e8cd9fc8 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh @@ -16,35 +16,7 @@ if [[ -z "${GITHUB_TOKEN}" ]]; then exit 0 fi -if [[ ! -f "${SHARED_DIR}/eval-cases" ]]; then - echo "No eval-cases file found, skipping cleanup." - exit 0 -fi - -mapfile -t CASE_LIST < "${SHARED_DIR}/eval-cases" -echo "Cleaning up ${#CASE_LIST[@]} cases..." - -for case_name in "${CASE_LIST[@]}"; do - PR_NUM="" - if [[ -f "${SHARED_DIR}/${case_name}.pr-number" ]]; then - PR_NUM=$(cat "${SHARED_DIR}/${case_name}.pr-number") - fi - - if [[ -n "${PR_NUM}" ]]; then - echo "[${case_name}] Closing PR #${PR_NUM} and deleting branch..." - gh pr close "${PR_NUM}" --repo "${UPSTREAM_REPO}" --delete-branch || true - else - CLAUDE_BRANCH="" - if [[ -f "${SHARED_DIR}/${case_name}.claude-branch" ]]; then - CLAUDE_BRANCH=$(cat "${SHARED_DIR}/${case_name}.claude-branch") - fi - if [[ -n "${CLAUDE_BRANCH}" ]]; then - echo "[${case_name}] No PR found, deleting branch ${CLAUDE_BRANCH}..." - gh api "repos/${UPSTREAM_REPO}/git/refs/heads/${CLAUDE_BRANCH}" -X DELETE || true - else - echo "[${case_name}] Nothing to clean up." - fi - fi -done +prow-agent-eval cleanup \ + --shared-dir="${SHARED_DIR}" || true echo "=== TRT Eval Cleanup Complete ===" diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml index a3b04b3b20d1a..c475442c2a916 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml @@ -11,6 +11,5 @@ ref: cpu: 100m memory: 100Mi documentation: |- - Cleans up eval artifacts: closes the eval PR and deletes the - timestamped branch from the eval repo. Runs as a post-step so - cleanup happens even if the test phase fails. + Uses prow-agent-eval CLI to close PRs and delete branches. + Runs as a post-step so cleanup happens even if the test phase fails. diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh index 943de303871e4..f5f190122979f 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh @@ -6,57 +6,55 @@ set -o pipefail echo "=== TRT Eval Init ===" -# --- Gangway override --- if [[ -n "${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE:-}" ]]; then - echo "Applying Gangway override: EVAL_CASE=${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE}" EVAL_CASE="${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE}" fi -ALL_CASES_DIR="/opt/ai-helpers/evals/jira-solver/cases" - -# --- Build case list --- -CASE_LIST=() -if [[ -n "${EVAL_CASE:-}" ]]; then - CASE_LIST=("${EVAL_CASE}") +set +x +GITHUB_TOKEN=$(cat "${SHARED_DIR}/gh-upstream-token") +export GITHUB_TOKEN +set -x + +EVAL_CONFIG_DIR="/opt/ai-helpers/evals/jira-solver" + +# Write eval config if not already present in the image +if [[ ! -f "${EVAL_CONFIG_DIR}/eval.yaml" ]]; then + cat > /tmp/eval.yaml <<'EVALCFG' +name: jira-solver-eval +init: + repo: "${UPSTREAM_REPO}" +dataset: + path: /opt/ai-helpers/evals/jira-solver/cases +collect: + build_result: true + test_result: true + expected_branch_diff: true +judges: + - name: branch_created + - name: pr_exists + - name: build_passed + - name: test_passed + - name: file_overlap + - name: pr_description_exists + - name: diff_size_ratio + - name: function_overlap +thresholds: {} +EVALCFG + sed -i "s|\${UPSTREAM_REPO}|${UPSTREAM_REPO}|g" /tmp/eval.yaml + EVAL_CONFIG="/tmp/eval.yaml" else - for d in "${ALL_CASES_DIR}"/*/; do - CASE_LIST+=("$(basename "$d")") - done + EVAL_CONFIG="${EVAL_CONFIG_DIR}/eval.yaml" fi -[[ ${#CASE_LIST[@]} -gt 0 ]] || { echo "ERROR: No eval cases found."; exit 1; } -echo "Cases to run: ${CASE_LIST[*]}" - -# --- Set up per-case metadata --- -# input.yaml must use flat "key: value" format — no nesting, quoting, or indentation -yaml_val() { grep "^${1}:" "$2" | cut -d' ' -f2-; } - -for case_name in "${CASE_LIST[@]}"; do - CASE_SRC="${ALL_CASES_DIR}/${case_name}" - [[ -d "${CASE_SRC}" ]] || { echo "ERROR: Case directory not found: ${CASE_SRC}"; exit 1; } - - INPUT_FILE="${CASE_SRC}/input.yaml" - [[ -f "${INPUT_FILE}" ]] || { echo "ERROR: input.yaml not found in ${CASE_SRC}"; exit 1; } - - JIRA_ISSUE_KEY=$(yaml_val jira_key "${INPUT_FILE}") - BASE_BRANCH=$(yaml_val base_branch "${INPUT_FILE}") - EXPECTED_BRANCH=$(yaml_val expected_branch "${INPUT_FILE}") - - [[ -n "${JIRA_ISSUE_KEY}" ]] || { echo "ERROR: ${case_name}: missing jira_key in input.yaml"; exit 1; } - [[ -n "${BASE_BRANCH}" ]] || { echo "ERROR: ${case_name}: missing base_branch in input.yaml"; exit 1; } - [[ -n "${EXPECTED_BRANCH}" ]] || { echo "ERROR: ${case_name}: missing expected_branch in input.yaml"; exit 1; } - - echo "${JIRA_ISSUE_KEY}" > "${SHARED_DIR}/${case_name}.jira-issue-key" - cp "${CASE_SRC}/jira-issue.json" "${SHARED_DIR}/${case_name}.jira-issue.json" - echo "${BASE_BRANCH}" > "${SHARED_DIR}/${case_name}.eval-base-branch" - echo "${EXPECTED_BRANCH}" > "${SHARED_DIR}/${case_name}.eval-expected-branch" - echo "${case_name}" > "${SHARED_DIR}/${case_name}.eval-case" - - SUMMARY=$(jq -r '.fields.summary // .summary // "N/A"' "${SHARED_DIR}/${case_name}.jira-issue.json") - echo " ${case_name}: ${JIRA_ISSUE_KEY} - ${SUMMARY}" -done +CASE_FLAG="" +if [[ -n "${EVAL_CASE:-}" ]]; then + CASE_FLAG="--case=${EVAL_CASE}" +fi -# --- Write case list for downstream steps --- -printf '%s\n' "${CASE_LIST[@]}" > "${SHARED_DIR}/eval-cases" +prow-agent-eval init \ + --config="${EVAL_CONFIG}" \ + --shared-dir="${SHARED_DIR}" \ + --mode=solve \ + ${CASE_FLAG} echo "=== TRT Eval Init Complete ===" diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml index 452868fad2c93..a578caeaabd60 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml @@ -3,6 +3,9 @@ ref: from: agentic-dev commands: openshift-agentic-trt-eval-init-commands.sh env: + - name: UPSTREAM_REPO + documentation: |- + Target repo for the eval (org/repo format). - name: EVAL_CASE default: "" documentation: |- @@ -19,6 +22,5 @@ ref: memory: 256Mi timeout: 10m0s documentation: |- - Eval init step. Reads case configuration from the ai-helpers image, - copies JIRA JSON snapshots to SHARED_DIR, and writes per-case - metadata (jira key, base/expected branches) for downstream steps. + Eval init step. Uses prow-agent-eval CLI to discover cases, + validate inputs, and write per-case metadata to SHARED_DIR. diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh index 54b7394a8d366..ff8442c2b9c42 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh @@ -6,422 +6,44 @@ set -o pipefail echo "=== TRT Eval Judge ===" -# --- Read case list --- -mapfile -t CASE_LIST < "${SHARED_DIR}/eval-cases" -echo "Cases to judge (${#CASE_LIST[@]}): ${CASE_LIST[*]}" - -# --- Clone repo template --- -TEMPLATE_DIR="/tmp/eval-repo-template" -git clone "https://github.com/${UPSTREAM_REPO}.git" "${TEMPLATE_DIR}" - set +x GITHUB_TOKEN=$(cat "${SHARED_DIR}/gh-upstream-token") export GITHUB_TOKEN set -x -# --- Utilities --- -diff_stat_total() { - local stat_line=$1 - local ins del - ins=$(echo "${stat_line}" | grep -oE '[0-9]+ insertion' | grep -oE '[0-9]+' || echo "0") - del=$(echo "${stat_line}" | grep -oE '[0-9]+ deletion' | grep -oE '[0-9]+' || echo "0") - echo $(( ins + del )) -} - -jaccard_similarity() { - local file_a=$1 file_b=$2 - python3 -c " -import sys -a = set(open(sys.argv[1]).read().strip().split('\n')) if open(sys.argv[1]).read().strip() else set() -b = set(open(sys.argv[2]).read().strip().split('\n')) if open(sys.argv[2]).read().strip() else set() -if not a and not b: print('1.0') -elif not a or not b: print('0.0') -else: print(f'{len(a & b) / len(a | b):.2f}') -" "${file_a}" "${file_b}" -} - -# --- Aggregate accumulators --- -TOTAL_CHECKS_PASS=0 -TOTAL_CHECKS_TOTAL=0 -ALL_JUNIT_TESTCASES="" -ALL_SUMMARY_ROWS="" -ALL_CASE_DETAILS="" - -# ============================================= -# JUDGE EACH CASE -# ============================================= -for case_name in "${CASE_LIST[@]}"; do - echo "" - echo "==========================================" - echo "Judging: ${case_name}" - echo "==========================================" - - EVAL_CASE=$(cat "${SHARED_DIR}/${case_name}.eval-case") - BASE_BRANCH=$(cat "${SHARED_DIR}/${case_name}.eval-base-branch") - EXPECTED_BRANCH=$(cat "${SHARED_DIR}/${case_name}.eval-expected-branch") - JIRA_ISSUE_KEY=$(cat "${SHARED_DIR}/${case_name}.jira-issue-key") - - echo " JIRA: ${JIRA_ISSUE_KEY} | Base: ${BASE_BRANCH} | Expected: ${EXPECTED_BRANCH}" - - # Set up per-case workspace from template - CASE_WORKDIR="/workspace/${case_name}" - cp -r "${TEMPLATE_DIR}" "${CASE_WORKDIR}" - cd "${CASE_WORKDIR}" - - # Check out Claude's branch - CLAUDE_BRANCH="" - if [[ -f "${SHARED_DIR}/${case_name}.claude-branch" ]]; then - CLAUDE_BRANCH=$(cat "${SHARED_DIR}/${case_name}.claude-branch") - git fetch origin "${CLAUDE_BRANCH}" - git checkout "${CLAUDE_BRANCH}" - fi - echo " Claude's branch: ${CLAUDE_BRANCH:-}" - - git fetch origin "${BASE_BRANCH}" "${EXPECTED_BRANCH}" 2>/dev/null || true - - # --- Per-case results --- - unset CHECKS 2>/dev/null || true - declare -A CHECKS - CHECKS_PASS=0 - CHECKS_TOTAL=0 - SCORES="" - - record_check() { - local name=$1 result=$2 - CHECKS["${name}"]="${result}" - CHECKS_TOTAL=$(( CHECKS_TOTAL + 1 )) - if [[ "${result}" == "pass" ]]; then - CHECKS_PASS=$(( CHECKS_PASS + 1 )) - echo " [PASS] ${name}" - else - echo " [FAIL] ${name}" - fi - } - - record_score() { - local name=$1 value=$2 - SCORES="${SCORES}${name}: ${value}\n" - echo " [SCORE] ${name}: ${value}" - } - - # --- Hardcoded checks --- - echo " --- Checks ---" - - # branch_created - if [[ -n "${CLAUDE_BRANCH}" && "${CLAUDE_BRANCH}" != "main" && "${CLAUDE_BRANCH}" != "master" && "${CLAUDE_BRANCH}" != "${BASE_BRANCH}" ]]; then - record_check "branch_created" "pass" - else - record_check "branch_created" "fail" - fi - - # code_compiles - BUILD_LOG="${ARTIFACT_DIR}/${case_name}-build.log" - echo " Running: make build..." - if make build > "${BUILD_LOG}" 2>&1; then - record_check "code_compiles" "pass" - else - record_check "code_compiles" "fail" - echo " Build output (last 20 lines):" - tail -20 "${BUILD_LOG}" | sed 's/^/ /' - fi - - # tests_pass - TEST_LOG="${ARTIFACT_DIR}/${case_name}-test.log" - echo " Running: make test..." - if make test > "${TEST_LOG}" 2>&1; then - record_check "tests_pass" "pass" - else - record_check "tests_pass" "fail" - echo " Test output (last 20 lines):" - tail -20 "${TEST_LOG}" | sed 's/^/ /' - fi - - # pr_created - PR_NUM="" - if [[ -f "${SHARED_DIR}/${case_name}.pr-number" ]]; then - PR_NUM=$(cat "${SHARED_DIR}/${case_name}.pr-number") - fi - if [[ -n "${PR_NUM}" ]]; then - record_check "pr_created" "pass" - echo " PR #${PR_NUM}" - else - PR_SEARCH=$(gh pr list --repo "${UPSTREAM_REPO}" --state open --search "${JIRA_ISSUE_KEY}" --json number --limit 1 2>/dev/null || echo "[]") - PR_NUM=$(echo "${PR_SEARCH}" | jq -r '.[0].number // empty' 2>/dev/null || echo "") - if [[ -n "${PR_NUM}" ]]; then - record_check "pr_created" "pass" - echo " Found PR #${PR_NUM}" - else - record_check "pr_created" "fail" - fi - fi - - # pr_description_exists - if [[ -s "${SHARED_DIR}/${case_name}.pr-description.md" ]]; then - record_check "pr_description_exists" "pass" - else - record_check "pr_description_exists" "fail" - fi - - # --- Diff-based scoring --- - echo " --- Scores ---" - - CLAUDE_FILES=$(git diff "origin/${BASE_BRANCH}" --name-only 2>/dev/null | sort || echo "") - EXPECTED_FILES=$(git diff "origin/${BASE_BRANCH}" "origin/${EXPECTED_BRANCH}" --name-only 2>/dev/null | sort || echo "") - - echo "${CLAUDE_FILES}" > /tmp/eval-claude-files.txt - echo "${EXPECTED_FILES}" > /tmp/eval-expected-files.txt - if [[ -n "${CLAUDE_FILES}" || -n "${EXPECTED_FILES}" ]]; then - OVERLAP=$(jaccard_similarity /tmp/eval-claude-files.txt /tmp/eval-expected-files.txt) - record_score "file_overlap" "${OVERLAP}" - else - OVERLAP="0.0" - record_score "file_overlap" "0.0" - fi - - CLAUDE_STAT=$(git diff "origin/${BASE_BRANCH}" --stat 2>/dev/null | tail -1 || echo "") - CLAUDE_TOTAL=$(diff_stat_total "${CLAUDE_STAT}") - - EXPECTED_STAT=$(git diff "origin/${BASE_BRANCH}" "origin/${EXPECTED_BRANCH}" --stat 2>/dev/null | tail -1 || echo "") - EXPECTED_TOTAL=$(diff_stat_total "${EXPECTED_STAT}") - - if [[ "${EXPECTED_TOTAL}" -gt 0 ]]; then - RATIO=$(python3 -c "print(f'{${CLAUDE_TOTAL} / ${EXPECTED_TOTAL}:.2f}')") - record_score "diff_size_ratio" "${RATIO}" - else - RATIO="N/A" - record_score "diff_size_ratio" "N/A" - fi - - CLAUDE_FUNCS=$(git diff "origin/${BASE_BRANCH}" -U0 2>/dev/null | grep -E '^\+.*func |^\+.*def |^\+.*function |^@@.*@@.*func |^@@.*@@.*def |^@@.*@@.*function ' | sed 's/.*func /func /;s/.*def /def /;s/.*function /function /' | sort -u || echo "") - EXPECTED_FUNCS=$(git diff "origin/${BASE_BRANCH}" "origin/${EXPECTED_BRANCH}" -U0 2>/dev/null | grep -E '^\+.*func |^\+.*def |^\+.*function |^@@.*@@.*func |^@@.*@@.*def |^@@.*@@.*function ' | sed 's/.*func /func /;s/.*def /def /;s/.*function /function /' | sort -u || echo "") - - echo "${CLAUDE_FUNCS}" > /tmp/eval-claude-funcs.txt - echo "${EXPECTED_FUNCS}" > /tmp/eval-expected-funcs.txt - if [[ -n "${CLAUDE_FUNCS}" || -n "${EXPECTED_FUNCS}" ]]; then - FUNC_OVERLAP=$(jaccard_similarity /tmp/eval-claude-funcs.txt /tmp/eval-expected-funcs.txt) - record_score "function_overlap" "${FUNC_OVERLAP}" - else - FUNC_OVERLAP="N/A" - record_score "function_overlap" "N/A" - fi - - # --- Score-based checks --- - if python3 -c "exit(0 if float('${OVERLAP}') >= 0.25 else 1)" 2>/dev/null; then - record_check "file_overlap_threshold" "pass" - else - record_check "file_overlap_threshold" "fail" - fi - - if [[ "${RATIO}" == "N/A" ]]; then - record_check "diff_size_threshold" "pass" - elif python3 -c "exit(0 if float('${RATIO}') >= 0.1 else 1)" 2>/dev/null; then - record_check "diff_size_threshold" "pass" - else - record_check "diff_size_threshold" "fail" - fi - - # --- Per-case summary --- - echo " Checks: ${CHECKS_PASS}/${CHECKS_TOTAL} passed" - - # --- Accumulate aggregates --- - TOTAL_CHECKS_PASS=$(( TOTAL_CHECKS_PASS + CHECKS_PASS )) - TOTAL_CHECKS_TOTAL=$(( TOTAL_CHECKS_TOTAL + CHECKS_TOTAL )) - - # JUnit test cases - for check_name in branch_created code_compiles tests_pass pr_created pr_description_exists file_overlap_threshold diff_size_threshold; do - result="${CHECKS[${check_name}]}" - if [[ "${result}" == "pass" ]]; then - ALL_JUNIT_TESTCASES="${ALL_JUNIT_TESTCASES} - " - else - ALL_JUNIT_TESTCASES="${ALL_JUNIT_TESTCASES} - - ${check_name} check did not pass. - " - fi - done - - # --- Per-case YAML --- - if [[ -n "${CLAUDE_FILES}" ]]; then - CLAUDE_FILES_YAML=$(echo "${CLAUDE_FILES}" | sed 's/^/ - /') - else - CLAUDE_FILES_YAML=" - (none)" - fi - if [[ -n "${EXPECTED_FILES}" ]]; then - EXPECTED_FILES_YAML=$(echo "${EXPECTED_FILES}" | sed 's/^/ - /') - else - EXPECTED_FILES_YAML=" - (none)" - fi - - cat > "${ARTIFACT_DIR}/eval-${EVAL_CASE}.yaml" <#${PR_NUM}" - else - PR_LINK_HTML="none" - fi - - # --- Per-case HTML detail (inline into summary) --- - CHECKS_HTML="" - check_icon() { if [[ "$1" == "pass" ]]; then echo "✅"; else echo "❌"; fi; } - for check_name in branch_created code_compiles tests_pass pr_created pr_description_exists file_overlap_threshold diff_size_threshold; do - result="${CHECKS[${check_name}]}" - CHECKS_HTML="${CHECKS_HTML}$(check_icon "${result}")${check_name}${result}" - done - - if [[ -n "${CLAUDE_FILES}" ]]; then - CLAUDE_FILES_HTML=$(echo "${CLAUDE_FILES}" | while IFS= read -r f; do echo "
  • ${f}
    • "; done) - else - CLAUDE_FILES_HTML="
  • (none)
    • " - fi - if [[ -n "${EXPECTED_FILES}" ]]; then - EXPECTED_FILES_HTML=$(echo "${EXPECTED_FILES}" | while IFS= read -r f; do echo "
  • ${f}
    • "; done) - else - EXPECTED_FILES_HTML="
  • (none)
    • " - fi - - ALL_CASE_DETAILS="${ALL_CASE_DETAILS} -
    -${EVAL_CASE} — ${JIRA_ISSUE_KEY} — ${CHECKS_PASS}/${CHECKS_TOTAL} checks -
    - - - - - - - -
    JIRA${JIRA_ISSUE_KEY}
    Claude branch${CLAUDE_BRANCH:-none}
    Base branch${BASE_BRANCH}
    Expected branch${EXPECTED_BRANCH}
    PR${PR_LINK_HTML}
    Expected diffview
    -

    Checks

    - - - ${CHECKS_HTML} -
    CheckResult
    -

    Scores

    - - - - - -
    MetricValue
    File overlap (Jaccard)${OVERLAP:-N/A}
    Diff size ratio (claude/expected)${RATIO:-N/A}
    Function overlap${FUNC_OVERLAP:-N/A}
    -

    Files Changed

    - - - - - - -
    Claude (${CLAUDE_TOTAL} lines)Expected (${EXPECTED_TOTAL} lines)
      ${CLAUDE_FILES_HTML}
      ${EXPECTED_FILES_HTML}
    -
    -
    " - - # --- Accumulate summary table row --- - ALL_SUMMARY_ROWS="${ALL_SUMMARY_ROWS}${EVAL_CASE}${JIRA_ISSUE_KEY}${CHECKS_PASS}/${CHECKS_TOTAL}${OVERLAP:-N/A}${RATIO:-N/A}${FUNC_OVERLAP:-N/A}${PR_LINK_HTML}diff" - -done - -# ============================================= -# AGGREGATE SUMMARY -# ============================================= -echo "" -echo "==========================================" -echo "Aggregate: ${TOTAL_CHECKS_PASS}/${TOTAL_CHECKS_TOTAL} checks passed across ${#CASE_LIST[@]} cases" -echo "==========================================" - -# --- JUnit XML --- -TOTAL_FAILURES=$(( TOTAL_CHECKS_TOTAL - TOTAL_CHECKS_PASS )) -cat > "${ARTIFACT_DIR}/junit_jira-solver-eval.xml" < - -${ALL_JUNIT_TESTCASES} - -JUNIT_EOF -echo "JUnit XML written to ${ARTIFACT_DIR}/junit_jira-solver-eval.xml" - -# --- Summary YAML --- -cat > "${ARTIFACT_DIR}/eval-summary.yaml" < "${ARTIFACT_DIR}/eval-summary.html" < - - -Jira-Solver Eval Summary - - - -

    Jira-Solver Eval Results

    -
    - ${TOTAL_CHECKS_PASS}/${TOTAL_CHECKS_TOTAL} checks passed across ${#CASE_LIST[@]} cases -
    - -

    Overview

    - - - ${ALL_SUMMARY_ROWS} -
    CaseJIRAChecksFile OverlapDiff RatioFunc OverlapPRExpected
    - -

    Case Details

    -${ALL_CASE_DETAILS} -
    jira-solver-eval
    - - -HTML_EOF -echo "HTML summary written to ${ARTIFACT_DIR}/eval-summary.html" - -if [[ "${TOTAL_CHECKS_PASS}" -lt "${TOTAL_CHECKS_TOTAL}" ]]; then - echo "FAILED: ${TOTAL_CHECKS_PASS}/${TOTAL_CHECKS_TOTAL} checks passed." - exit 1 +EVAL_CONFIG_DIR="/opt/ai-helpers/evals/jira-solver" + +if [[ ! -f "${EVAL_CONFIG_DIR}/eval.yaml" ]]; then + cat > /tmp/eval.yaml <<'EVALCFG' +name: jira-solver-eval +init: + repo: "${UPSTREAM_REPO}" +dataset: + path: /opt/ai-helpers/evals/jira-solver/cases +collect: + build_result: true + test_result: true + expected_branch_diff: true +judges: + - name: branch_created + - name: pr_exists + - name: build_passed + - name: test_passed + - name: file_overlap + - name: pr_description_exists + - name: diff_size_ratio + - name: function_overlap +thresholds: {} +EVALCFG + sed -i "s|\${UPSTREAM_REPO}|${UPSTREAM_REPO}|g" /tmp/eval.yaml + EVAL_CONFIG="/tmp/eval.yaml" +else + EVAL_CONFIG="${EVAL_CONFIG_DIR}/eval.yaml" fi +prow-agent-eval judge \ + --config="${EVAL_CONFIG}" \ + --shared-dir="${SHARED_DIR}" \ + --artifact-dir="${ARTIFACT_DIR}" + echo "=== TRT Eval Judge Complete ===" diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml index c0b32f4f545cb..8c352f8c6f849 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml @@ -13,7 +13,5 @@ ref: timeout: 1h0m0s grace_period: 1m0s documentation: |- - Eval judge step. Compares Claude's solution against the known-good - expected branch. Runs hardcoded checks (compiles, tests pass, PR - created) and diff-based scoring (file overlap, diff size ratio, - function overlap). Outputs JUnit XML and summary YAML. + Eval judge step. Uses prow-agent-eval CLI to collect post-agent + state, run judges, and emit JUnit XML, YAML, and HTML reports.