From 95dc59f98f6aa5907e01dcd6c31782c8ae8b494c Mon Sep 17 00:00:00 2001 From: smg247 Date: Fri, 7 Aug 2026 15:54:14 -0400 Subject: [PATCH 1/3] Replace eval step bash with prow-agent-eval CLI Bake the prow-agent-eval binary into the agentic-dev image and replace hundreds of lines of bash in the init, judge, and cleanup steps with CLI invocations. The CLI handles case discovery, metadata exchange, fixture SHA resolution, judge execution, and report generation. Co-Authored-By: Claude Opus 4.6 --- ...nshift-release-main__jira-solver-eval.yaml | 9 + ...shift-agentic-trt-eval-cleanup-commands.sh | 32 +- ...penshift-agentic-trt-eval-cleanup-ref.yaml | 5 +- ...penshift-agentic-trt-eval-init-commands.sh | 83 ++-- .../openshift-agentic-trt-eval-init-ref.yaml | 8 +- ...enshift-agentic-trt-eval-judge-commands.sh | 441 ++---------------- .../openshift-agentic-trt-eval-judge-ref.yaml | 6 +- 7 files changed, 89 insertions(+), 495 deletions(-) diff --git a/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml b/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml index a72e6761df238..0ea4fc7f14655 100644 --- a/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml +++ b/ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml @@ -7,6 +7,10 @@ base_images: name: nested-podman namespace: ci tag: latest + prow-agent-eval: + name: prow-agent-eval + namespace: ci + tag: prow-agent-eval sippy-devcontainer: name: sippy namespace: ci @@ -21,6 +25,7 @@ images: - dockerfile_literal: | FROM claude-ai-helpers AS ai-helpers FROM nested-podman AS podman + FROM prow-agent-eval AS proweval FROM sippy-devcontainer USER root RUN dnf install -y podman fuse-overlayfs shadow-utils openssl && dnf clean all @@ -36,6 +41,7 @@ images: /home/vscode/.claude/plugins/known_marketplaces.json RUN chgrp -R 0 /opt/ai-helpers /home/vscode/.claude && \ chmod -R g=u /opt/ai-helpers /home/vscode/.claude + COPY --from=proweval /usr/local/bin/prow-agent-eval /usr/local/bin/prow-agent-eval COPY ci-operator/step-registry/openshift/agentic/trt/jira-solver/openshift-agentic-trt-jira-solver-commands.sh /opt/scripts/solve.sh RUN chmod +x /opt/scripts/solve.sh RUN mkdir -p /workspace && chgrp 0 /workspace && chmod g=u /workspace @@ -52,6 +58,9 @@ images: nested-podman: as: - nested-podman + prow-agent-eval: + as: + - prow-agent-eval to: agentic-dev run_if_changed: ^ci-operator/step-registry/openshift/agentic/trt/ resources: diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh index 0373b3824354b..79cf0e8cd9fc8 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh @@ -16,35 +16,7 @@ if [[ -z "${GITHUB_TOKEN}" ]]; then exit 0 fi -if [[ ! -f "${SHARED_DIR}/eval-cases" ]]; then - echo "No eval-cases file found, skipping cleanup." - exit 0 -fi - -mapfile -t CASE_LIST < "${SHARED_DIR}/eval-cases" -echo "Cleaning up ${#CASE_LIST[@]} cases..." - -for case_name in "${CASE_LIST[@]}"; do - PR_NUM="" - if [[ -f "${SHARED_DIR}/${case_name}.pr-number" ]]; then - PR_NUM=$(cat "${SHARED_DIR}/${case_name}.pr-number") - fi - - if [[ -n "${PR_NUM}" ]]; then - echo "[${case_name}] Closing PR #${PR_NUM} and deleting branch..." - gh pr close "${PR_NUM}" --repo "${UPSTREAM_REPO}" --delete-branch || true - else - CLAUDE_BRANCH="" - if [[ -f "${SHARED_DIR}/${case_name}.claude-branch" ]]; then - CLAUDE_BRANCH=$(cat "${SHARED_DIR}/${case_name}.claude-branch") - fi - if [[ -n "${CLAUDE_BRANCH}" ]]; then - echo "[${case_name}] No PR found, deleting branch ${CLAUDE_BRANCH}..." - gh api "repos/${UPSTREAM_REPO}/git/refs/heads/${CLAUDE_BRANCH}" -X DELETE || true - else - echo "[${case_name}] Nothing to clean up." - fi - fi -done +prow-agent-eval cleanup \ + --shared-dir="${SHARED_DIR}" || true echo "=== TRT Eval Cleanup Complete ===" diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml index a3b04b3b20d1a..c475442c2a916 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml @@ -11,6 +11,5 @@ ref: cpu: 100m memory: 100Mi documentation: |- - Cleans up eval artifacts: closes the eval PR and deletes the - timestamped branch from the eval repo. Runs as a post-step so - cleanup happens even if the test phase fails. + Uses prow-agent-eval CLI to close PRs and delete branches. + Runs as a post-step so cleanup happens even if the test phase fails. diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh index 943de303871e4..1496f72b8363f 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh @@ -6,57 +6,52 @@ set -o pipefail echo "=== TRT Eval Init ===" -# --- Gangway override --- if [[ -n "${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE:-}" ]]; then - echo "Applying Gangway override: EVAL_CASE=${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE}" EVAL_CASE="${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE}" fi -ALL_CASES_DIR="/opt/ai-helpers/evals/jira-solver/cases" - -# --- Build case list --- -CASE_LIST=() -if [[ -n "${EVAL_CASE:-}" ]]; then - CASE_LIST=("${EVAL_CASE}") +set +x +GITHUB_TOKEN=$(cat "${SHARED_DIR}/gh-upstream-token") +export GITHUB_TOKEN +set -x + +EVAL_CONFIG_DIR="/opt/ai-helpers/evals/jira-solver" + +# Write eval config if not already present in the image +if [[ ! -f "${EVAL_CONFIG_DIR}/eval.yaml" ]]; then + cat > /tmp/eval.yaml <<'EVALCFG' +name: jira-solver-eval +init: + repo: "${UPSTREAM_REPO}" +dataset: + path: cases +collect: + build_result: true + test_result: true + expected_branch_diff: true +judges: + - name: branch_created + - name: pr_exists + - name: build_passed + - name: test_passed + - name: file_overlap +thresholds: {} +EVALCFG + sed -i "s|\${UPSTREAM_REPO}|${UPSTREAM_REPO}|g" /tmp/eval.yaml + EVAL_CONFIG="/tmp/eval.yaml" else - for d in "${ALL_CASES_DIR}"/*/; do - CASE_LIST+=("$(basename "$d")") - done + EVAL_CONFIG="${EVAL_CONFIG_DIR}/eval.yaml" fi -[[ ${#CASE_LIST[@]} -gt 0 ]] || { echo "ERROR: No eval cases found."; exit 1; } -echo "Cases to run: ${CASE_LIST[*]}" - -# --- Set up per-case metadata --- -# input.yaml must use flat "key: value" format — no nesting, quoting, or indentation -yaml_val() { grep "^${1}:" "$2" | cut -d' ' -f2-; } - -for case_name in "${CASE_LIST[@]}"; do - CASE_SRC="${ALL_CASES_DIR}/${case_name}" - [[ -d "${CASE_SRC}" ]] || { echo "ERROR: Case directory not found: ${CASE_SRC}"; exit 1; } - - INPUT_FILE="${CASE_SRC}/input.yaml" - [[ -f "${INPUT_FILE}" ]] || { echo "ERROR: input.yaml not found in ${CASE_SRC}"; exit 1; } - - JIRA_ISSUE_KEY=$(yaml_val jira_key "${INPUT_FILE}") - BASE_BRANCH=$(yaml_val base_branch "${INPUT_FILE}") - EXPECTED_BRANCH=$(yaml_val expected_branch "${INPUT_FILE}") - - [[ -n "${JIRA_ISSUE_KEY}" ]] || { echo "ERROR: ${case_name}: missing jira_key in input.yaml"; exit 1; } - [[ -n "${BASE_BRANCH}" ]] || { echo "ERROR: ${case_name}: missing base_branch in input.yaml"; exit 1; } - [[ -n "${EXPECTED_BRANCH}" ]] || { echo "ERROR: ${case_name}: missing expected_branch in input.yaml"; exit 1; } - - echo "${JIRA_ISSUE_KEY}" > "${SHARED_DIR}/${case_name}.jira-issue-key" - cp "${CASE_SRC}/jira-issue.json" "${SHARED_DIR}/${case_name}.jira-issue.json" - echo "${BASE_BRANCH}" > "${SHARED_DIR}/${case_name}.eval-base-branch" - echo "${EXPECTED_BRANCH}" > "${SHARED_DIR}/${case_name}.eval-expected-branch" - echo "${case_name}" > "${SHARED_DIR}/${case_name}.eval-case" - - SUMMARY=$(jq -r '.fields.summary // .summary // "N/A"' "${SHARED_DIR}/${case_name}.jira-issue.json") - echo " ${case_name}: ${JIRA_ISSUE_KEY} - ${SUMMARY}" -done +CASE_FLAG="" +if [[ -n "${EVAL_CASE:-}" ]]; then + CASE_FLAG="--case=${EVAL_CASE}" +fi -# --- Write case list for downstream steps --- -printf '%s\n' "${CASE_LIST[@]}" > "${SHARED_DIR}/eval-cases" +prow-agent-eval init \ + --config="${EVAL_CONFIG}" \ + --shared-dir="${SHARED_DIR}" \ + --mode=solve \ + ${CASE_FLAG} echo "=== TRT Eval Init Complete ===" diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml index 452868fad2c93..a578caeaabd60 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml @@ -3,6 +3,9 @@ ref: from: agentic-dev commands: openshift-agentic-trt-eval-init-commands.sh env: + - name: UPSTREAM_REPO + documentation: |- + Target repo for the eval (org/repo format). - name: EVAL_CASE default: "" documentation: |- @@ -19,6 +22,5 @@ ref: memory: 256Mi timeout: 10m0s documentation: |- - Eval init step. Reads case configuration from the ai-helpers image, - copies JIRA JSON snapshots to SHARED_DIR, and writes per-case - metadata (jira key, base/expected branches) for downstream steps. + Eval init step. Uses prow-agent-eval CLI to discover cases, + validate inputs, and write per-case metadata to SHARED_DIR. diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh index 54b7394a8d366..2e51c4c5731f1 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh @@ -6,422 +6,41 @@ set -o pipefail echo "=== TRT Eval Judge ===" -# --- Read case list --- -mapfile -t CASE_LIST < "${SHARED_DIR}/eval-cases" -echo "Cases to judge (${#CASE_LIST[@]}): ${CASE_LIST[*]}" - -# --- Clone repo template --- -TEMPLATE_DIR="/tmp/eval-repo-template" -git clone "https://github.com/${UPSTREAM_REPO}.git" "${TEMPLATE_DIR}" - set +x GITHUB_TOKEN=$(cat "${SHARED_DIR}/gh-upstream-token") export GITHUB_TOKEN set -x -# --- Utilities --- -diff_stat_total() { - local stat_line=$1 - local ins del - ins=$(echo "${stat_line}" | grep -oE '[0-9]+ insertion' | grep -oE '[0-9]+' || echo "0") - del=$(echo "${stat_line}" | grep -oE '[0-9]+ deletion' | grep -oE '[0-9]+' || echo "0") - echo $(( ins + del )) -} - -jaccard_similarity() { - local file_a=$1 file_b=$2 - python3 -c " -import sys -a = set(open(sys.argv[1]).read().strip().split('\n')) if open(sys.argv[1]).read().strip() else set() -b = set(open(sys.argv[2]).read().strip().split('\n')) if open(sys.argv[2]).read().strip() else set() -if not a and not b: print('1.0') -elif not a or not b: print('0.0') -else: print(f'{len(a & b) / len(a | b):.2f}') -" "${file_a}" "${file_b}" -} - -# --- Aggregate accumulators --- -TOTAL_CHECKS_PASS=0 -TOTAL_CHECKS_TOTAL=0 -ALL_JUNIT_TESTCASES="" -ALL_SUMMARY_ROWS="" -ALL_CASE_DETAILS="" - -# ============================================= -# JUDGE EACH CASE -# ============================================= -for case_name in "${CASE_LIST[@]}"; do - echo "" - echo "==========================================" - echo "Judging: ${case_name}" - echo "==========================================" - - EVAL_CASE=$(cat "${SHARED_DIR}/${case_name}.eval-case") - BASE_BRANCH=$(cat "${SHARED_DIR}/${case_name}.eval-base-branch") - EXPECTED_BRANCH=$(cat "${SHARED_DIR}/${case_name}.eval-expected-branch") - JIRA_ISSUE_KEY=$(cat "${SHARED_DIR}/${case_name}.jira-issue-key") - - echo " JIRA: ${JIRA_ISSUE_KEY} | Base: ${BASE_BRANCH} | Expected: ${EXPECTED_BRANCH}" - - # Set up per-case workspace from template - CASE_WORKDIR="/workspace/${case_name}" - cp -r "${TEMPLATE_DIR}" "${CASE_WORKDIR}" - cd "${CASE_WORKDIR}" - - # Check out Claude's branch - CLAUDE_BRANCH="" - if [[ -f "${SHARED_DIR}/${case_name}.claude-branch" ]]; then - CLAUDE_BRANCH=$(cat "${SHARED_DIR}/${case_name}.claude-branch") - git fetch origin "${CLAUDE_BRANCH}" - git checkout "${CLAUDE_BRANCH}" - fi - echo " Claude's branch: ${CLAUDE_BRANCH:-}" - - git fetch origin "${BASE_BRANCH}" "${EXPECTED_BRANCH}" 2>/dev/null || true - - # --- Per-case results --- - unset CHECKS 2>/dev/null || true - declare -A CHECKS - CHECKS_PASS=0 - CHECKS_TOTAL=0 - SCORES="" - - record_check() { - local name=$1 result=$2 - CHECKS["${name}"]="${result}" - CHECKS_TOTAL=$(( CHECKS_TOTAL + 1 )) - if [[ "${result}" == "pass" ]]; then - CHECKS_PASS=$(( CHECKS_PASS + 1 )) - echo " [PASS] ${name}" - else - echo " [FAIL] ${name}" - fi - } - - record_score() { - local name=$1 value=$2 - SCORES="${SCORES}${name}: ${value}\n" - echo " [SCORE] ${name}: ${value}" - } - - # --- Hardcoded checks --- - echo " --- Checks ---" - - # branch_created - if [[ -n "${CLAUDE_BRANCH}" && "${CLAUDE_BRANCH}" != "main" && "${CLAUDE_BRANCH}" != "master" && "${CLAUDE_BRANCH}" != "${BASE_BRANCH}" ]]; then - record_check "branch_created" "pass" - else - record_check "branch_created" "fail" - fi - - # code_compiles - BUILD_LOG="${ARTIFACT_DIR}/${case_name}-build.log" - echo " Running: make build..." - if make build > "${BUILD_LOG}" 2>&1; then - record_check "code_compiles" "pass" - else - record_check "code_compiles" "fail" - echo " Build output (last 20 lines):" - tail -20 "${BUILD_LOG}" | sed 's/^/ /' - fi - - # tests_pass - TEST_LOG="${ARTIFACT_DIR}/${case_name}-test.log" - echo " Running: make test..." - if make test > "${TEST_LOG}" 2>&1; then - record_check "tests_pass" "pass" - else - record_check "tests_pass" "fail" - echo " Test output (last 20 lines):" - tail -20 "${TEST_LOG}" | sed 's/^/ /' - fi - - # pr_created - PR_NUM="" - if [[ -f "${SHARED_DIR}/${case_name}.pr-number" ]]; then - PR_NUM=$(cat "${SHARED_DIR}/${case_name}.pr-number") - fi - if [[ -n "${PR_NUM}" ]]; then - record_check "pr_created" "pass" - echo " PR #${PR_NUM}" - else - PR_SEARCH=$(gh pr list --repo "${UPSTREAM_REPO}" --state open --search "${JIRA_ISSUE_KEY}" --json number --limit 1 2>/dev/null || echo "[]") - PR_NUM=$(echo "${PR_SEARCH}" | jq -r '.[0].number // empty' 2>/dev/null || echo "") - if [[ -n "${PR_NUM}" ]]; then - record_check "pr_created" "pass" - echo " Found PR #${PR_NUM}" - else - record_check "pr_created" "fail" - fi - fi - - # pr_description_exists - if [[ -s "${SHARED_DIR}/${case_name}.pr-description.md" ]]; then - record_check "pr_description_exists" "pass" - else - record_check "pr_description_exists" "fail" - fi - - # --- Diff-based scoring --- - echo " --- Scores ---" - - CLAUDE_FILES=$(git diff "origin/${BASE_BRANCH}" --name-only 2>/dev/null | sort || echo "") - EXPECTED_FILES=$(git diff "origin/${BASE_BRANCH}" "origin/${EXPECTED_BRANCH}" --name-only 2>/dev/null | sort || echo "") - - echo "${CLAUDE_FILES}" > /tmp/eval-claude-files.txt - echo "${EXPECTED_FILES}" > /tmp/eval-expected-files.txt - if [[ -n "${CLAUDE_FILES}" || -n "${EXPECTED_FILES}" ]]; then - OVERLAP=$(jaccard_similarity /tmp/eval-claude-files.txt /tmp/eval-expected-files.txt) - record_score "file_overlap" "${OVERLAP}" - else - OVERLAP="0.0" - record_score "file_overlap" "0.0" - fi - - CLAUDE_STAT=$(git diff "origin/${BASE_BRANCH}" --stat 2>/dev/null | tail -1 || echo "") - CLAUDE_TOTAL=$(diff_stat_total "${CLAUDE_STAT}") - - EXPECTED_STAT=$(git diff "origin/${BASE_BRANCH}" "origin/${EXPECTED_BRANCH}" --stat 2>/dev/null | tail -1 || echo "") - EXPECTED_TOTAL=$(diff_stat_total "${EXPECTED_STAT}") - - if [[ "${EXPECTED_TOTAL}" -gt 0 ]]; then - RATIO=$(python3 -c "print(f'{${CLAUDE_TOTAL} / ${EXPECTED_TOTAL}:.2f}')") - record_score "diff_size_ratio" "${RATIO}" - else - RATIO="N/A" - record_score "diff_size_ratio" "N/A" - fi - - CLAUDE_FUNCS=$(git diff "origin/${BASE_BRANCH}" -U0 2>/dev/null | grep -E '^\+.*func |^\+.*def |^\+.*function |^@@.*@@.*func |^@@.*@@.*def |^@@.*@@.*function ' | sed 's/.*func /func /;s/.*def /def /;s/.*function /function /' | sort -u || echo "") - EXPECTED_FUNCS=$(git diff "origin/${BASE_BRANCH}" "origin/${EXPECTED_BRANCH}" -U0 2>/dev/null | grep -E '^\+.*func |^\+.*def |^\+.*function |^@@.*@@.*func |^@@.*@@.*def |^@@.*@@.*function ' | sed 's/.*func /func /;s/.*def /def /;s/.*function /function /' | sort -u || echo "") - - echo "${CLAUDE_FUNCS}" > /tmp/eval-claude-funcs.txt - echo "${EXPECTED_FUNCS}" > /tmp/eval-expected-funcs.txt - if [[ -n "${CLAUDE_FUNCS}" || -n "${EXPECTED_FUNCS}" ]]; then - FUNC_OVERLAP=$(jaccard_similarity /tmp/eval-claude-funcs.txt /tmp/eval-expected-funcs.txt) - record_score "function_overlap" "${FUNC_OVERLAP}" - else - FUNC_OVERLAP="N/A" - record_score "function_overlap" "N/A" - fi - - # --- Score-based checks --- - if python3 -c "exit(0 if float('${OVERLAP}') >= 0.25 else 1)" 2>/dev/null; then - record_check "file_overlap_threshold" "pass" - else - record_check "file_overlap_threshold" "fail" - fi - - if [[ "${RATIO}" == "N/A" ]]; then - record_check "diff_size_threshold" "pass" - elif python3 -c "exit(0 if float('${RATIO}') >= 0.1 else 1)" 2>/dev/null; then - record_check "diff_size_threshold" "pass" - else - record_check "diff_size_threshold" "fail" - fi - - # --- Per-case summary --- - echo " Checks: ${CHECKS_PASS}/${CHECKS_TOTAL} passed" - - # --- Accumulate aggregates --- - TOTAL_CHECKS_PASS=$(( TOTAL_CHECKS_PASS + CHECKS_PASS )) - TOTAL_CHECKS_TOTAL=$(( TOTAL_CHECKS_TOTAL + CHECKS_TOTAL )) - - # JUnit test cases - for check_name in branch_created code_compiles tests_pass pr_created pr_description_exists file_overlap_threshold diff_size_threshold; do - result="${CHECKS[${check_name}]}" - if [[ "${result}" == "pass" ]]; then - ALL_JUNIT_TESTCASES="${ALL_JUNIT_TESTCASES} - " - else - ALL_JUNIT_TESTCASES="${ALL_JUNIT_TESTCASES} - - ${check_name} check did not pass. - " - fi - done - - # --- Per-case YAML --- - if [[ -n "${CLAUDE_FILES}" ]]; then - CLAUDE_FILES_YAML=$(echo "${CLAUDE_FILES}" | sed 's/^/ - /') - else - CLAUDE_FILES_YAML=" - (none)" - fi - if [[ -n "${EXPECTED_FILES}" ]]; then - EXPECTED_FILES_YAML=$(echo "${EXPECTED_FILES}" | sed 's/^/ - /') - else - EXPECTED_FILES_YAML=" - (none)" - fi - - cat > "${ARTIFACT_DIR}/eval-${EVAL_CASE}.yaml" <#${PR_NUM}" - else - PR_LINK_HTML="none" - fi - - # --- Per-case HTML detail (inline into summary) --- - CHECKS_HTML="" - check_icon() { if [[ "$1" == "pass" ]]; then echo "✅"; else echo "❌"; fi; } - for check_name in branch_created code_compiles tests_pass pr_created pr_description_exists file_overlap_threshold diff_size_threshold; do - result="${CHECKS[${check_name}]}" - CHECKS_HTML="${CHECKS_HTML}$(check_icon "${result}")${check_name}${result}" - done - - if [[ -n "${CLAUDE_FILES}" ]]; then - CLAUDE_FILES_HTML=$(echo "${CLAUDE_FILES}" | while IFS= read -r f; do echo "
  • ${f}
    • "; done) - else - CLAUDE_FILES_HTML="
  • (none)
    • " - fi - if [[ -n "${EXPECTED_FILES}" ]]; then - EXPECTED_FILES_HTML=$(echo "${EXPECTED_FILES}" | while IFS= read -r f; do echo "
  • ${f}
    • "; done) - else - EXPECTED_FILES_HTML="
  • (none)
    • " - fi - - ALL_CASE_DETAILS="${ALL_CASE_DETAILS} -
    -${EVAL_CASE} — ${JIRA_ISSUE_KEY} — ${CHECKS_PASS}/${CHECKS_TOTAL} checks -
    - - - - - - - -
    JIRA${JIRA_ISSUE_KEY}
    Claude branch${CLAUDE_BRANCH:-none}
    Base branch${BASE_BRANCH}
    Expected branch${EXPECTED_BRANCH}
    PR${PR_LINK_HTML}
    Expected diffview
    -

    Checks

    - - - ${CHECKS_HTML} -
    CheckResult
    -

    Scores

    - - - - - -
    MetricValue
    File overlap (Jaccard)${OVERLAP:-N/A}
    Diff size ratio (claude/expected)${RATIO:-N/A}
    Function overlap${FUNC_OVERLAP:-N/A}
    -

    Files Changed

    - - - - - - -
    Claude (${CLAUDE_TOTAL} lines)Expected (${EXPECTED_TOTAL} lines)
      ${CLAUDE_FILES_HTML}
      ${EXPECTED_FILES_HTML}
    -
    -
    " - - # --- Accumulate summary table row --- - ALL_SUMMARY_ROWS="${ALL_SUMMARY_ROWS}${EVAL_CASE}${JIRA_ISSUE_KEY}${CHECKS_PASS}/${CHECKS_TOTAL}${OVERLAP:-N/A}${RATIO:-N/A}${FUNC_OVERLAP:-N/A}${PR_LINK_HTML}diff" - -done - -# ============================================= -# AGGREGATE SUMMARY -# ============================================= -echo "" -echo "==========================================" -echo "Aggregate: ${TOTAL_CHECKS_PASS}/${TOTAL_CHECKS_TOTAL} checks passed across ${#CASE_LIST[@]} cases" -echo "==========================================" - -# --- JUnit XML --- -TOTAL_FAILURES=$(( TOTAL_CHECKS_TOTAL - TOTAL_CHECKS_PASS )) -cat > "${ARTIFACT_DIR}/junit_jira-solver-eval.xml" < - -${ALL_JUNIT_TESTCASES} - -JUNIT_EOF -echo "JUnit XML written to ${ARTIFACT_DIR}/junit_jira-solver-eval.xml" - -# --- Summary YAML --- -cat > "${ARTIFACT_DIR}/eval-summary.yaml" < "${ARTIFACT_DIR}/eval-summary.html" < - - -Jira-Solver Eval Summary - - - -

    Jira-Solver Eval Results

    -
    - ${TOTAL_CHECKS_PASS}/${TOTAL_CHECKS_TOTAL} checks passed across ${#CASE_LIST[@]} cases -
    - -

    Overview

    - - - ${ALL_SUMMARY_ROWS} -
    CaseJIRAChecksFile OverlapDiff RatioFunc OverlapPRExpected
    - -

    Case Details

    -${ALL_CASE_DETAILS} -
    jira-solver-eval
    - - -HTML_EOF -echo "HTML summary written to ${ARTIFACT_DIR}/eval-summary.html" - -if [[ "${TOTAL_CHECKS_PASS}" -lt "${TOTAL_CHECKS_TOTAL}" ]]; then - echo "FAILED: ${TOTAL_CHECKS_PASS}/${TOTAL_CHECKS_TOTAL} checks passed." - exit 1 +EVAL_CONFIG_DIR="/opt/ai-helpers/evals/jira-solver" + +if [[ ! -f "${EVAL_CONFIG_DIR}/eval.yaml" ]]; then + cat > /tmp/eval.yaml <<'EVALCFG' +name: jira-solver-eval +init: + repo: "${UPSTREAM_REPO}" +dataset: + path: cases +collect: + build_result: true + test_result: true + expected_branch_diff: true +judges: + - name: branch_created + - name: pr_exists + - name: build_passed + - name: test_passed + - name: file_overlap +thresholds: {} +EVALCFG + sed -i "s|\${UPSTREAM_REPO}|${UPSTREAM_REPO}|g" /tmp/eval.yaml + EVAL_CONFIG="/tmp/eval.yaml" +else + EVAL_CONFIG="${EVAL_CONFIG_DIR}/eval.yaml" fi +prow-agent-eval judge \ + --config="${EVAL_CONFIG}" \ + --shared-dir="${SHARED_DIR}" \ + --artifact-dir="${ARTIFACT_DIR}" + echo "=== TRT Eval Judge Complete ===" diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml index c0b32f4f545cb..8c352f8c6f849 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-ref.yaml @@ -13,7 +13,5 @@ ref: timeout: 1h0m0s grace_period: 1m0s documentation: |- - Eval judge step. Compares Claude's solution against the known-good - expected branch. Runs hardcoded checks (compiles, tests pass, PR - created) and diff-based scoring (file overlap, diff size ratio, - function overlap). Outputs JUnit XML and summary YAML. + Eval judge step. Uses prow-agent-eval CLI to collect post-agent + state, run judges, and emit JUnit XML, YAML, and HTML reports. From 0d4524a3df4aa611b7cbf923f67636721074b804 Mon Sep 17 00:00:00 2001 From: smg247 Date: Fri, 7 Aug 2026 16:38:22 -0400 Subject: [PATCH 2/3] Fix eval dataset path to use absolute path to ai-helpers cases The inline eval config used a relative `cases` path which resolved against /tmp (where the config is written), not ai-helpers. Co-Authored-By: Claude Opus 4.6 --- .../trt/eval/init/openshift-agentic-trt-eval-init-commands.sh | 2 +- .../trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh index 1496f72b8363f..efb0e9dfab88a 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh @@ -24,7 +24,7 @@ name: jira-solver-eval init: repo: "${UPSTREAM_REPO}" dataset: - path: cases + path: /opt/ai-helpers/evals/jira-solver/cases collect: build_result: true test_result: true diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh index 2e51c4c5731f1..e7d5e380bc6e9 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh @@ -19,7 +19,7 @@ name: jira-solver-eval init: repo: "${UPSTREAM_REPO}" dataset: - path: cases + path: /opt/ai-helpers/evals/jira-solver/cases collect: build_result: true test_result: true From 89e01fcc0f31954e7a651f05536328e45ff1bf61 Mon Sep 17 00:00:00 2001 From: smg247 Date: Mon, 10 Aug 2026 10:06:38 -0400 Subject: [PATCH 3/3] Add new judges to eval inline config Add pr_description_exists, diff_size_ratio, and function_overlap to the inline eval config in both init and judge step scripts. Co-Authored-By: Claude Opus 4.6 --- .../trt/eval/init/openshift-agentic-trt-eval-init-commands.sh | 3 +++ .../eval/judge/openshift-agentic-trt-eval-judge-commands.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh index efb0e9dfab88a..f5f190122979f 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh @@ -35,6 +35,9 @@ judges: - name: build_passed - name: test_passed - name: file_overlap + - name: pr_description_exists + - name: diff_size_ratio + - name: function_overlap thresholds: {} EVALCFG sed -i "s|\${UPSTREAM_REPO}|${UPSTREAM_REPO}|g" /tmp/eval.yaml diff --git a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh index e7d5e380bc6e9..ff8442c2b9c42 100644 --- a/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh +++ b/ci-operator/step-registry/openshift/agentic/trt/eval/judge/openshift-agentic-trt-eval-judge-commands.sh @@ -30,6 +30,9 @@ judges: - name: build_passed - name: test_passed - name: file_overlap + - name: pr_description_exists + - name: diff_size_ratio + - name: function_overlap thresholds: {} EVALCFG sed -i "s|\${UPSTREAM_REPO}|${UPSTREAM_REPO}|g" /tmp/eval.yaml