WIP: Replace eval step bash with prow-agent-eval CLI

ci-operator/config/openshift/release/openshift-release-main__jira-solver-eval.yaml

@@ -7,6 +7,10 @@ base_images:
     name: nested-podman
     namespace: ci
     tag: latest
+  prow-agent-eval:
+    name: prow-agent-eval
+    namespace: ci
+    tag: prow-agent-eval
   sippy-devcontainer:
     name: sippy
     namespace: ci
@@ -21,6 +25,7 @@ images:
   - dockerfile_literal: |
       FROM claude-ai-helpers AS ai-helpers
       FROM nested-podman AS podman
+      FROM prow-agent-eval AS proweval
       FROM sippy-devcontainer
       USER root
       RUN dnf install -y podman fuse-overlayfs shadow-utils openssl && dnf clean all
@@ -36,6 +41,7 @@ images:
            /home/vscode/.claude/plugins/known_marketplaces.json
       RUN chgrp -R 0 /opt/ai-helpers /home/vscode/.claude && \
           chmod -R g=u /opt/ai-helpers /home/vscode/.claude
+      COPY --from=proweval /usr/local/bin/prow-agent-eval /usr/local/bin/prow-agent-eval
       COPY ci-operator/step-registry/openshift/agentic/trt/jira-solver/openshift-agentic-trt-jira-solver-commands.sh /opt/scripts/solve.sh
       RUN chmod +x /opt/scripts/solve.sh
       RUN mkdir -p /workspace && chgrp 0 /workspace && chmod g=u /workspace
@@ -52,6 +58,9 @@ images:
       nested-podman:
         as:
         - nested-podman
+      prow-agent-eval:
+        as:
+        - prow-agent-eval
     to: agentic-dev
   run_if_changed: ^ci-operator/step-registry/openshift/agentic/trt/
 resources:

ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-commands.sh

@@ -16,35 +16,7 @@ if [[ -z "${GITHUB_TOKEN}" ]]; then
     exit 0
 fi
 
-if [[ ! -f "${SHARED_DIR}/eval-cases" ]]; then
-    echo "No eval-cases file found, skipping cleanup."
-    exit 0
-fi
-
-mapfile -t CASE_LIST < "${SHARED_DIR}/eval-cases"
-echo "Cleaning up ${#CASE_LIST[@]} cases..."
-
-for case_name in "${CASE_LIST[@]}"; do
-    PR_NUM=""
-    if [[ -f "${SHARED_DIR}/${case_name}.pr-number" ]]; then
-        PR_NUM=$(cat "${SHARED_DIR}/${case_name}.pr-number")
-    fi
-
-    if [[ -n "${PR_NUM}" ]]; then
-        echo "[${case_name}] Closing PR #${PR_NUM} and deleting branch..."
-        gh pr close "${PR_NUM}" --repo "${UPSTREAM_REPO}" --delete-branch || true
-    else
-        CLAUDE_BRANCH=""
-        if [[ -f "${SHARED_DIR}/${case_name}.claude-branch" ]]; then
-            CLAUDE_BRANCH=$(cat "${SHARED_DIR}/${case_name}.claude-branch")
-        fi
-        if [[ -n "${CLAUDE_BRANCH}" ]]; then
-            echo "[${case_name}] No PR found, deleting branch ${CLAUDE_BRANCH}..."
-            gh api "repos/${UPSTREAM_REPO}/git/refs/heads/${CLAUDE_BRANCH}" -X DELETE || true
-        else
-            echo "[${case_name}] Nothing to clean up."
-        fi
-    fi
-done
+prow-agent-eval cleanup \
+    --shared-dir="${SHARED_DIR}" || true
 
 echo "=== TRT Eval Cleanup Complete ==="

ci-operator/step-registry/openshift/agentic/trt/eval/cleanup/openshift-agentic-trt-eval-cleanup-ref.yaml

@@ -11,6 +11,5 @@ ref:
       cpu: 100m
       memory: 100Mi
   documentation: |-
-    Cleans up eval artifacts: closes the eval PR and deletes the
-    timestamped branch from the eval repo. Runs as a post-step so
-    cleanup happens even if the test phase fails.
+    Uses prow-agent-eval CLI to close PRs and delete branches.
+    Runs as a post-step so cleanup happens even if the test phase fails.

ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-commands.sh

@@ -6,57 +6,55 @@ set -o pipefail
 
 echo "=== TRT Eval Init ==="
 
-# --- Gangway override ---
 if [[ -n "${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE:-}" ]]; then
-    echo "Applying Gangway override: EVAL_CASE=${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE}"
     EVAL_CASE="${MULTISTAGE_PARAM_OVERRIDE_EVAL_CASE}"
 fi
 
-ALL_CASES_DIR="/opt/ai-helpers/evals/jira-solver/cases"
-
-# --- Build case list ---
-CASE_LIST=()
-if [[ -n "${EVAL_CASE:-}" ]]; then
-    CASE_LIST=("${EVAL_CASE}")
+set +x
+GITHUB_TOKEN=$(cat "${SHARED_DIR}/gh-upstream-token")
+export GITHUB_TOKEN
+set -x
+
+EVAL_CONFIG_DIR="/opt/ai-helpers/evals/jira-solver"
+
+# Write eval config if not already present in the image
+if [[ ! -f "${EVAL_CONFIG_DIR}/eval.yaml" ]]; then
+    cat > /tmp/eval.yaml <<'EVALCFG'
+name: jira-solver-eval
+init:
+  repo: "${UPSTREAM_REPO}"
+dataset:
+  path: /opt/ai-helpers/evals/jira-solver/cases
+collect:
+  build_result: true
+  test_result: true
+  expected_branch_diff: true
+judges:
+  - name: branch_created
+  - name: pr_exists
+  - name: build_passed
+  - name: test_passed
+  - name: file_overlap
+  - name: pr_description_exists
+  - name: diff_size_ratio
+  - name: function_overlap
+thresholds: {}
+EVALCFG
+    sed -i "s|\${UPSTREAM_REPO}|${UPSTREAM_REPO}|g" /tmp/eval.yaml
+    EVAL_CONFIG="/tmp/eval.yaml"
 else
-    for d in "${ALL_CASES_DIR}"/*/; do
-        CASE_LIST+=("$(basename "$d")")
-    done
+    EVAL_CONFIG="${EVAL_CONFIG_DIR}/eval.yaml"
 fi
 
-[[ ${#CASE_LIST[@]} -gt 0 ]] || { echo "ERROR: No eval cases found."; exit 1; }
-echo "Cases to run: ${CASE_LIST[*]}"
-
-# --- Set up per-case metadata ---
-# input.yaml must use flat "key: value" format — no nesting, quoting, or indentation
-yaml_val() { grep "^${1}:" "$2" | cut -d' ' -f2-; }
-
-for case_name in "${CASE_LIST[@]}"; do
-    CASE_SRC="${ALL_CASES_DIR}/${case_name}"
-    [[ -d "${CASE_SRC}" ]] || { echo "ERROR: Case directory not found: ${CASE_SRC}"; exit 1; }
-
-    INPUT_FILE="${CASE_SRC}/input.yaml"
-    [[ -f "${INPUT_FILE}" ]] || { echo "ERROR: input.yaml not found in ${CASE_SRC}"; exit 1; }
-
-    JIRA_ISSUE_KEY=$(yaml_val jira_key "${INPUT_FILE}")
-    BASE_BRANCH=$(yaml_val base_branch "${INPUT_FILE}")
-    EXPECTED_BRANCH=$(yaml_val expected_branch "${INPUT_FILE}")
-
-    [[ -n "${JIRA_ISSUE_KEY}" ]] || { echo "ERROR: ${case_name}: missing jira_key in input.yaml"; exit 1; }
-    [[ -n "${BASE_BRANCH}" ]] || { echo "ERROR: ${case_name}: missing base_branch in input.yaml"; exit 1; }
-    [[ -n "${EXPECTED_BRANCH}" ]] || { echo "ERROR: ${case_name}: missing expected_branch in input.yaml"; exit 1; }
-
-    echo "${JIRA_ISSUE_KEY}" > "${SHARED_DIR}/${case_name}.jira-issue-key"
-    cp "${CASE_SRC}/jira-issue.json" "${SHARED_DIR}/${case_name}.jira-issue.json"
-    echo "${BASE_BRANCH}" > "${SHARED_DIR}/${case_name}.eval-base-branch"
-    echo "${EXPECTED_BRANCH}" > "${SHARED_DIR}/${case_name}.eval-expected-branch"
-    echo "${case_name}" > "${SHARED_DIR}/${case_name}.eval-case"
-
-    SUMMARY=$(jq -r '.fields.summary // .summary // "N/A"' "${SHARED_DIR}/${case_name}.jira-issue.json")
-    echo "  ${case_name}: ${JIRA_ISSUE_KEY} - ${SUMMARY}"
-done
+CASE_FLAG=""
+if [[ -n "${EVAL_CASE:-}" ]]; then
+    CASE_FLAG="--case=${EVAL_CASE}"
+fi
 
-# --- Write case list for downstream steps ---
-printf '%s\n' "${CASE_LIST[@]}" > "${SHARED_DIR}/eval-cases"
+prow-agent-eval init \
+    --config="${EVAL_CONFIG}" \
+    --shared-dir="${SHARED_DIR}" \
+    --mode=solve \
+    ${CASE_FLAG}
 
 echo "=== TRT Eval Init Complete ==="

ci-operator/step-registry/openshift/agentic/trt/eval/init/openshift-agentic-trt-eval-init-ref.yaml

@@ -3,6 +3,9 @@ ref:
   from: agentic-dev
   commands: openshift-agentic-trt-eval-init-commands.sh
   env:
+  - name: UPSTREAM_REPO
+    documentation: |-
+      Target repo for the eval (org/repo format).
   - name: EVAL_CASE
     default: ""
     documentation: |-
@@ -19,6 +22,5 @@ ref:
       memory: 256Mi
   timeout: 10m0s
   documentation: |-
-    Eval init step. Reads case configuration from the ai-helpers image,
-    copies JIRA JSON snapshots to SHARED_DIR, and writes per-case
-    metadata (jira key, base/expected branches) for downstream steps.
+    Eval init step. Uses prow-agent-eval CLI to discover cases,
+    validate inputs, and write per-case metadata to SHARED_DIR.