test(e2e): add runner matrix live fixture

e2e/runner-matrix/.gitignore

@@ -0,0 +1,4 @@
+runs/
+evals/reports/
+evals/judges/
+evals/scripts/__pycache__/

e2e/runner-matrix/assemblies/live-confirmation.yaml

@@ -0,0 +1,25 @@
+name: live-confirmation
+model: claude-haiku-4-5
+
+matrix:
+  runner:
+    - { name: anthropic-haiku-4-5, provider: anthropic, display: "Haiku 4.5", model: claude-haiku-4-5 }
+    - { name: anthropic-sonnet-4-6, provider: anthropic, display: "Sonnet 4.6", model: claude-sonnet-4-6 }
+    - { name: anthropic-sonnet-5, provider: anthropic, display: "Sonnet 5", model: claude-sonnet-5 }
+    - { name: anthropic-opus-4-8, provider: anthropic, display: "Opus 4.8", model: claude-opus-4-8 }
+    - { name: anthropic-fable, provider: anthropic, display: "Fable", model: claude-fable-5 }
+    - { name: openai-gpt-5-5, provider: openai, display: "GPT-5.5", model: gpt-5.5 }
+    - { name: openai-gpt-5-4, provider: openai, display: "GPT-5.4", model: gpt-5.4 }
+    - { name: openai-gpt-5-4-mini, provider: openai, display: "GPT-5.4-mini", model: gpt-5.4-mini }
+    - { name: google-gemini-3-5-flash, provider: google, display: "Gemini 3.5 Flash", model: gemini-3.5-flash }
+    - { name: google-gemini-3-1-pro-preview, provider: google, display: "Gemini 3.1 Pro (Preview)", model: gemini-3.1-pro-preview }
+    - { name: google-gemini-3-1-flash-lite, provider: google, display: "Gemini 3.1 Flash-Lite", model: gemini-3.1-flash-lite }
+    - { name: google-gemini-2-5-pro, provider: google, display: "Gemini 2.5 Pro", model: gemini-2.5-pro }
+    - { name: bedrock-llama-3-3, provider: bedrock, display: "Llama 3.3", model: "bedrock:meta.llama3-3-70b-instruct-v1:0" }
+    - { name: bedrock-llama-4-scout, provider: bedrock, display: "Llama 4 Scout", model: "bedrock:meta.llama4-scout-17b-instruct-v1:0" }
+    - { name: bedrock-mistral-large-3, provider: bedrock, display: "Mistral Large 3", model: "bedrock:mistral.mistral-large-3-675b-instruct" }
+    - { name: bedrock-cohere-r-plus, provider: bedrock, display: "Cohere R+", model: "bedrock:cohere.command-r-plus-v1:0" }
+
+conversation:
+  - { role: user, path: prompts/live-confirmation.md }
+  - { role: assistant }

e2e/runner-matrix/ci.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# CI driver for the runner-matrix e2e project.
+#
+# Always assembles the complete issue #29 runner matrix offline. Live runs are
+# provider-scoped and credential-gated, so a default CI environment with no
+# secrets skips live calls without hiding rows from the committed matrix.
+
+set -euo pipefail
+
+project_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_root="$(cd "${project_dir}/../.." && pwd)"
+
+cd "${repo_root}"
+
+rm -rf "${project_dir}/runs" "${project_dir}/evals/reports" "${project_dir}/evals/judges"
+
+run_dir="$(cargo run --quiet -- -p "${project_dir}" assemble live-confirmation)"
+
+shopt -s nullglob
+conversations=("${run_dir}"/*.yaml)
+shopt -u nullglob
+
+expected=16
+if [[ ${#conversations[@]} -ne ${expected} ]]; then
+  echo "FAIL: ailly assemble produced ${#conversations[@]} conversation file(s) under ${run_dir}; expected ${expected}." >&2
+  exit 1
+fi
+
+echo "OK: ailly assemble produced ${#conversations[@]} runner conversation file(s)."
+
+has_dotenv_value() {
+  local name="$1"
+  local env_file="${project_dir}/.env"
+  [[ -f "${env_file}" ]] && grep -Eq "^[[:space:]]*(export[[:space:]]+)?${name}=" "${env_file}"
+}
+
+has_config_value() {
+  local name="$1"
+  [[ -n "${!name:-}" ]] || has_dotenv_value "${name}"
+}
+
+ran_live=0
+
+run_provider() {
+  local label="$1"
+  shift
+  local -a cases=("$@")
+  local -a run_args=(cargo run --quiet -- -p "${project_dir}" run "${run_dir}")
+  local -a eval_args=(cargo run --quiet -- -p "${project_dir}" eval live-confirmation --over "${run_dir}")
+
+  for case_name in "${cases[@]}"; do
+    run_args+=(--case "${case_name}")
+    eval_args+=(--case "${case_name}")
+  done
+
+  echo "RUN: ${label} (${#cases[@]} runner row(s))"
+  "${run_args[@]}"
+  "${eval_args[@]}"
+  ran_live=1
+}
+
+if has_config_value ANTHROPIC_API_KEY; then
+  run_provider anthropic \
+    anthropic-haiku-4-5 \
+    anthropic-sonnet-4-6 \
+    anthropic-sonnet-5 \
+    anthropic-opus-4-8 \
+    anthropic-fable
+else
+  echo "SKIP: anthropic rows need ANTHROPIC_API_KEY."
+fi
+
+if has_config_value OPENAI_API_KEY; then
+  run_provider openai \
+    openai-gpt-5-5 \
+    openai-gpt-5-4 \
+    openai-gpt-5-4-mini
+else
+  echo "SKIP: openai rows need OPENAI_API_KEY."
+fi
+
+if has_config_value GEMINI_API_KEY; then
+  run_provider google \
+    google-gemini-3-5-flash \
+    google-gemini-3-1-pro-preview \
+    google-gemini-3-1-flash-lite \
+    google-gemini-2-5-pro
+else
+  echo "SKIP: google rows need GEMINI_API_KEY."
+fi
+
+if (has_config_value AWS_REGION || has_config_value AWS_DEFAULT_REGION) \
+  && (has_config_value AWS_BEARER_TOKEN_BEDROCK \
+    || has_config_value AWS_PROFILE \
+    || (has_config_value AWS_ACCESS_KEY_ID && has_config_value AWS_SECRET_ACCESS_KEY)); then
+  run_provider bedrock \
+    bedrock-llama-3-3 \
+    bedrock-llama-4-scout \
+    bedrock-mistral-large-3 \
+    bedrock-cohere-r-plus
+else
+  echo "SKIP: bedrock rows need AWS_REGION or AWS_DEFAULT_REGION plus a Bedrock bearer token, AWS profile, or AWS access keys."
+fi
+
+if [[ ${ran_live} -eq 0 ]]; then
+  echo "OK: no provider credentials detected; live runner rows skipped after offline assemble."
+else
+  echo "OK: completed credentialed live runner rows."
+fi

e2e/runner-matrix/evals/live-confirmation.yaml

@@ -0,0 +1,50 @@
+name: live-confirmation
+cases:
+  - name: anthropic-haiku-4-5
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: anthropic-sonnet-4-6
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: anthropic-sonnet-5
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: anthropic-opus-4-8
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: anthropic-fable
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: openai-gpt-5-5
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: openai-gpt-5-4
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: openai-gpt-5-4-mini
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: google-gemini-3-5-flash
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: google-gemini-3-1-pro-preview
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: google-gemini-3-1-flash-lite
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: google-gemini-2-5-pro
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: bedrock-llama-3-3
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: bedrock-llama-4-scout
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: bedrock-mistral-large-3
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }
+  - name: bedrock-cohere-r-plus
+    assertions:
+      - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" }

e2e/runner-matrix/prompts/live-confirmation.md

@@ -0,0 +1 @@
+Reply in one short sentence. Include the exact marker `AILLY-RUNNER-MATRIX-LIVE` and the model name you are using.