From ced6f84e91f87615d64643ea93ef0eee8f2d5139 Mon Sep 17 00:00:00 2001 From: David Souther Date: Wed, 8 Jul 2026 16:39:25 -0400 Subject: [PATCH] test(e2e): add runner matrix live fixture Co-Authored-By: "Ailly " --- e2e/runner-matrix/.gitignore | 4 + .../assemblies/live-confirmation.yaml | 25 ++ e2e/runner-matrix/ci.sh | 109 +++++++ .../evals/live-confirmation.yaml | 50 ++++ .../prompts/live-confirmation.md | 1 + tests/e2e_runner_matrix.rs | 275 ++++++++++++++++++ 6 files changed, 464 insertions(+) create mode 100644 e2e/runner-matrix/.gitignore create mode 100644 e2e/runner-matrix/assemblies/live-confirmation.yaml create mode 100755 e2e/runner-matrix/ci.sh create mode 100644 e2e/runner-matrix/evals/live-confirmation.yaml create mode 100644 e2e/runner-matrix/prompts/live-confirmation.md create mode 100644 tests/e2e_runner_matrix.rs diff --git a/e2e/runner-matrix/.gitignore b/e2e/runner-matrix/.gitignore new file mode 100644 index 00000000..c50ef069 --- /dev/null +++ b/e2e/runner-matrix/.gitignore @@ -0,0 +1,4 @@ +runs/ +evals/reports/ +evals/judges/ +evals/scripts/__pycache__/ diff --git a/e2e/runner-matrix/assemblies/live-confirmation.yaml b/e2e/runner-matrix/assemblies/live-confirmation.yaml new file mode 100644 index 00000000..64026c09 --- /dev/null +++ b/e2e/runner-matrix/assemblies/live-confirmation.yaml @@ -0,0 +1,25 @@ +name: live-confirmation +model: claude-haiku-4-5 + +matrix: + runner: + - { name: anthropic-haiku-4-5, provider: anthropic, display: "Haiku 4.5", model: claude-haiku-4-5 } + - { name: anthropic-sonnet-4-6, provider: anthropic, display: "Sonnet 4.6", model: claude-sonnet-4-6 } + - { name: anthropic-sonnet-5, provider: anthropic, display: "Sonnet 5", model: claude-sonnet-5 } + - { name: anthropic-opus-4-8, provider: anthropic, display: "Opus 4.8", model: claude-opus-4-8 } + - { name: anthropic-fable, provider: anthropic, display: "Fable", model: claude-fable-5 } + - { name: openai-gpt-5-5, provider: openai, display: "GPT-5.5", model: gpt-5.5 } + - { name: openai-gpt-5-4, provider: openai, display: "GPT-5.4", model: gpt-5.4 } + - { name: openai-gpt-5-4-mini, provider: openai, display: "GPT-5.4-mini", model: gpt-5.4-mini } + - { name: google-gemini-3-5-flash, provider: google, display: "Gemini 3.5 Flash", model: gemini-3.5-flash } + - { name: google-gemini-3-1-pro-preview, provider: google, display: "Gemini 3.1 Pro (Preview)", model: gemini-3.1-pro-preview } + - { name: google-gemini-3-1-flash-lite, provider: google, display: "Gemini 3.1 Flash-Lite", model: gemini-3.1-flash-lite } + - { name: google-gemini-2-5-pro, provider: google, display: "Gemini 2.5 Pro", model: gemini-2.5-pro } + - { name: bedrock-llama-3-3, provider: bedrock, display: "Llama 3.3", model: "bedrock:meta.llama3-3-70b-instruct-v1:0" } + - { name: bedrock-llama-4-scout, provider: bedrock, display: "Llama 4 Scout", model: "bedrock:meta.llama4-scout-17b-instruct-v1:0" } + - { name: bedrock-mistral-large-3, provider: bedrock, display: "Mistral Large 3", model: "bedrock:mistral.mistral-large-3-675b-instruct" } + - { name: bedrock-cohere-r-plus, provider: bedrock, display: "Cohere R+", model: "bedrock:cohere.command-r-plus-v1:0" } + +conversation: + - { role: user, path: prompts/live-confirmation.md } + - { role: assistant } diff --git a/e2e/runner-matrix/ci.sh b/e2e/runner-matrix/ci.sh new file mode 100755 index 00000000..3ac8a8f9 --- /dev/null +++ b/e2e/runner-matrix/ci.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# CI driver for the runner-matrix e2e project. +# +# Always assembles the complete issue #29 runner matrix offline. Live runs are +# provider-scoped and credential-gated, so a default CI environment with no +# secrets skips live calls without hiding rows from the committed matrix. + +set -euo pipefail + +project_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${project_dir}/../.." && pwd)" + +cd "${repo_root}" + +rm -rf "${project_dir}/runs" "${project_dir}/evals/reports" "${project_dir}/evals/judges" + +run_dir="$(cargo run --quiet -- -p "${project_dir}" assemble live-confirmation)" + +shopt -s nullglob +conversations=("${run_dir}"/*.yaml) +shopt -u nullglob + +expected=16 +if [[ ${#conversations[@]} -ne ${expected} ]]; then + echo "FAIL: ailly assemble produced ${#conversations[@]} conversation file(s) under ${run_dir}; expected ${expected}." >&2 + exit 1 +fi + +echo "OK: ailly assemble produced ${#conversations[@]} runner conversation file(s)." + +has_dotenv_value() { + local name="$1" + local env_file="${project_dir}/.env" + [[ -f "${env_file}" ]] && grep -Eq "^[[:space:]]*(export[[:space:]]+)?${name}=" "${env_file}" +} + +has_config_value() { + local name="$1" + [[ -n "${!name:-}" ]] || has_dotenv_value "${name}" +} + +ran_live=0 + +run_provider() { + local label="$1" + shift + local -a cases=("$@") + local -a run_args=(cargo run --quiet -- -p "${project_dir}" run "${run_dir}") + local -a eval_args=(cargo run --quiet -- -p "${project_dir}" eval live-confirmation --over "${run_dir}") + + for case_name in "${cases[@]}"; do + run_args+=(--case "${case_name}") + eval_args+=(--case "${case_name}") + done + + echo "RUN: ${label} (${#cases[@]} runner row(s))" + "${run_args[@]}" + "${eval_args[@]}" + ran_live=1 +} + +if has_config_value ANTHROPIC_API_KEY; then + run_provider anthropic \ + anthropic-haiku-4-5 \ + anthropic-sonnet-4-6 \ + anthropic-sonnet-5 \ + anthropic-opus-4-8 \ + anthropic-fable +else + echo "SKIP: anthropic rows need ANTHROPIC_API_KEY." +fi + +if has_config_value OPENAI_API_KEY; then + run_provider openai \ + openai-gpt-5-5 \ + openai-gpt-5-4 \ + openai-gpt-5-4-mini +else + echo "SKIP: openai rows need OPENAI_API_KEY." +fi + +if has_config_value GEMINI_API_KEY; then + run_provider google \ + google-gemini-3-5-flash \ + google-gemini-3-1-pro-preview \ + google-gemini-3-1-flash-lite \ + google-gemini-2-5-pro +else + echo "SKIP: google rows need GEMINI_API_KEY." +fi + +if (has_config_value AWS_REGION || has_config_value AWS_DEFAULT_REGION) \ + && (has_config_value AWS_BEARER_TOKEN_BEDROCK \ + || has_config_value AWS_PROFILE \ + || (has_config_value AWS_ACCESS_KEY_ID && has_config_value AWS_SECRET_ACCESS_KEY)); then + run_provider bedrock \ + bedrock-llama-3-3 \ + bedrock-llama-4-scout \ + bedrock-mistral-large-3 \ + bedrock-cohere-r-plus +else + echo "SKIP: bedrock rows need AWS_REGION or AWS_DEFAULT_REGION plus a Bedrock bearer token, AWS profile, or AWS access keys." +fi + +if [[ ${ran_live} -eq 0 ]]; then + echo "OK: no provider credentials detected; live runner rows skipped after offline assemble." +else + echo "OK: completed credentialed live runner rows." +fi diff --git a/e2e/runner-matrix/evals/live-confirmation.yaml b/e2e/runner-matrix/evals/live-confirmation.yaml new file mode 100644 index 00000000..0436f9b3 --- /dev/null +++ b/e2e/runner-matrix/evals/live-confirmation.yaml @@ -0,0 +1,50 @@ +name: live-confirmation +cases: + - name: anthropic-haiku-4-5 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: anthropic-sonnet-4-6 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: anthropic-sonnet-5 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: anthropic-opus-4-8 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: anthropic-fable + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: openai-gpt-5-5 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: openai-gpt-5-4 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: openai-gpt-5-4-mini + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: google-gemini-3-5-flash + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: google-gemini-3-1-pro-preview + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: google-gemini-3-1-flash-lite + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: google-gemini-2-5-pro + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: bedrock-llama-3-3 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: bedrock-llama-4-scout + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: bedrock-mistral-large-3 + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } + - name: bedrock-cohere-r-plus + assertions: + - { type: text_contains, value: "AILLY-RUNNER-MATRIX-LIVE" } diff --git a/e2e/runner-matrix/prompts/live-confirmation.md b/e2e/runner-matrix/prompts/live-confirmation.md new file mode 100644 index 00000000..cbc927fc --- /dev/null +++ b/e2e/runner-matrix/prompts/live-confirmation.md @@ -0,0 +1 @@ +Reply in one short sentence. Include the exact marker `AILLY-RUNNER-MATRIX-LIVE` and the model name you are using. diff --git a/tests/e2e_runner_matrix.rs b/tests/e2e_runner_matrix.rs new file mode 100644 index 00000000..e0e8e0c4 --- /dev/null +++ b/tests/e2e_runner_matrix.rs @@ -0,0 +1,275 @@ +//! Feature test for the runner-matrix e2e slice. +//! +//! User story: a developer preparing a release-readiness check wants one +//! committed Ailly matrix covering every runner named in domain-driven-design +//! issue #29. Offline, they can assemble the matrix and verify the structure: +//! one conversation per runner, each with provider metadata in the source row, +//! a scalar runner binding, and a provider-routed `meta.model`. With +//! credentials, the same run directory becomes the live confirmation input. +//! +//! The fixture stays useful even without credentials: the offline assertions +//! catch missing rows, broken case filtering, and model ids that no longer +//! route through the intended provider family. + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +use ailly_two::cli::assemble::AssembleArgs; +use ailly_two::cli::assemble::run as assemble_run; +use ailly_two::content::conversation::Conversation; +use serde_yaml_ng::Mapping; +use serde_yaml_ng::Value; + +#[derive(Clone, Copy, Debug)] +struct ExpectedRunner { + slug: &'static str, + provider: &'static str, + display: &'static str, + model_prefix: &'static str, +} + +const EXPECTED_RUNNERS: &[ExpectedRunner] = &[ + ExpectedRunner { + slug: "anthropic-haiku-4-5", + provider: "anthropic", + display: "Haiku 4.5", + model_prefix: "claude-", + }, + ExpectedRunner { + slug: "anthropic-sonnet-4-6", + provider: "anthropic", + display: "Sonnet 4.6", + model_prefix: "claude-", + }, + ExpectedRunner { + slug: "anthropic-sonnet-5", + provider: "anthropic", + display: "Sonnet 5", + model_prefix: "claude-", + }, + ExpectedRunner { + slug: "anthropic-opus-4-8", + provider: "anthropic", + display: "Opus 4.8", + model_prefix: "claude-", + }, + ExpectedRunner { + slug: "anthropic-fable", + provider: "anthropic", + display: "Fable", + model_prefix: "claude-", + }, + ExpectedRunner { + slug: "openai-gpt-5-5", + provider: "openai", + display: "GPT-5.5", + model_prefix: "gpt-", + }, + ExpectedRunner { + slug: "openai-gpt-5-4", + provider: "openai", + display: "GPT-5.4", + model_prefix: "gpt-", + }, + ExpectedRunner { + slug: "openai-gpt-5-4-mini", + provider: "openai", + display: "GPT-5.4-mini", + model_prefix: "gpt-", + }, + ExpectedRunner { + slug: "google-gemini-3-5-flash", + provider: "google", + display: "Gemini 3.5 Flash", + model_prefix: "gemini-", + }, + ExpectedRunner { + slug: "google-gemini-3-1-pro-preview", + provider: "google", + display: "Gemini 3.1 Pro (Preview)", + model_prefix: "gemini-", + }, + ExpectedRunner { + slug: "google-gemini-3-1-flash-lite", + provider: "google", + display: "Gemini 3.1 Flash-Lite", + model_prefix: "gemini-", + }, + ExpectedRunner { + slug: "google-gemini-2-5-pro", + provider: "google", + display: "Gemini 2.5 Pro", + model_prefix: "gemini-", + }, + ExpectedRunner { + slug: "bedrock-llama-3-3", + provider: "bedrock", + display: "Llama 3.3", + model_prefix: "bedrock:", + }, + ExpectedRunner { + slug: "bedrock-llama-4-scout", + provider: "bedrock", + display: "Llama 4 Scout", + model_prefix: "bedrock:", + }, + ExpectedRunner { + slug: "bedrock-mistral-large-3", + provider: "bedrock", + display: "Mistral Large 3", + model_prefix: "bedrock:", + }, + ExpectedRunner { + slug: "bedrock-cohere-r-plus", + provider: "bedrock", + display: "Cohere R+", + model_prefix: "bedrock:", + }, +]; + +#[test] +fn runner_matrix_assembles_every_issue_29_runner_with_provider_metadata() { + let project = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("e2e") + .join("runner-matrix"); + let assembly_path = project.join("assemblies").join("live-confirmation.yaml"); + + let assembly_yaml = fs::read_to_string(&assembly_path) + .unwrap_or_else(|err| panic!("read runner matrix assembly {assembly_path:?}: {err}")); + let assembly_doc: Value = + serde_yaml_ng::from_str(&assembly_yaml).expect("runner matrix assembly parses as YAML"); + + let rows = runner_rows(&assembly_doc); + assert_eq!( + rows.len(), + EXPECTED_RUNNERS.len(), + "the source matrix must contain every issue #29 runner row" + ); + + for expected in EXPECTED_RUNNERS { + let row = rows + .get(expected.slug) + .unwrap_or_else(|| panic!("missing runner row {}", expected.slug)); + assert_eq!( + row.get("provider").map(String::as_str), + Some(expected.provider), + "{} provider metadata", + expected.slug + ); + assert_eq!( + row.get("display").map(String::as_str), + Some(expected.display), + "{} display name", + expected.slug + ); + let model = row + .get("model") + .unwrap_or_else(|| panic!("{} must include explicit model id", expected.slug)); + assert!( + model.starts_with(expected.model_prefix), + "{} model id {model:?} must route via provider prefix {:?}", + expected.slug, + expected.model_prefix + ); + } + + let run_dir = assemble_run(AssembleArgs { + project: project.clone(), + name: String::from("live-confirmation"), + ..Default::default() + }) + .expect("unfiltered runner matrix assembles offline"); + + let files = yaml_files(&run_dir); + assert_eq!( + files.len(), + EXPECTED_RUNNERS.len(), + "unfiltered assemble writes one conversation per runner" + ); + + for expected in EXPECTED_RUNNERS { + let filename = format!("{}.yaml", expected.slug); + assert!( + files.contains(&filename), + "unfiltered assemble should write {filename}" + ); + let yaml = fs::read_to_string(run_dir.join(&filename)) + .unwrap_or_else(|err| panic!("read assembled {filename}: {err}")); + let conv = Conversation::from_yaml_str(&yaml) + .unwrap_or_else(|err| panic!("parse assembled {filename}: {err}")); + assert!( + conv.meta.model.as_ref().starts_with(expected.model_prefix), + "{filename} meta.model {:?} must route through {:?}", + conv.meta.model.as_ref(), + expected.model_prefix + ); + assert_eq!( + conv.meta.binding.get("runner").and_then(Value::as_str), + Some(expected.slug), + "{filename} records scalar runner binding" + ); + } + + let filtered_run_dir = assemble_run(AssembleArgs { + project, + name: String::from("live-confirmation"), + cases: vec![String::from("openai-gpt-5-4-mini")], + }) + .expect("runner matrix supports --case for one row"); + assert_eq!( + yaml_files(&filtered_run_dir), + vec![String::from("openai-gpt-5-4-mini.yaml")], + "--case scopes the matrix to the selected runner" + ); +} + +fn runner_rows(doc: &Value) -> BTreeMap> { + let root = as_mapping(doc, "assembly root"); + let matrix = as_mapping( + root.get(Value::from("matrix")) + .expect("assembly has matrix"), + "matrix", + ); + let runner = matrix + .get(Value::from("runner")) + .and_then(Value::as_sequence) + .expect("matrix.runner is a sequence"); + + let mut rows = BTreeMap::new(); + for value in runner { + let row = as_mapping(value, "runner row"); + let name = required_str(row, "name").to_string(); + let fields = ["provider", "display", "model"] + .into_iter() + .map(|key| (key.to_string(), required_str(row, key).to_string())) + .collect(); + rows.insert(name, fields); + } + rows +} + +fn as_mapping<'a>(value: &'a Value, label: &str) -> &'a Mapping { + match value { + Value::Mapping(map) => map, + other => panic!("{label} must be a YAML mapping, got {other:?}"), + } +} + +fn required_str<'a>(map: &'a Mapping, key: &str) -> &'a str { + map.get(Value::from(key)) + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("runner row must include string field {key:?}")) +} + +fn yaml_files(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .expect("read run dir") + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "yaml")) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +}