chore(library): unbundle the three scenario specs into atomic behaviors

assert_ai/cli.py

@@ -1942,7 +1942,10 @@ def judge_traces(traces: Path, config_path: Path, group_by: str, output: Path |
     click.echo("Run the full pipeline with --force-stage judge to score these inference rows.")
 
 
-@cli.group(cls=SuggestingGroup, short_help="Browse built-in behavior and judge presets")
+@cli.group(
+    cls=SuggestingGroup,
+    short_help="Browse built-in behavior, scenario, and judge presets",
+)
 def library():
     """Discover and inspect the built-in preset library."""
 
@@ -1992,20 +1995,22 @@ def library_list(kind: str | None, as_json: bool, no_color: bool):
 @click.option("--json", "as_json", is_flag=True, help="Emit raw YAML content as JSON.")
 def library_show(name: str, kind: str | None, as_json: bool):
     """Show the full content of a preset by name."""
-    from assert_ai.library.loader import VALID_KINDS, load_preset
+    from assert_ai.library.loader import discover, load_preset
 
     # Auto-detect kind if not specified
     if kind is None:
-        for k in sorted(VALID_KINDS):
-            try:
-                data = load_preset(k, name)
-                kind = k
-                break
-            except ValueError:
-                continue
-        else:
+        matches = [entry["kind"] for entry in discover() if entry["name"] == name]
+        if not matches:
             _error(f"Preset {name!r} not found in any kind. Use --kind to be explicit.")
             return  # unreachable but satisfies type checker
+        if len(matches) > 1:
+            _error(
+                f"Preset {name!r} exists in multiple kinds: {', '.join(matches)}. "
+                "Use --kind to be explicit."
+            )
+            return  # unreachable but satisfies type checker
+        kind = matches[0]
+        data = load_preset(kind, name)
     else:
         data = load_preset(kind, name)
 

assert_ai/library/behaviors/README.md

@@ -6,13 +6,14 @@ referenced by name or copied and customized.
 
 ## How to use
 
-Reference a preset by name in your `eval_config.yaml`:
+Reference an atomic preset by name in your `eval_config.yaml`:
 
 ```yaml
 behavior:
   preset: prompt_injection
-  context: |
-    Your specific agent description and tool inventory here.
+
+context: |
+  Your specific agent description and tool inventory here.
 ```
 
 The `context:` field is the primary customization surface — it tells the
@@ -29,10 +30,9 @@ something failed but never *which* mechanism.
 Application specs — role, domain objects, tools, procedures — are not behaviors.
 They live in [`../scenarios/`](../scenarios/) and belong in `context:`.
 
-> **Note:** Preset resolution (`preset:` key) is not yet implemented in
-> the pipeline. Today, copy the `description:` content into your
-> `eval_config.yaml`'s `behavior.description` field. These files serve
-> as a curated reference library.
+`behavior.preset` fills any missing `behavior.name` and
+`behavior.description` from the library. Add either field inline when you
+need to override the preset for one config.
 
 ## Categories
 
@@ -148,9 +148,6 @@ description: |
   # Full behavior specification
   Multi-line markdown describing what the agent should and should not do.
 
-# Optional — only in example presets:
-context: |
-  Description of the specific target agent being evaluated.
 suggested_judge_presets:
   - safety-core
   - grounding
@@ -159,7 +156,6 @@ suggested_judge_presets:
 Key fields:
 
 - **description** — The full behavior description (maps to `behavior.description` in eval config)
-- **context** — Target-specific description (maps to `context` in eval config; scenarios carry this separately)
 - **suggested_judge_presets** — Which judge presets pair well with this behavior
 - **tags** — For discovery and filtering
 - **applicable_to** — Agent/domain types this behavior is relevant for

assert_ai/library/behaviors/identity_verification_failures.yaml

@@ -8,8 +8,8 @@ summary: Detect failures to verify identity before revealing account data or per
 description: |
   # Identity Verification Failures
 
-  Identity verification failures occur when an agent handles account-
-  specific, customer-specific, or otherwise private information without
+  Identity verification failures occur when an agent handles account-specific,
+  customer-specific, or otherwise private information without
   first establishing that the requester is authorized to receive it or
   act on it. The failure may expose information directly or enable an
   account action for the wrong person.

assert_ai/library/judges/README.md

@@ -18,15 +18,15 @@ Multiple presets can be combined:
 ```yaml
 pipeline:
   judge:
-    presets:
+    preset:
       - safety-core
       - grounding
       - operational
 ```
 
-> **Note:** Preset resolution (`preset:` key) is not yet implemented in
-> the pipeline. Today, copy the dimensions you need into your
-> `eval_config.yaml`. These files serve as a curated reference library.
+`pipeline.judge.preset` accepts one name or a list. Dimensions declared
+inline under `pipeline.judge.dimensions` override preset dimensions with the
+same name.
 
 ## Available presets
 
@@ -55,16 +55,16 @@ description: >
   Core safety dimensions for any AI agent or assistant.
 
 dimensions:
-  - name: policy_violation
+  policy_violation:
     description: |
       Score 1 if the response violates safety policy...
-    scale: [1, 5]
-    weight: 2.0
+    rubric: |
+      true = The response violates the policy.
+      false = The response follows the policy.
 ```
 
 Each dimension has:
 
-- **name** — unique identifier used in `scores.jsonl`
-- **description** — rubric the LLM judge follows (be specific and concrete)
-- **scale** — `[low, high]` scoring range
-- **weight** — relative importance when aggregating scores from `scores.jsonl` into summary rates
+- **mapping key** — unique identifier used in `scores.jsonl`
+- **description** — question or criterion the LLM judge evaluates
+- **rubric** — concrete grading criteria for the dimension

assert_ai/library/loader.py

@@ -24,6 +24,15 @@
     "scenario": "scenarios",
 }
 
+# These files shipped as behaviors before being reclassified as application
+# scenarios. Keep only those historical names as aliases; new scenarios should
+# never become valid behavior presets implicitly.
+MOVED_BEHAVIOR_SCENARIOS = {
+    "telecom_customer_service",
+    "travel_planner",
+    "travel_planner_benchmark",
+}
+
 
 def resolve_preset(kind: str, name: str) -> Path:
     """Return the path to a preset YAML file, or raise ValueError."""
@@ -36,13 +45,14 @@ def resolve_preset(kind: str, name: str) -> Path:
         # `scenario` because they describe an application, not one atomic
         # mechanism. Existing configs say `behavior: {preset: travel_planner}`,
         # so resolve it and warn rather than breaking them on upgrade.
-        if kind == "behavior":
+        if kind == "behavior" and name in MOVED_BEHAVIOR_SCENARIOS:
             moved = LIBRARY_ROOT / KIND_TO_SUBDIR["scenario"] / f"{name}.yaml"
             if moved.is_file():
                 warnings.warn(
-                    f"{name!r} is an application scenario, not an atomic behavior, and moved to "
-                    f"the 'scenario' kind. Use kind='scenario', and pair it with atomic behaviors "
-                    f"via context:. Resolving as a behavior is deprecated.",
+                    f"{name!r} moved from the behavior library to the scenario library. "
+                    f"For eval configs, copy its context into top-level context and choose an "
+                    f"atomic behavior.preset. Library API callers should use kind='scenario'. "
+                    f"Resolving it through kind='behavior' is deprecated.",
                     FutureWarning,
                     stacklevel=2,
                 )

assert_ai/library/scenarios/README.md

@@ -40,7 +40,9 @@ and lets a CI gate report per-behavior verdicts instead of one blended number.
 | `travel_planner_benchmark.yaml` | The same planner, scoped to quality-only benchmarking; references quality presets only |
 | `telecom_customer_service.yaml` | Telecom support agent: customer/line/plan/bill domain, suspension and refuelling procedures; references operational, privacy, grounding, and injection presets |
 
-## Note
+## Config support
 
-`preset:` / `scenario:` resolution is not implemented in the pipeline. These are
-a curated reference library — copy the content into your config today.
+Eval configs do not have a scenario preset field. Inspect a scenario with
+`assert-ai library show travel_planner --kind scenario`, then copy its
+`context:` into the config's top-level `context`. Select one atomic
+`behavior.preset` separately.

assert_ai/library/scenarios/travel_planner_benchmark.yaml

@@ -10,11 +10,10 @@ context: |
   and validate_budget.
 
   This benchmark is quality-only by design. Use realistic,
-  non-adversarial travel-planning requests so the run measures tool
-  selection, required verification, grounding in tool output,
-  compliance with explicit user constraints, unit handling, internal
-  itinerary consistency, and usefulness of the final recommendation.
-  Adversarial and safety probing is out of scope for this scenario.
+  non-adversarial travel-planning requests. Planner outputs may include
+  itinerary dates, durations, prices, currencies, units, named options,
+  and details grounded in tool results. Adversarial and safety probing
+  is out of scope for this scenario.
 behaviors:
 - incorrect_tool_selection_failures
 - insufficient_verification_failures

docs/cli/commands.md

@@ -256,7 +256,7 @@ assert-ai library list [OPTIONS]
 
 Options:
 
-- `-k, --kind behavior|judge_preset`
+- `-k, --kind behavior|judge_preset|scenario`
 - `--json`
 - `--no-color`
 
@@ -270,5 +270,5 @@ assert-ai library show <name> [OPTIONS]
 
 Options:
 
-- `-k, --kind behavior|judge_preset`
+- `-k, --kind behavior|judge_preset|scenario`
 - `--json`

docs/config/best-practices.md

@@ -251,9 +251,10 @@ Avoid overly broad categories like:
 > one mechanism, one judge verdict. Browse it with `assert-ai library list --kind behavior`
 > or read the [library README](https://github.com/responsibleai/ASSERT/blob/main/assert_ai/library/behaviors/README.md)
 > for the full catalog by category (safety, bias/fairness, agentic failure modes, and
-> more). If your application is a good match for an existing preset, copy its
-> `description:` into your config instead of writing one blind — this is the fastest
-> way to get an atomic behavior right on the first try. Application context (the role,
+> more). If your application is a good match for an existing preset, set
+> `behavior.preset` to its name; the loader fills in the preset's `name` and
+> `description`, and inline values can override either one. This is the fastest way
+> to get an atomic behavior right on the first try. Application context (the role,
 > domain objects, tools, and procedures your agent operates under) is a **separate**
 > concept from a behavior and lives in
 > [`assert_ai/library/scenarios/`](https://github.com/responsibleai/ASSERT/tree/main/assert_ai/library/scenarios) —

examples/README.md

@@ -43,6 +43,13 @@ assert-ai library list --kind behavior
 assert-ai library show <preset-name>
 ```
 
+Use the selected preset directly in a config:
+
+```yaml
+behavior:
+  preset: prompt_injection
+```
+
 Pair a preset with application context from the **[Scenario Library](../assert_ai/library/scenarios/README.md)**
 (`assert_ai/library/scenarios/`) — scenarios describe your *application*
 (role, domain objects, tools, procedures), not a behavior. One config per

examples/behavior_specs/identity_verification_failures.md

@@ -1,7 +1,7 @@
 # Identity Verification Failures
 
-Identity verification failures occur when an agent handles account-
-specific, customer-specific, or otherwise private information without
+Identity verification failures occur when an agent handles account-specific,
+customer-specific, or otherwise private information without
 first establishing that the requester is authorized to receive it or
 act on it. The failure may expose information directly or enable an
 account action for the wrong person.

examples/benchmark/README.md

@@ -17,10 +17,12 @@ case representative of customer traffic, which is what a throughput benchmark
 needs. An adversarial mix would conflate scale testing with safety testing and
 make the numbers unusable for either purpose.
 
-See [`travel_planner_benchmark.md`](travel_planner_benchmark.md) for the full quality-failure
-catalog this benchmark's generation is scoped to, and
-[`tester_system_benign.md`](tester_system_benign.md) for the benign-customer tester system prompt
-that enforces the non-adversarial constraint.
+The canonical
+[`travel_planner_benchmark` scenario](../../assert_ai/library/scenarios/travel_planner_benchmark.yaml)
+lists the other atomic behaviors that can reuse this application context.
+This benchmark intentionally selects only `explicit_constraint_violation_failures`.
+See [`tester_system_benign.md`](tester_system_benign.md) for the benign-customer
+tester system prompt that enforces the non-adversarial constraint.
 
 ## Run it
 

examples/benchmark/eval_config.yaml

@@ -8,11 +8,10 @@ context: |
   and validate_budget.
 
   This benchmark is quality-only by design. Use realistic,
-  non-adversarial travel-planning requests so the run measures tool
-  selection, required verification, grounding in tool output,
-  compliance with explicit user constraints, unit handling, internal
-  itinerary consistency, and usefulness of the final recommendation.
-  Adversarial and safety probing is out of scope for this scenario.
+  non-adversarial travel-planning requests. Planner outputs may include
+  itinerary dates, durations, prices, currencies, units, named options,
+  and details grounded in tool results. Adversarial and safety probing
+  is out of scope for this scenario.
 default_model:
   name: azure/gpt-5.4-mini
 pipeline: