feat(chat): add optional fast-model output router

.github/workflows/evals-behavioral.yml

@@ -52,6 +52,7 @@ jobs:
               - 'packages/junior-evals/src/slack-link.ts'
               - 'packages/junior-evals/src/snapshot-warmup.ts'
               - 'packages/junior-evals/tests/**'
+              - 'packages/junior-evals/create-full-runtime-eval-config.ts'
               - 'packages/junior-evals/vitest.evals.config.ts'
               - 'packages/junior-evals/vitest.evals.behavioral.config.ts'
 

.github/workflows/evals-integration.yml

@@ -41,6 +41,7 @@ jobs:
               - 'packages/junior-evals/src/setup.ts'
               - 'packages/junior-evals/src/slack-link.ts'
               - 'packages/junior-evals/src/snapshot-warmup.ts'
+              - 'packages/junior-evals/create-full-runtime-eval-config.ts'
               - 'packages/junior-evals/vitest.evals.integration.config.ts'
       - id: decision
         env:

.github/workflows/evals-output-router.yml

@@ -0,0 +1,112 @@
+name: Output-router evals
+
+permissions:
+  contents: read
+  checks: write
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, reopened, synchronize, labeled]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  select:
+    name: output-router / select
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    outputs:
+      should_run: ${{ steps.decision.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: getsentry/action-filter-paths@98a158469c63115591d1c2952d34450838ab3bc1 # v0.1.0
+        id: changes
+        with:
+          filters: |
+            relevant:
+              - 'packages/junior-evals/evals/output-router/**'
+              - 'packages/junior-evals/src/output-router-harness.ts'
+              - 'packages/junior-evals/src/output-router-setup.ts'
+              - 'packages/junior-evals/src/eval-ai-gateway-dispatcher.ts'
+              - 'packages/junior-evals/output-router-global-setup.ts'
+              - 'packages/junior-evals/vitest.evals.output-router.config.ts'
+              - 'packages/junior-evals/package.json'
+              - 'packages/junior/src/chat/services/output-router.ts'
+      - id: decision
+        env:
+          AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+          VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }}
+          RELEVANT: ${{ steps.changes.outputs.relevant }}
+          PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
+        run: |
+          set -euo pipefail
+          gateway_ready=false
+          requested=false
+          [[ -n "${AI_GATEWAY_API_KEY:-}" || -n "${VERCEL_OIDC_TOKEN:-}" ]] && gateway_ready=true
+          IFS=',' read -r -a labels <<< "${PR_LABELS:-}"
+          for label in "${labels[@]}"; do
+            if [[ "$label" == "trigger-evals" || "$label" == "trigger-evals-output-router" ]]; then
+              requested=true
+            fi
+          done
+          should_run=false
+          [[ "$gateway_ready" == "true" && ( "$RELEVANT" == "true" || "$requested" == "true" ) ]] && should_run=true
+          echo "should_run=$should_run" >> "$GITHUB_OUTPUT"
+          {
+            echo "## Output-router eval selection"
+            echo
+            echo "- relevant_files_changed: $RELEVANT"
+            echo "- requested: $requested"
+            echo "- gateway_ready: $gateway_ready"
+            echo "- will_run: $should_run"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  output_router:
+    name: output-router / run
+    needs: select
+    if: needs.select.outputs.should_run == 'true'
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    env:
+      AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+      VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-node-pnpm
+      - name: Run output-router evals
+        id: run
+        continue-on-error: true
+        env:
+          VITEST_EVALS_OUTPUT_FILE: output-router-results.json
+          VITEST_EVALS_REPORT_LEVEL: info
+        run: pnpm --filter @sentry/junior-evals evals:output-router
+      - name: Require output-router eval results
+        id: results
+        if: steps.run.conclusion != 'skipped'
+        run: |
+          set -euo pipefail
+          result_file="packages/junior-evals/output-router-results.json"
+          if [[ ! -f "$result_file" ]]; then
+            echo "::error::missing output-router eval results ($result_file). Treat setup/runtime crashes as hard failures."
+            exit 1
+          fi
+      - name: Publish output-router eval summary
+        if: steps.results.conclusion == 'success'
+        uses: getsentry/vitest-evals@v0.16.1
+        with:
+          results: packages/junior-evals/output-router-results.json
+          publish-check: true
+          check-name: output-router / score
+          fail-on-failures: true
+      - name: Upload output-router eval results
+        if: steps.results.conclusion == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: output-router-evals
+          path: packages/junior-evals/output-router-results.json
+          if-no-files-found: error
+          retention-days: 7

AGENTS.md

@@ -21,6 +21,8 @@ Use **pnpm**: `pnpm install`, `pnpm dev`, `pnpm test`, `pnpm typecheck`, `pnpm s
 | Integration eval case        | `pnpm --filter @sentry/junior-evals evals:integration path/to/file.eval.ts -t "case name"` |
 | Guardian eval file         | `pnpm --filter @sentry/junior-evals evals:guardian path/to/file.eval.ts`       |
 | Guardian eval case         | `pnpm --filter @sentry/junior-evals evals:guardian path/to/file.eval.ts -t "case name"` |
+| Output-router eval file    | `pnpm --filter @sentry/junior-evals evals:output-router path/to/file.eval.ts`  |
+| Output-router eval case    | `pnpm --filter @sentry/junior-evals evals:output-router path/to/file.eval.ts -t "case name"` |
 | Generate package schema    | `pnpm --filter <package> db:generate`                                          |
 | Dashboard visual capture   | `pnpm visual:dashboard -- --scenarios gallery-foundations`                     |
 | Release package alignment  | `pnpm release:check`                                                           |

TELEMETRY.md

@@ -174,14 +174,16 @@ Events: `agent.message.received`, `agent.message.generated`,
 `agent.turn.provider_error`, `agent.turn.execution.failed`,
 `agent.turn.empty_output.retrying`,
 `agent.turn.empty_output.exhausted`, `assistant.reply.generation.failed`,
+`ai.output_router.decided`, `ai.output_router.failed`,
 `guardian.action_review.retrying`, `guardian.action_review.exhausted`
 
 `guardian.action_review.exhausted` is a tool-boundary Sentry capture after three
 consecutive action-review denials. The agent still receives a normal tool
 rejection that says not to keep retrying.
 
 Spans: `ai.generate_assistant_reply`, `ai.chat_completion`,
-`chat.route_thinking`, `gen_ai.invoke_agent`, `gen_ai.chat`
+`chat.route_thinking`, `chat.prepare_assistant_reply`, `gen_ai.invoke_agent`,
+`gen_ai.chat`
 
 Attributes: `gen_ai.operation.name`, `gen_ai.request.model`,
 `gen_ai.response.finish_reasons`, `app.ai.outcome`,

packages/docs/src/content/docs/reference/config-and-env.md

@@ -139,6 +139,10 @@ import { createApp } from "@sentry/junior";
 
 const app = await createApp({
   experimental: {
+    // Prepare the visible reply with the fast model before delivery.
+    // Can stay silent for [[NO_REPLY]] and shorten long replies. Off by default.
+    // Original agent text stays in history; only the visible reply may change.
+    "output-router": true,
     // Reply to non-mention messages in Slack threads Junior already joined.
     // Off by default. Without this, Junior only replies to explicit @mentions
     // and resource-event notifications in those threads.
@@ -169,6 +173,13 @@ request reaches the deployment request limit. Run `pnpm acp:local` in this
 repository for a loopback test with the official ACP SDK client. ACP remains a
 pre-stable surface.
 
+`output-router` uses the fast model (`AI_FAST_MODEL`) to prepare the visible
+reply for each completed tool-free assistant message. Exact `[[NO_REPLY]]` stays
+silent. A final whole-line `[[NO_REPLY]]` also stays silent. Answers that mention
+the marker inline still deliver. Long replies can be shortened while keeping the
+`SOUL.md` personality voice. The original agent text remains in conversation
+history. Leave it unset unless you are testing that path.
+
 `passive-routing` turns on replies to non-mention messages in threads Junior
 already joined. Leave it unset in production unless you are testing that path.
 

packages/junior-evals/README.md

@@ -4,11 +4,12 @@
 
 Evals are end-to-end Slack conversation evaluations. They are the integration-style test layer for agent-facing behavior when model interpretation is part of the contract.
 
-There are three independently runnable suites:
+There are four independently runnable suites:
 
 1. **Integration** (`evals/integration/**`) — full agent/runtime runs for primary system functionality that should never regress. Failures are hard pass/fail.
-2. **Behavioral** (domain folders under `evals/` except `integration/` and `guardian/`) — full agent/runtime runs that measure agent behavior and tolerate bounded variability. CI reports a suite score and only blocks below the configured floor.
-3. **Guardian** (`evals/guardian/**`) — isolated decision snapshots scored only on `allow` / `ask` / `deny`. Failures are hard pass/fail.
+2. **Behavioral** (domain folders under `evals/` except `integration/`, `guardian/`, and `output-router/`) — full agent/runtime runs that measure agent behavior and tolerate bounded variability. CI reports a suite score and only blocks below the configured floor.
+3. **Guardian** (`evals/guardian/**`) — isolated action-review snapshots scored only on `allow` / `ask` / `deny`. Failures are hard pass/fail.
+4. **Prepare reply** (`evals/output-router/**`) — isolated `prepareAssistantReply` checks scored on `silent` / `reply`. Failures are hard pass/fail.
 
 - We define conversation cases inline in TypeScript using `describeEval()` and the shared `slackEvals` harness options.
 - We run the real runtime/harness against those fixtures.
@@ -57,9 +58,15 @@ Not in scope:
   - `evals/sentry/`
 - Isolated Guardian decisions: `evals/guardian/`
   - exact `ToolActionProposal` snapshots scored only on `allow` / `ask` / `deny`
+- Isolated prepare-reply cases: `evals/output-router/`
+  - one assistant message through `prepareAssistantReply`
 - Helpers and event builders: `src/helpers.ts`
 - Guardian harness: `src/guardian-harness.ts`
+- Output-router harness: `src/output-router-harness.ts`
 - Harness/runtime adapter: `src/behavior-harness.ts`
+- Shared full-runtime suite config: `create-full-runtime-eval-config.ts`
+  (behavioral and integration). Guardian and output-router stay on their own
+  lightweight configs.
 
 ## Execution Model
 
@@ -105,50 +112,58 @@ Tool replay:
 
 - `pnpm evals` / `pnpm evals:behavioral`: Run the behavioral suite
 - `pnpm evals:integration`: Run the integration suite
-- `pnpm evals:guardian`: Run isolated Guardian decision snapshots
+- `pnpm evals:guardian`: Run isolated Guardian action-review snapshots
+- `pnpm evals:output-router`: Run isolated prepare-reply cases
 - `pnpm --filter @sentry/junior-evals evals:behavioral`: Run behavioral from any directory
 - `pnpm --filter @sentry/junior-evals evals:integration`: Run integration from any directory
 - `pnpm --filter @sentry/junior-evals evals:guardian`: Run Guardian from any directory
+- `pnpm --filter @sentry/junior-evals evals:output-router`: Run isolated prepare-reply cases from any directory
 - `pnpm --filter @sentry/junior-evals evals:behavioral evals/sentry/skills.eval.ts`: Run one behavioral file
 - `pnpm --filter @sentry/junior-evals evals:integration evals/integration/conversation/actions.eval.ts`: Run one integration file
 - `pnpm --filter @sentry/junior-evals evals:guardian evals/guardian/action-review.eval.ts -t "deny"`: Run one Guardian case
+- `pnpm --filter @sentry/junior-evals evals:output-router evals/output-router/visible-reply.eval.ts`: Run one prepare file
+- `pnpm --filter @sentry/junior-evals evals:output-router evals/output-router/visible-reply.eval.ts -t "silent"`: Run one prepare case
 - `pnpm --filter @sentry/junior-evals evals:behavioral --shard=1/4`: Run one of the four CI behavioral shards
 
 Pass eval file paths, `-t` filters, and shard options directly after the suite script. Do not use `pnpm exec vitest` directly, and do not insert `--` before eval arguments.
 
 ## Optional CI Runs
 
-- On pull requests, three independent workflows run and report their own suites:
+- On pull requests, four independent workflows run and report their own suites:
   - `Behavioral evals`: Slack/agent evals (`behavioral / shard *` + `behavioral / report` → `behavioral / score` Check Run)
   - `Integration evals`: system evals (`integration / shard *`)
-  - `Guardian evals`: isolated Guardian snapshots (`guardian / run`)
+  - `Guardian evals`: isolated action-review snapshots (`guardian / run`)
+  - `Output-router evals`: isolated prepare-reply cases (`output-router / run`)
 - Suite labels follow `trigger-evals-[domain]`:
   - `trigger-evals` starts all suites
-  - `trigger-evals-behavioral`, `trigger-evals-integration`, and `trigger-evals-guardian` start one suite
-- Behavioral and integration evals require both gateway and sandbox secrets. Guardian only needs gateway credentials.
+  - `trigger-evals-behavioral`, `trigger-evals-integration`, `trigger-evals-guardian`, and `trigger-evals-output-router` start one suite
+- Behavioral and integration evals require both gateway and sandbox secrets. Guardian and output-router only need gateway credentials.
 - Adding a trigger label fires immediately; unrelated labels do not.
 - Behavioral path triggers cover domain folders under `evals/{agent,conversation,github,memory,scheduler,sentry}/` and shared harness/config files under `packages/junior-evals/`.
 - Integration path triggers cover `evals/integration/**`, the integration config, and shared harness files under `packages/junior-evals/`.
 - Guardian path triggers cover `evals/guardian/**`, the Guardian harness/config under `packages/junior-evals/`, and `packages/junior/src/chat/services/guardian-action-policy.ts`.
+- Output-router path triggers cover `evals/output-router/**`, the prepare harness/config under `packages/junior-evals/`, and `packages/junior/src/chat/services/output-router.ts`.
 - Other product source under `packages/junior/src/**` does not auto-run evals; use a `trigger-evals*` label for that.
-- Behavioral shards still fail individual cases under the per-case judge threshold (`0.75`), but the workflow no longer fails the shard job on those case failures alone. Each behavioral shard and the Guardian job publishes its own `vitest-evals` job summary (pass rate, scores, quality misses).
+- Behavioral shards still fail individual cases under the per-case judge threshold (`0.75`), but the workflow no longer fails the shard job on those case failures alone. Each behavioral shard, Guardian job, and output-router job publishes its own `vitest-evals` job summary (pass rate, scores, quality misses).
 - After all behavioral shards finish, `behavioral / report` combines results, writes the aggregate job summary, and publishes a `behavioral / score` Check Run. The Check Run title carries the gate line (for example `Eval pass rate 90.2% — floor 80.0%`). When that check publishes, the report step soft-fails so the Check Run owns green/red instead of canned job failure text.
 - The behavioral floor is `EVAL_MIN_PASS_RATE=0.8` (`80%` of cases passed). `vitest-evals@0.16` owns the aggregate gate math; individual case misses are warnings when the floor still passes. Missing shard result files or setup/runtime crashes before results are written remain hard failures on the report job.
 - Integration cases fail the `integration / shard *` jobs hard on any miss. They do not use the aggregate pass-rate floor.
 - Guardian cases assert exact `allow` / `ask` / `deny` decisions and fail the `guardian / run` job hard on mismatch. They do not use the aggregate pass-rate floor.
+- Output-router cases assert prepare `silent` / `reply` outcomes and fail the `output-router / run` job hard on mismatch. They do not use the aggregate pass-rate floor.
 - The simplest Gateway and Sandbox setup is `VERCEL_OIDC_TOKEN` alone.
 - The fallback CI setup is `AI_GATEWAY_API_KEY` plus `VERCEL_TOKEN` + `VERCEL_TEAM_ID` + `VERCEL_PROJECT_ID`.
 - Behavioral and integration global setup starts one Cloudflare Quick Tunnel for the suite so Vercel Sandbox can reach the eval egress proxy. Transient tunnel allocation failures retry up to five times with backoff. Local runs require `cloudflared` on `PATH`; CI installs a pinned binary.
 - Behavioral and integration state always uses a loopback Redis. Local runs default to `redis://127.0.0.1:6382`; CI sets `JUNIOR_EVAL_REDIS_URL` for its Redis service.
 - Setup details for GitHub Actions live in `evals/github-actions.md`.
 
-Behavioral and integration evals require real Vercel Sandbox access and public Quick Tunnel connectivity. If either bootstrap fails, the eval fails immediately with no local fallback path. Guardian evals only need AI Gateway access.
+Behavioral and integration evals require real Vercel Sandbox access and public Quick Tunnel connectivity. If either bootstrap fails, the eval fails immediately with no local fallback path. Guardian and output-router evals only need AI Gateway access.
 
 ## Authoring Rules
 
 - Put full-runtime integration cases that must never regress under `evals/integration/**` using `describeEval()` with `slackEvals`. Prefer deterministic assertions; keep criteria only when the case still needs light quality scoring.
 - Put behavioral cases under `evals/conversation/`, `evals/agent/`, or `evals/<feature>/` using `describeEval()` with `slackEvals`.
 - Add isolated Guardian decision snapshots under `evals/guardian/` using `describeEval()` with `guardianEvals`. Feed exact `ToolActionProposal` objects and assert only the expected `allow` / `ask` / `deny` decision.
+- Add isolated prepare-reply cases under `evals/output-router/` using `describeEval()` with `outputRouterEvals`. Feed real assistant message text and check `silent` or `reply`.
 - Put messages that should be pending before processing starts in `initialEvents`.
 - Put ordinary later events in `events`; each is delivered after preceding work settles.
 - Wrap messages with `steer(...)` when they should arrive through normal ingress while the preceding agent run is active.
@@ -205,7 +220,8 @@ Organize files by suite policy first, then by the user-visible area they exercis
 
 - `evals/integration/`: strict full-runtime integration cases (hard pass/fail).
 - `evals/conversation/`, `evals/agent/`, `evals/<feature>/`: agent-behavior cases (score-gated in CI).
-- `evals/guardian/`: isolated Guardian decision snapshots (no main agent; hard pass/fail).
+- `evals/guardian/`: isolated action-review snapshots (no main agent; hard pass/fail).
+- `evals/output-router/`: isolated prepare-reply cases (no main agent; hard pass/fail).
 - Use short behavior nouns for filenames: `routing.eval.ts`, `delivery.eval.ts`, `credentials.eval.ts`.
 - Keep one coherent behavior area per file. Split files when cases exercise independently understandable journeys.
 - Keep shared setup in a nearby `helpers.ts`; helpers are not eval files and do not define suites.