diff --git a/.github/workflows/evals-behavioral.yml b/.github/workflows/evals-behavioral.yml index c97ac2850f..fab2d58cb1 100644 --- a/.github/workflows/evals-behavioral.yml +++ b/.github/workflows/evals-behavioral.yml @@ -52,6 +52,7 @@ jobs: - 'packages/junior-evals/src/slack-link.ts' - 'packages/junior-evals/src/snapshot-warmup.ts' - 'packages/junior-evals/tests/**' + - 'packages/junior-evals/create-full-runtime-eval-config.ts' - 'packages/junior-evals/vitest.evals.config.ts' - 'packages/junior-evals/vitest.evals.behavioral.config.ts' diff --git a/.github/workflows/evals-integration.yml b/.github/workflows/evals-integration.yml index 1e5eb8d57e..6c54382460 100644 --- a/.github/workflows/evals-integration.yml +++ b/.github/workflows/evals-integration.yml @@ -41,6 +41,7 @@ jobs: - 'packages/junior-evals/src/setup.ts' - 'packages/junior-evals/src/slack-link.ts' - 'packages/junior-evals/src/snapshot-warmup.ts' + - 'packages/junior-evals/create-full-runtime-eval-config.ts' - 'packages/junior-evals/vitest.evals.integration.config.ts' - id: decision env: diff --git a/.github/workflows/evals-output-router.yml b/.github/workflows/evals-output-router.yml new file mode 100644 index 0000000000..8290fb1776 --- /dev/null +++ b/.github/workflows/evals-output-router.yml @@ -0,0 +1,112 @@ +name: Output-router evals + +permissions: + contents: read + checks: write + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize, labeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + select: + name: output-router / select + runs-on: blacksmith-4vcpu-ubuntu-2404 + outputs: + should_run: ${{ steps.decision.outputs.should_run }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: getsentry/action-filter-paths@98a158469c63115591d1c2952d34450838ab3bc1 # v0.1.0 + id: changes + with: + filters: | + relevant: + - 'packages/junior-evals/evals/output-router/**' + - 'packages/junior-evals/src/output-router-harness.ts' + - 'packages/junior-evals/src/output-router-setup.ts' + - 'packages/junior-evals/src/eval-ai-gateway-dispatcher.ts' + - 'packages/junior-evals/output-router-global-setup.ts' + - 'packages/junior-evals/vitest.evals.output-router.config.ts' + - 'packages/junior-evals/package.json' + - 'packages/junior/src/chat/services/output-router.ts' + - id: decision + env: + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + RELEVANT: ${{ steps.changes.outputs.relevant }} + PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + run: | + set -euo pipefail + gateway_ready=false + requested=false + [[ -n "${AI_GATEWAY_API_KEY:-}" || -n "${VERCEL_OIDC_TOKEN:-}" ]] && gateway_ready=true + IFS=',' read -r -a labels <<< "${PR_LABELS:-}" + for label in "${labels[@]}"; do + if [[ "$label" == "trigger-evals" || "$label" == "trigger-evals-output-router" ]]; then + requested=true + fi + done + should_run=false + [[ "$gateway_ready" == "true" && ( "$RELEVANT" == "true" || "$requested" == "true" ) ]] && should_run=true + echo "should_run=$should_run" >> "$GITHUB_OUTPUT" + { + echo "## Output-router eval selection" + echo + echo "- relevant_files_changed: $RELEVANT" + echo "- requested: $requested" + echo "- gateway_ready: $gateway_ready" + echo "- will_run: $should_run" + } >> "$GITHUB_STEP_SUMMARY" + + output_router: + name: output-router / run + needs: select + if: needs.select.outputs.should_run == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + env: + AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-node-pnpm + - name: Run output-router evals + id: run + continue-on-error: true + env: + VITEST_EVALS_OUTPUT_FILE: output-router-results.json + VITEST_EVALS_REPORT_LEVEL: info + run: pnpm --filter @sentry/junior-evals evals:output-router + - name: Require output-router eval results + id: results + if: steps.run.conclusion != 'skipped' + run: | + set -euo pipefail + result_file="packages/junior-evals/output-router-results.json" + if [[ ! -f "$result_file" ]]; then + echo "::error::missing output-router eval results ($result_file). Treat setup/runtime crashes as hard failures." + exit 1 + fi + - name: Publish output-router eval summary + if: steps.results.conclusion == 'success' + uses: getsentry/vitest-evals@v0.16.1 + with: + results: packages/junior-evals/output-router-results.json + publish-check: true + check-name: output-router / score + fail-on-failures: true + - name: Upload output-router eval results + if: steps.results.conclusion == 'success' + uses: actions/upload-artifact@v4 + with: + name: output-router-evals + path: packages/junior-evals/output-router-results.json + if-no-files-found: error + retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index d58af00abd..37b6f9d2e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,8 @@ Use **pnpm**: `pnpm install`, `pnpm dev`, `pnpm test`, `pnpm typecheck`, `pnpm s | Integration eval case | `pnpm --filter @sentry/junior-evals evals:integration path/to/file.eval.ts -t "case name"` | | Guardian eval file | `pnpm --filter @sentry/junior-evals evals:guardian path/to/file.eval.ts` | | Guardian eval case | `pnpm --filter @sentry/junior-evals evals:guardian path/to/file.eval.ts -t "case name"` | +| Output-router eval file | `pnpm --filter @sentry/junior-evals evals:output-router path/to/file.eval.ts` | +| Output-router eval case | `pnpm --filter @sentry/junior-evals evals:output-router path/to/file.eval.ts -t "case name"` | | Generate package schema | `pnpm --filter db:generate` | | Dashboard visual capture | `pnpm visual:dashboard -- --scenarios gallery-foundations` | | Release package alignment | `pnpm release:check` | diff --git a/TELEMETRY.md b/TELEMETRY.md index 0f19af7d15..18fdb2dc05 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -174,6 +174,7 @@ Events: `agent.message.received`, `agent.message.generated`, `agent.turn.provider_error`, `agent.turn.execution.failed`, `agent.turn.empty_output.retrying`, `agent.turn.empty_output.exhausted`, `assistant.reply.generation.failed`, +`ai.output_router.decided`, `ai.output_router.failed`, `guardian.action_review.retrying`, `guardian.action_review.exhausted` `guardian.action_review.exhausted` is a tool-boundary Sentry capture after three @@ -181,7 +182,8 @@ consecutive action-review denials. The agent still receives a normal tool rejection that says not to keep retrying. Spans: `ai.generate_assistant_reply`, `ai.chat_completion`, -`chat.route_thinking`, `gen_ai.invoke_agent`, `gen_ai.chat` +`chat.route_thinking`, `chat.prepare_assistant_reply`, `gen_ai.invoke_agent`, +`gen_ai.chat` Attributes: `gen_ai.operation.name`, `gen_ai.request.model`, `gen_ai.response.finish_reasons`, `app.ai.outcome`, diff --git a/packages/docs/src/content/docs/reference/config-and-env.md b/packages/docs/src/content/docs/reference/config-and-env.md index 3898582a55..89e1887c76 100644 --- a/packages/docs/src/content/docs/reference/config-and-env.md +++ b/packages/docs/src/content/docs/reference/config-and-env.md @@ -139,6 +139,10 @@ import { createApp } from "@sentry/junior"; const app = await createApp({ experimental: { + // Prepare the visible reply with the fast model before delivery. + // Can stay silent for [[NO_REPLY]] and shorten long replies. Off by default. + // Original agent text stays in history; only the visible reply may change. + "output-router": true, // Reply to non-mention messages in Slack threads Junior already joined. // Off by default. Without this, Junior only replies to explicit @mentions // and resource-event notifications in those threads. @@ -169,6 +173,13 @@ request reaches the deployment request limit. Run `pnpm acp:local` in this repository for a loopback test with the official ACP SDK client. ACP remains a pre-stable surface. +`output-router` uses the fast model (`AI_FAST_MODEL`) to prepare the visible +reply for each completed tool-free assistant message. Exact `[[NO_REPLY]]` stays +silent. A final whole-line `[[NO_REPLY]]` also stays silent. Answers that mention +the marker inline still deliver. Long replies can be shortened while keeping the +`SOUL.md` personality voice. The original agent text remains in conversation +history. Leave it unset unless you are testing that path. + `passive-routing` turns on replies to non-mention messages in threads Junior already joined. Leave it unset in production unless you are testing that path. diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md index a0d1e1d98b..8d7a6354fb 100644 --- a/packages/junior-evals/README.md +++ b/packages/junior-evals/README.md @@ -4,11 +4,12 @@ Evals are end-to-end Slack conversation evaluations. They are the integration-style test layer for agent-facing behavior when model interpretation is part of the contract. -There are three independently runnable suites: +There are four independently runnable suites: 1. **Integration** (`evals/integration/**`) — full agent/runtime runs for primary system functionality that should never regress. Failures are hard pass/fail. -2. **Behavioral** (domain folders under `evals/` except `integration/` and `guardian/`) — full agent/runtime runs that measure agent behavior and tolerate bounded variability. CI reports a suite score and only blocks below the configured floor. -3. **Guardian** (`evals/guardian/**`) — isolated decision snapshots scored only on `allow` / `ask` / `deny`. Failures are hard pass/fail. +2. **Behavioral** (domain folders under `evals/` except `integration/`, `guardian/`, and `output-router/`) — full agent/runtime runs that measure agent behavior and tolerate bounded variability. CI reports a suite score and only blocks below the configured floor. +3. **Guardian** (`evals/guardian/**`) — isolated action-review snapshots scored only on `allow` / `ask` / `deny`. Failures are hard pass/fail. +4. **Prepare reply** (`evals/output-router/**`) — isolated `prepareAssistantReply` checks scored on `silent` / `reply`. Failures are hard pass/fail. - We define conversation cases inline in TypeScript using `describeEval()` and the shared `slackEvals` harness options. - We run the real runtime/harness against those fixtures. @@ -57,9 +58,15 @@ Not in scope: - `evals/sentry/` - Isolated Guardian decisions: `evals/guardian/` - exact `ToolActionProposal` snapshots scored only on `allow` / `ask` / `deny` +- Isolated prepare-reply cases: `evals/output-router/` + - one assistant message through `prepareAssistantReply` - Helpers and event builders: `src/helpers.ts` - Guardian harness: `src/guardian-harness.ts` +- Output-router harness: `src/output-router-harness.ts` - Harness/runtime adapter: `src/behavior-harness.ts` +- Shared full-runtime suite config: `create-full-runtime-eval-config.ts` + (behavioral and integration). Guardian and output-router stay on their own + lightweight configs. ## Execution Model @@ -105,50 +112,58 @@ Tool replay: - `pnpm evals` / `pnpm evals:behavioral`: Run the behavioral suite - `pnpm evals:integration`: Run the integration suite -- `pnpm evals:guardian`: Run isolated Guardian decision snapshots +- `pnpm evals:guardian`: Run isolated Guardian action-review snapshots +- `pnpm evals:output-router`: Run isolated prepare-reply cases - `pnpm --filter @sentry/junior-evals evals:behavioral`: Run behavioral from any directory - `pnpm --filter @sentry/junior-evals evals:integration`: Run integration from any directory - `pnpm --filter @sentry/junior-evals evals:guardian`: Run Guardian from any directory +- `pnpm --filter @sentry/junior-evals evals:output-router`: Run isolated prepare-reply cases from any directory - `pnpm --filter @sentry/junior-evals evals:behavioral evals/sentry/skills.eval.ts`: Run one behavioral file - `pnpm --filter @sentry/junior-evals evals:integration evals/integration/conversation/actions.eval.ts`: Run one integration file - `pnpm --filter @sentry/junior-evals evals:guardian evals/guardian/action-review.eval.ts -t "deny"`: Run one Guardian case +- `pnpm --filter @sentry/junior-evals evals:output-router evals/output-router/visible-reply.eval.ts`: Run one prepare file +- `pnpm --filter @sentry/junior-evals evals:output-router evals/output-router/visible-reply.eval.ts -t "silent"`: Run one prepare case - `pnpm --filter @sentry/junior-evals evals:behavioral --shard=1/4`: Run one of the four CI behavioral shards Pass eval file paths, `-t` filters, and shard options directly after the suite script. Do not use `pnpm exec vitest` directly, and do not insert `--` before eval arguments. ## Optional CI Runs -- On pull requests, three independent workflows run and report their own suites: +- On pull requests, four independent workflows run and report their own suites: - `Behavioral evals`: Slack/agent evals (`behavioral / shard *` + `behavioral / report` → `behavioral / score` Check Run) - `Integration evals`: system evals (`integration / shard *`) - - `Guardian evals`: isolated Guardian snapshots (`guardian / run`) + - `Guardian evals`: isolated action-review snapshots (`guardian / run`) + - `Output-router evals`: isolated prepare-reply cases (`output-router / run`) - Suite labels follow `trigger-evals-[domain]`: - `trigger-evals` starts all suites - - `trigger-evals-behavioral`, `trigger-evals-integration`, and `trigger-evals-guardian` start one suite -- Behavioral and integration evals require both gateway and sandbox secrets. Guardian only needs gateway credentials. + - `trigger-evals-behavioral`, `trigger-evals-integration`, `trigger-evals-guardian`, and `trigger-evals-output-router` start one suite +- Behavioral and integration evals require both gateway and sandbox secrets. Guardian and output-router only need gateway credentials. - Adding a trigger label fires immediately; unrelated labels do not. - Behavioral path triggers cover domain folders under `evals/{agent,conversation,github,memory,scheduler,sentry}/` and shared harness/config files under `packages/junior-evals/`. - Integration path triggers cover `evals/integration/**`, the integration config, and shared harness files under `packages/junior-evals/`. - Guardian path triggers cover `evals/guardian/**`, the Guardian harness/config under `packages/junior-evals/`, and `packages/junior/src/chat/services/guardian-action-policy.ts`. +- Output-router path triggers cover `evals/output-router/**`, the prepare harness/config under `packages/junior-evals/`, and `packages/junior/src/chat/services/output-router.ts`. - Other product source under `packages/junior/src/**` does not auto-run evals; use a `trigger-evals*` label for that. -- Behavioral shards still fail individual cases under the per-case judge threshold (`0.75`), but the workflow no longer fails the shard job on those case failures alone. Each behavioral shard and the Guardian job publishes its own `vitest-evals` job summary (pass rate, scores, quality misses). +- Behavioral shards still fail individual cases under the per-case judge threshold (`0.75`), but the workflow no longer fails the shard job on those case failures alone. Each behavioral shard, Guardian job, and output-router job publishes its own `vitest-evals` job summary (pass rate, scores, quality misses). - After all behavioral shards finish, `behavioral / report` combines results, writes the aggregate job summary, and publishes a `behavioral / score` Check Run. The Check Run title carries the gate line (for example `Eval pass rate 90.2% — floor 80.0%`). When that check publishes, the report step soft-fails so the Check Run owns green/red instead of canned job failure text. - The behavioral floor is `EVAL_MIN_PASS_RATE=0.8` (`80%` of cases passed). `vitest-evals@0.16` owns the aggregate gate math; individual case misses are warnings when the floor still passes. Missing shard result files or setup/runtime crashes before results are written remain hard failures on the report job. - Integration cases fail the `integration / shard *` jobs hard on any miss. They do not use the aggregate pass-rate floor. - Guardian cases assert exact `allow` / `ask` / `deny` decisions and fail the `guardian / run` job hard on mismatch. They do not use the aggregate pass-rate floor. +- Output-router cases assert prepare `silent` / `reply` outcomes and fail the `output-router / run` job hard on mismatch. They do not use the aggregate pass-rate floor. - The simplest Gateway and Sandbox setup is `VERCEL_OIDC_TOKEN` alone. - The fallback CI setup is `AI_GATEWAY_API_KEY` plus `VERCEL_TOKEN` + `VERCEL_TEAM_ID` + `VERCEL_PROJECT_ID`. - Behavioral and integration global setup starts one Cloudflare Quick Tunnel for the suite so Vercel Sandbox can reach the eval egress proxy. Transient tunnel allocation failures retry up to five times with backoff. Local runs require `cloudflared` on `PATH`; CI installs a pinned binary. - Behavioral and integration state always uses a loopback Redis. Local runs default to `redis://127.0.0.1:6382`; CI sets `JUNIOR_EVAL_REDIS_URL` for its Redis service. - Setup details for GitHub Actions live in `evals/github-actions.md`. -Behavioral and integration evals require real Vercel Sandbox access and public Quick Tunnel connectivity. If either bootstrap fails, the eval fails immediately with no local fallback path. Guardian evals only need AI Gateway access. +Behavioral and integration evals require real Vercel Sandbox access and public Quick Tunnel connectivity. If either bootstrap fails, the eval fails immediately with no local fallback path. Guardian and output-router evals only need AI Gateway access. ## Authoring Rules - Put full-runtime integration cases that must never regress under `evals/integration/**` using `describeEval()` with `slackEvals`. Prefer deterministic assertions; keep criteria only when the case still needs light quality scoring. - Put behavioral cases under `evals/conversation/`, `evals/agent/`, or `evals//` using `describeEval()` with `slackEvals`. - Add isolated Guardian decision snapshots under `evals/guardian/` using `describeEval()` with `guardianEvals`. Feed exact `ToolActionProposal` objects and assert only the expected `allow` / `ask` / `deny` decision. +- Add isolated prepare-reply cases under `evals/output-router/` using `describeEval()` with `outputRouterEvals`. Feed real assistant message text and check `silent` or `reply`. - Put messages that should be pending before processing starts in `initialEvents`. - Put ordinary later events in `events`; each is delivered after preceding work settles. - Wrap messages with `steer(...)` when they should arrive through normal ingress while the preceding agent run is active. @@ -205,7 +220,8 @@ Organize files by suite policy first, then by the user-visible area they exercis - `evals/integration/`: strict full-runtime integration cases (hard pass/fail). - `evals/conversation/`, `evals/agent/`, `evals//`: agent-behavior cases (score-gated in CI). -- `evals/guardian/`: isolated Guardian decision snapshots (no main agent; hard pass/fail). +- `evals/guardian/`: isolated action-review snapshots (no main agent; hard pass/fail). +- `evals/output-router/`: isolated prepare-reply cases (no main agent; hard pass/fail). - Use short behavior nouns for filenames: `routing.eval.ts`, `delivery.eval.ts`, `credentials.eval.ts`. - Keep one coherent behavior area per file. Split files when cases exercise independently understandable journeys. - Keep shared setup in a nearby `helpers.ts`; helpers are not eval files and do not define suites. diff --git a/packages/junior-evals/create-full-runtime-eval-config.ts b/packages/junior-evals/create-full-runtime-eval-config.ts new file mode 100644 index 0000000000..b9277b6aa1 --- /dev/null +++ b/packages/junior-evals/create-full-runtime-eval-config.ts @@ -0,0 +1,109 @@ +import { randomUUID } from "node:crypto"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; +import DefaultEvalReporter from "vitest-evals/reporter"; +import { loadJuniorTestEnvFiles } from "../junior/tests/fixtures/env"; + +const evalsPackageRoot = path.dirname(fileURLToPath(import.meta.url)); +const juniorPackageRoot = path.resolve(evalsPackageRoot, "../junior"); +const workspaceRoot = path.resolve(evalsPackageRoot, "../.."); +const pluginApiPackageRoot = path.resolve( + evalsPackageRoot, + "../junior-plugin-api", +); +const memoryPackageRoot = path.resolve(evalsPackageRoot, "../junior-memory"); + +// Leave room for harness cleanup and rubric judging after a reply reaches its +// separate 60-second behavior budget. +const EVAL_TEST_TIMEOUT_MS = 120_000; + +export type FullRuntimeEvalSuiteOptions = { + /** Suite id used for Redis key prefix and default results file name. */ + name: string; + include: string[]; + exclude?: string[]; + /** Extra setup files after the shared full-runtime setup chain. */ + setupFiles?: string[]; + env?: Record; +}; + +/** + * Shared Vitest config for full Slack/runtime eval suites. + * + * Suite configs stay thin: name, include/exclude, and optional env/setup only. + * Guardian and output-router stay on their own lightweight configs. + */ +export function createFullRuntimeEvalConfig( + options: FullRuntimeEvalSuiteOptions, +) { + const evalReportPath = path.resolve( + evalsPackageRoot, + process.env.VITEST_EVALS_OUTPUT_FILE ?? `${options.name}-results.json`, + ); + + loadJuniorTestEnvFiles({ + workspaceRoot, + packageRoots: [juniorPackageRoot, evalsPackageRoot], + }); + + process.env.JUNIOR_SECRET = "junior-test-secret"; + process.env.JUNIOR_BASE_URL ??= "https://junior.example.com"; + process.env.JUNIOR_STATE_ADAPTER = "redis"; + process.env.JUNIOR_STATE_KEY_PREFIX ??= `junior:eval-${options.name}:${randomUUID()}`; + process.env.REDIS_URL = + process.env.JUNIOR_EVAL_REDIS_URL?.trim() || "redis://127.0.0.1:6382"; + const evalRedisHostname = new URL(process.env.REDIS_URL).hostname; + if (evalRedisHostname !== "localhost" && evalRedisHostname !== "127.0.0.1") { + throw new Error( + `JUNIOR_EVAL_REDIS_URL must point at localhost or 127.0.0.1, got ${evalRedisHostname}`, + ); + } + process.env.AI_MODEL = "xai/grok-4.5"; + process.env.AI_FAST_MODEL = "anthropic/claude-haiku-4.5"; + process.env.AI_GUARDIAN_MODEL = "openai/gpt-5.6-luna"; + process.env.AI_HANDOFF_MODEL = "openai/gpt-5.6-sol"; + process.env.AI_MODEL_PROFILES = JSON.stringify({ + coding: "openai/gpt-5.6-sol", + }); + process.env.VITEST_EVALS_REPLAY_MODE ??= "auto"; + + for (const [key, value] of Object.entries(options.env ?? {})) { + process.env[key] = value; + } + + return defineConfig({ + resolve: { + alias: { + "@": path.resolve(juniorPackageRoot, "src"), + "@sentry/junior-memory": path.resolve(memoryPackageRoot, "src/index.ts"), + "@sentry/junior-plugin-api": path.resolve( + pluginApiPackageRoot, + "src/index.ts", + ), + }, + // Vite 8 resolves tsconfig `paths` natively here: + // https://vite.dev/config/shared-options.html#resolve-tsconfigpaths + // The aliases above keep workspace package internals on source instead of package dist. + tsconfigPaths: true, + }, + test: { + environment: "node", + fileParallelism: false, + globalSetup: [path.resolve(evalsPackageRoot, "global-setup.ts")], + include: options.include, + ...(options.exclude ? { exclude: options.exclude } : undefined), + maxWorkers: 1, + setupFiles: [ + path.resolve(evalsPackageRoot, "src/setup.ts"), + path.resolve(juniorPackageRoot, "tests/msw/setup.ts"), + path.resolve(juniorPackageRoot, "tests/fixtures/postgres/setup.ts"), + path.resolve(juniorPackageRoot, "tests/fixtures/experimental-setup.ts"), + ...(options.setupFiles ?? []), + ], + outputFile: { json: evalReportPath }, + reporters: [new DefaultEvalReporter(), "json"], + testTimeout: EVAL_TEST_TIMEOUT_MS, + }, + }); +} diff --git a/packages/junior-evals/evals/github-actions.md b/packages/junior-evals/evals/github-actions.md index 25954ee30a..e0c6040c1a 100644 --- a/packages/junior-evals/evals/github-actions.md +++ b/packages/junior-evals/evals/github-actions.md @@ -65,26 +65,28 @@ Only needed for the token-based fallback above. Create an AI Gateway key in the ## Triggering Evals On A PR -Three independent workflows run on pull requests: +Four independent workflows run on pull requests: - `Behavioral evals` runs Slack/agent evals when behavioral eval files/harness changed or the PR has `trigger-evals-behavioral` / `trigger-evals` - `Integration evals` runs system evals when integration eval files/harness changed or the PR has `trigger-evals-integration` / `trigger-evals` -- `Guardian evals` runs isolated Guardian snapshots when Guardian eval files/harness changed or the PR has `trigger-evals-guardian` / `trigger-evals` +- `Guardian evals` runs isolated action-review snapshots when Guardian eval files/harness changed, Guardian policy changed, or the PR has `trigger-evals-guardian` / `trigger-evals` +- `Output-router evals` runs isolated prepare-reply cases when those eval files/harness changed, `output-router.ts` changed, or the PR has `trigger-evals-output-router` / `trigger-evals` -Suite labels follow `trigger-evals-[domain]`. Adding a trigger label fires immediately. If the label is already on the PR, future `synchronize` events still run the matching suite(s). Product source under `packages/junior/src/**` does not auto-run evals, except Guardian policy changes in `packages/junior/src/chat/services/guardian-action-policy.ts`. +Suite labels follow `trigger-evals-[domain]`. Adding a trigger label fires immediately. If the label is already on the PR, future `synchronize` events still run the matching suite(s). Product source under `packages/junior/src/**` does not auto-run evals, except Guardian policy changes in `packages/junior/src/chat/services/guardian-action-policy.ts` and prepare-path changes in `packages/junior/src/chat/services/output-router.ts`. -Guardian evals only need gateway credentials. Behavioral and integration evals still need gateway plus sandbox access. +Guardian and output-router evals only need gateway credentials. Behavioral and integration evals still need gateway plus sandbox access. ## Verification After adding secrets: 1. Push a commit to the PR, or add the matching `trigger-evals*` label. -2. Open the matching `Behavioral evals`, `Integration evals`, or `Guardian evals` workflow summary. +2. Open the matching `Behavioral evals`, `Integration evals`, `Guardian evals`, or `Output-router evals` workflow summary. 3. Confirm its `*/select` job reports `will_run: true` and the required credentials as ready. 4. For behavioral runs, confirm each `behavioral / shard *` job has a shard summary, `behavioral / report` has the combined summary, and the `behavioral / score` Check Run shows the pass-rate gate title. 5. For integration runs, confirm the `integration / shard *` jobs completed. Any case miss fails those jobs hard. 6. For Guardian runs, confirm the `guardian / run` job summary published and the job completed. Exact decision mismatches fail that job hard. +7. For output-router runs, confirm the `output-router / run` job summary published and the job completed. Prepare `silent` / `reply` mismatches fail that job hard. ## Score-Based CI Gate @@ -100,7 +102,7 @@ If Check Run publishing is skipped or fails, the report step still fails on a re When the aggregate gate passes, individual case misses are warnings rather than failures. Setup crashes and missing result files still fail the report job hard. -Integration shards fail hard on any case miss and do not use the aggregate floor. Guardian snapshots assert exact `allow` / `ask` / `deny` decisions, publish their own job summary, and fail `guardian / run` on mismatch. +Integration shards fail hard on any case miss and do not use the aggregate floor. Guardian snapshots assert exact `allow` / `ask` / `deny` decisions, publish their own job summary, and fail `guardian / run` on mismatch. Output-router cases assert prepare `silent` / `reply` outcomes, publish their own job summary, and fail `output-router / run` on mismatch. If `sandbox_ready` is false, either `VERCEL_OIDC_TOKEN` is missing or the fallback token set is incomplete. diff --git a/packages/junior-evals/evals/output-router/visible-reply.eval.ts b/packages/junior-evals/evals/output-router/visible-reply.eval.ts new file mode 100644 index 0000000000..533d2f4fb3 --- /dev/null +++ b/packages/junior-evals/evals/output-router/visible-reply.eval.ts @@ -0,0 +1,137 @@ +/** + * Isolated prepare-reply cases. + * + * Each case feeds real assistant message text into prepareAssistantReply and + * checks silent vs reply. This suite does not run the main agent or Slack + * transport. Delivery is covered elsewhere. + */ +import { describeEval } from "vitest-evals"; +import { NO_REPLY_MARKER } from "@/chat/no-reply"; +import { + OUTPUT_REPLY_HARD_MAX_CHARS, + OUTPUT_REPLY_SOFT_MAX_CHARS, +} from "@/chat/services/output-router"; +import { outputRouterEvals } from "../../src/output-router-harness"; + +/** Real long steering comparison that should not remain a wall of text. */ +const LONG_STEERING_ESSAY = [ + "**yeah — steering is the weaker half of this comparison.** openclaw treats mid-run guidance as the default path; junior treats it as a gated special case. that mismatch is the reliability gap.", + "", + "### what openclaw does", + "", + "- default queue mode is **`steer`** for normal inbound messages while a run is active", + "- injects at **tool-launch + model** boundaries; unfinished sequential tools get synthetic `Skipped due to queued user message.` results, then the steer is model-visible before the next decision", + "- if the runtime can’t accept a steer, it **falls back to followup** instead of dropping", + "- explicit `/steer ` works **independent of queue mode**; `/queue interrupt|followup|collect` are first-class", + "", + "### what junior does", + "", + "1. **hard eligibility gate before anything mid-run**", + ' - mailbox `delivery: "interrupt"` only if `isMention` **or** leading `!!`', + " - otherwise `defer` → waits for the turn to finish", + "", + "2. **cross-actor default is inverted vs openclaw**", + " - `JUNIOR_CROSS_ACTOR_MID_RUN_MODE` defaults to **`follow_up`**", + " - mid-run accept only if: mode=`steer` **or** same author as active turn **or** `!!`", + "", + "3. **drain is late**", + " - steers are injected in `prepareNextTurnWithContext` only (after a tool finishes, before the next model call)", + " - `beforeToolCall` does **not** skip unstarted tools for a pending steer", + " - long tool / parallel tool batches keep going until the next model boundary", + "", + "4. **docs oversell the UX**", + '- “reply with plain language… course-correct at any point”', + " - code reality: plain non-mention reply mid-run is usually a **follow-up**, not a steer", + " - `!!` exists as the override, but it’s basically an internal escape hatch, not a real user contract", + "", + "### why it feels flaky", + "", + "| situation | junior | openclaw |", + "|---|---|---|", + "| plain reply mid-run, no `@` | usually waits (`defer`) | steers (default) |", + "| `@jr stop` mid-run, same author | steers at **next model** boundary | steers earlier (tool-launch too) |", + "| other person `@jr …` | waits unless mode=`steer` or `!!` | steers (sender labeled) |", + "| steer while tools still queued | tools keep running | unstarted tools skipped |", + "| inject fails | often warn + soft fail | park → followup fallback |", + "", + "### bottom line", + "", + "openclaw: **steer-by-default + early boundary + never drop**.", + "", + "junior: **mention/`!!`-gated + model-boundary-only + cross-actor follow_up default**.", + "", + "so “ours isn’t working reliably” is less random failure and more **product/runtime contract**: most of what people think is steering is actually deferred follow-up, and even real steers land later than openclaw’s.", + "", + "if you want parity, the minimum set is roughly:", + "", + "1. default mid-run path closer to openclaw `steer` (at least same-thread subscribed traffic)", + "2. drain / skip at tool-launch, not only `prepareNextTurn`", + "3. guaranteed fallback to next-turn follow-up when inject can’t land", + "4. docs that match the real gates (`@` / `!!` / cross-actor mode)", + "", + "want me to turn that into a concrete junior issue/PR plan?", +].join("\n"); + +/** Real maintain-PR status chatter that should stay silent. */ +const STATUS_ONLY_WITH_MARKER = [ + "Same main baseline miss on createAgentDispatchWorkRouter — not caused by this PR. No PR fix.", + "", + NO_REPLY_MARKER, +].join("\n"); + +/** Real explanation of silence that must stay a reply. */ +const SILENCE_PROTOCOL_EXPLANATION = [ + `Intentional silence uses the exact whole-message marker ${NO_REPLY_MARKER}.`, + "If the marker is only mentioned in a normal answer, that answer should still post.", + "Only a message that is exactly the marker stays silent.", +].join(" "); + +describeEval("Visible Reply Prepare", outputRouterEvals, (it) => { + it("when the assistant writes a long explanatory essay, return a short reply", async ({ + run, + }) => { + if (LONG_STEERING_ESSAY.length <= OUTPUT_REPLY_SOFT_MAX_CHARS) { + throw new Error("fixture must exceed the soft max length"); + } + + await run({ + text: LONG_STEERING_ESSAY, + expectedKind: "reply", + // Soft max is the model target. Hard max is the product ceiling. + maxChars: OUTPUT_REPLY_HARD_MAX_CHARS, + // Condensation, not a near-full essay under the hard cap. + maxOriginalRatio: 0.5, + mustInclude: ["steer"], + mustNotInclude: ["### what openclaw does", "| situation | junior |"], + }); + }); + + it("when maintain work ends with status chatter and a silence marker, stay silent", async ({ + run, + }) => { + await run({ + text: STATUS_ONLY_WITH_MARKER, + expectedKind: "silent", + }); + }); + + it("when the message explains how silence works, keep the explanation", async ({ + run, + }) => { + await run({ + text: SILENCE_PROTOCOL_EXPLANATION, + expectedKind: "reply", + // Keep the marker when explaining silence. Do not assert incidental wording. + mustInclude: [NO_REPLY_MARKER], + }); + }); + + it("when the whole message is only the silence marker, stay silent", async ({ + run, + }) => { + await run({ + text: NO_REPLY_MARKER, + expectedKind: "silent", + }); + }); +}); diff --git a/packages/junior-evals/output-router-global-setup.ts b/packages/junior-evals/output-router-global-setup.ts new file mode 100644 index 0000000000..2715bbb216 --- /dev/null +++ b/packages/junior-evals/output-router-global-setup.ts @@ -0,0 +1,15 @@ +import { installEvalAiGatewayDispatcher } from "./src/eval-ai-gateway-dispatcher"; + +/** + * Set up the lightweight output-router eval invocation. + * + * These cases only need AI Gateway access. They intentionally skip Postgres, + * Redis fixtures, MSW, plugin catalogs, and sandbox egress. + */ +export default async function setup(): Promise<() => Promise> { + const restoreAiGatewayDispatcher = installEvalAiGatewayDispatcher(); + process.stdout.write( + "[evals:output-router] AI Gateway dispatcher ready (no sandbox egress)\n", + ); + return restoreAiGatewayDispatcher; +} diff --git a/packages/junior-evals/package.json b/packages/junior-evals/package.json index 129bebcf99..363c485bef 100644 --- a/packages/junior-evals/package.json +++ b/packages/junior-evals/package.json @@ -10,6 +10,7 @@ "evals:behavioral": "vitest run -c vitest.evals.behavioral.config.ts", "evals:integration": "vitest run -c vitest.evals.integration.config.ts", "evals:guardian": "vitest run -c vitest.evals.guardian.config.ts", + "evals:output-router": "vitest run -c vitest.evals.output-router.config.ts", "evals:record": "VITEST_EVALS_REPLAY_MODE=record vitest run -c vitest.evals.behavioral.config.ts" }, "devDependencies": { diff --git a/packages/junior-evals/src/output-router-harness.ts b/packages/junior-evals/src/output-router-harness.ts new file mode 100644 index 0000000000..3ac36005ad --- /dev/null +++ b/packages/junior-evals/src/output-router-harness.ts @@ -0,0 +1,198 @@ +/** + * Isolated prepare-reply harness. + * + * Calls prepareAssistantReply with one assistant message. No main agent, Slack + * transport, sandbox egress, or Postgres. + */ +import { + createHarness, + type DescribeEvalOptions, + type JsonValue, +} from "vitest-evals"; +import { completeObject } from "@/chat/pi/client"; +import { + OUTPUT_REPLY_SOFT_MAX_CHARS, + prepareAssistantReply, + type PreparedAssistantReply, +} from "@/chat/services/output-router"; + +export type OutputRouterEvalKind = "silent" | "reply"; + +export interface OutputRouterEvalInput { + /** Original assistant message text. */ + text: string; + /** Expected prepare kind. */ + expectedKind: OutputRouterEvalKind; + /** + * Optional upper bound for reply text length. Defaults to the soft max when + * expectedKind is reply and this is omitted. + */ + maxChars?: number; + /** + * Optional max ratio of original length for a reply. Use for long-input + * condensation checks without requiring the soft max exactly. + */ + maxOriginalRatio?: number; + /** Substrings that must appear in a reply (case-insensitive). */ + mustInclude?: string[]; + /** Substrings that must not appear in a reply (case-insensitive). */ + mustNotInclude?: string[]; +} + +export interface OutputRouterEvalOutput extends Record { + costUsd: number | null; + expectedKind: OutputRouterEvalKind; + kind: OutputRouterEvalKind; + reason: string; + text: string | null; + textLength: number | null; +} + +function resolveFastModelId(): string { + const configured = process.env.AI_FAST_MODEL?.trim(); + if (configured) { + return configured; + } + return "openai/gpt-5.6-luna"; +} + +function includesInsensitive(haystack: string, needle: string): boolean { + return haystack.toLowerCase().includes(needle.toLowerCase()); +} + +/** Run one assistant message through the production prepare boundary. */ +export async function prepareVisibleReply( + text: string, +): Promise { + return prepareAssistantReply({ + completeObject, + fastModelId: resolveFastModelId(), + text, + }); +} + +function preparedSummary(prepared: PreparedAssistantReply): string { + if (prepared.kind === "silent") { + return `kind=silent reason=${JSON.stringify(prepared.reason)}`; + } + return `kind=reply reason=${JSON.stringify(prepared.reason)} text=${JSON.stringify(prepared.text)}`; +} + +function assertPreparedReply( + input: OutputRouterEvalInput, + prepared: PreparedAssistantReply, +): void { + if (prepared.kind !== input.expectedKind) { + throw new Error( + `output-router prepared ${prepared.kind}; expected ${input.expectedKind} (${preparedSummary(prepared)})`, + ); + } + + if (prepared.kind === "silent") { + return; + } + + const maxChars = input.maxChars ?? OUTPUT_REPLY_SOFT_MAX_CHARS; + if (prepared.text.length > maxChars) { + throw new Error( + `output-router reply length ${prepared.text.length} exceeds max ${maxChars} (${preparedSummary(prepared)})`, + ); + } + + if (input.maxOriginalRatio !== undefined) { + const maxFromOriginal = Math.floor( + input.text.length * input.maxOriginalRatio, + ); + if (prepared.text.length > maxFromOriginal) { + throw new Error( + `output-router reply length ${prepared.text.length} exceeds ${input.maxOriginalRatio} of original ${input.text.length} (${preparedSummary(prepared)})`, + ); + } + } + + for (const needle of input.mustInclude ?? []) { + if (!includesInsensitive(prepared.text, needle)) { + throw new Error( + `output-router reply missing required text ${JSON.stringify(needle)} (${preparedSummary(prepared)})`, + ); + } + } + + for (const needle of input.mustNotInclude ?? []) { + if (includesInsensitive(prepared.text, needle)) { + throw new Error( + `output-router reply contains forbidden text ${JSON.stringify(needle)} (${preparedSummary(prepared)})`, + ); + } + } +} + +/** Vitest-evals harness for isolated prepare-reply cases. */ +export const outputRouterHarness = createHarness< + OutputRouterEvalInput, + OutputRouterEvalOutput +>({ + name: "output-router", + run: async ({ input }) => { + const prepared = await prepareVisibleReply(input.text); + assertPreparedReply(input, prepared); + + const output: OutputRouterEvalOutput = { + costUsd: prepared.costUsd ?? null, + expectedKind: input.expectedKind, + kind: prepared.kind, + reason: prepared.reason, + text: prepared.kind === "reply" ? prepared.text : null, + textLength: prepared.kind === "reply" ? prepared.text.length : null, + }; + + return { + output, + events: [ + { + type: "message", + role: "user", + content: [ + `Expected kind: ${input.expectedKind}`, + "", + "Original assistant text:", + input.text, + ].join("\n"), + }, + { + type: "message", + role: "assistant", + content: + prepared.kind === "silent" + ? [`Kind: silent`, `Reason: ${prepared.reason}`].join("\n") + : [ + `Kind: reply`, + `Reason: ${prepared.reason}`, + `Length: ${prepared.text.length}`, + "", + prepared.text, + ].join("\n"), + }, + ], + usage: { + provider: "vercel-ai-gateway", + model: resolveFastModelId(), + ...(prepared.costUsd !== undefined + ? { metadata: { costUsd: prepared.costUsd } } + : {}), + }, + }; + }, +}); + +/** Shared suite options for isolated prepare-reply evals. */ +export const outputRouterEvals = { + harness: outputRouterHarness, + // Kind, length, and required text are checked in the harness. + judges: [], + judgeThreshold: null, +} satisfies DescribeEvalOptions< + OutputRouterEvalInput, + OutputRouterEvalOutput, + typeof outputRouterHarness +>; diff --git a/packages/junior-evals/src/output-router-setup.ts b/packages/junior-evals/src/output-router-setup.ts new file mode 100644 index 0000000000..d45a226eab --- /dev/null +++ b/packages/junior-evals/src/output-router-setup.ts @@ -0,0 +1,6 @@ +/** + * Per-file setup for isolated output-router evals. + * + * Kept intentionally empty beyond documenting the boundary: these cases must + * not depend on Slack egress, Postgres, Redis fixture resets, or MSW. + */ diff --git a/packages/junior-evals/vitest.evals.behavioral.config.ts b/packages/junior-evals/vitest.evals.behavioral.config.ts index f53a802d15..c04f97e084 100644 --- a/packages/junior-evals/vitest.evals.behavioral.config.ts +++ b/packages/junior-evals/vitest.evals.behavioral.config.ts @@ -1,79 +1,13 @@ -import { defineConfig } from "vitest/config"; -import { randomUUID } from "node:crypto"; -import DefaultEvalReporter from "vitest-evals/reporter"; -import path from "node:path"; -import { loadJuniorTestEnvFiles } from "../junior/tests/fixtures/env"; - -const juniorPackageRoot = path.resolve(__dirname, "../junior"); -const workspaceRoot = path.resolve(__dirname, "../.."); -const evalsPackageRoot = __dirname; -const pluginApiPackageRoot = path.resolve(__dirname, "../junior-plugin-api"); -const memoryPackageRoot = path.resolve(__dirname, "../junior-memory"); -// Leave room for harness cleanup and rubric judging after a reply reaches its -// separate 60-second behavior budget. -const EVAL_TEST_TIMEOUT_MS = 120_000; -const evalReportPath = path.resolve( - evalsPackageRoot, - process.env.VITEST_EVALS_OUTPUT_FILE ?? "behavioral-results.json", -); - -loadJuniorTestEnvFiles({ - workspaceRoot, - packageRoots: [juniorPackageRoot, evalsPackageRoot], -}); - -process.env.JUNIOR_SECRET = "junior-test-secret"; -process.env.JUNIOR_BASE_URL ??= "https://junior.example.com"; -process.env.JUNIOR_STATE_ADAPTER = "redis"; -process.env.JUNIOR_STATE_KEY_PREFIX ??= `junior:eval-behavioral:${randomUUID()}`; -process.env.REDIS_URL = - process.env.JUNIOR_EVAL_REDIS_URL?.trim() || "redis://127.0.0.1:6382"; -const evalRedisHostname = new URL(process.env.REDIS_URL).hostname; -if (evalRedisHostname !== "localhost" && evalRedisHostname !== "127.0.0.1") { - throw new Error( - `JUNIOR_EVAL_REDIS_URL must point at localhost or 127.0.0.1, got ${evalRedisHostname}`, - ); -} -process.env.AI_MODEL = "xai/grok-4.5"; -process.env.AI_FAST_MODEL = "anthropic/claude-haiku-4.5"; -process.env.AI_GUARDIAN_MODEL = "openai/gpt-5.6-luna"; -process.env.AI_HANDOFF_MODEL = "openai/gpt-5.6-sol"; -process.env.AI_MODEL_PROFILES = JSON.stringify({ - coding: "openai/gpt-5.6-sol", -}); -process.env.VITEST_EVALS_REPLAY_MODE ??= "auto"; - -export default defineConfig({ - resolve: { - alias: { - "@": path.resolve(juniorPackageRoot, "src"), - "@sentry/junior-memory": path.resolve(memoryPackageRoot, "src/index.ts"), - "@sentry/junior-plugin-api": path.resolve( - pluginApiPackageRoot, - "src/index.ts", - ), - }, - // Vite 8 resolves tsconfig `paths` natively here: - // https://vite.dev/config/shared-options.html#resolve-tsconfigpaths - // The aliases above keep workspace package internals on source instead of package dist. - tsconfigPaths: true, - }, - test: { - environment: "node", - fileParallelism: false, - globalSetup: [path.resolve(__dirname, "global-setup.ts")], - // Behavioral quality cases. Integration and Guardian suites have their own configs. - include: ["evals/**/*.eval.ts"], - exclude: ["evals/guardian/**", "evals/integration/**"], - maxWorkers: 1, - setupFiles: [ - path.resolve(__dirname, "src/setup.ts"), - path.resolve(juniorPackageRoot, "tests/msw/setup.ts"), - path.resolve(juniorPackageRoot, "tests/fixtures/postgres/setup.ts"), - path.resolve(juniorPackageRoot, "tests/fixtures/experimental-setup.ts"), - ], - outputFile: { json: evalReportPath }, - reporters: [new DefaultEvalReporter(), "json"], - testTimeout: EVAL_TEST_TIMEOUT_MS, - }, +import { createFullRuntimeEvalConfig } from "./create-full-runtime-eval-config"; + +// Behavioral quality cases. Integration, Guardian, and output-router each have +// their own suite configs. +export default createFullRuntimeEvalConfig({ + name: "behavioral", + include: ["evals/**/*.eval.ts"], + exclude: [ + "evals/guardian/**", + "evals/integration/**", + "evals/output-router/**", + ], }); diff --git a/packages/junior-evals/vitest.evals.integration.config.ts b/packages/junior-evals/vitest.evals.integration.config.ts index 904c987fba..18d3a5c748 100644 --- a/packages/junior-evals/vitest.evals.integration.config.ts +++ b/packages/junior-evals/vitest.evals.integration.config.ts @@ -1,78 +1,7 @@ -import { defineConfig } from "vitest/config"; -import { randomUUID } from "node:crypto"; -import DefaultEvalReporter from "vitest-evals/reporter"; -import path from "node:path"; -import { loadJuniorTestEnvFiles } from "../junior/tests/fixtures/env"; +import { createFullRuntimeEvalConfig } from "./create-full-runtime-eval-config"; -const juniorPackageRoot = path.resolve(__dirname, "../junior"); -const workspaceRoot = path.resolve(__dirname, "../.."); -const evalsPackageRoot = __dirname; -const pluginApiPackageRoot = path.resolve(__dirname, "../junior-plugin-api"); -const memoryPackageRoot = path.resolve(__dirname, "../junior-memory"); -// Leave room for harness cleanup and rubric judging after a reply reaches its -// separate 60-second behavior budget. -const EVAL_TEST_TIMEOUT_MS = 120_000; -const evalReportPath = path.resolve( - evalsPackageRoot, - process.env.VITEST_EVALS_OUTPUT_FILE ?? "integration-results.json", -); - -loadJuniorTestEnvFiles({ - workspaceRoot, - packageRoots: [juniorPackageRoot, evalsPackageRoot], -}); - -process.env.JUNIOR_SECRET = "junior-test-secret"; -process.env.JUNIOR_BASE_URL ??= "https://junior.example.com"; -process.env.JUNIOR_STATE_ADAPTER = "redis"; -process.env.JUNIOR_STATE_KEY_PREFIX ??= `junior:eval-integration:${randomUUID()}`; -process.env.REDIS_URL = - process.env.JUNIOR_EVAL_REDIS_URL?.trim() || "redis://127.0.0.1:6382"; -const evalRedisHostname = new URL(process.env.REDIS_URL).hostname; -if (evalRedisHostname !== "localhost" && evalRedisHostname !== "127.0.0.1") { - throw new Error( - `JUNIOR_EVAL_REDIS_URL must point at localhost or 127.0.0.1, got ${evalRedisHostname}`, - ); -} -process.env.AI_MODEL = "xai/grok-4.5"; -process.env.AI_FAST_MODEL = "anthropic/claude-haiku-4.5"; -process.env.AI_GUARDIAN_MODEL = "openai/gpt-5.6-luna"; -process.env.AI_HANDOFF_MODEL = "openai/gpt-5.6-sol"; -process.env.AI_MODEL_PROFILES = JSON.stringify({ - coding: "openai/gpt-5.6-sol", -}); -process.env.VITEST_EVALS_REPLAY_MODE ??= "auto"; - -export default defineConfig({ - resolve: { - alias: { - "@": path.resolve(juniorPackageRoot, "src"), - "@sentry/junior-memory": path.resolve(memoryPackageRoot, "src/index.ts"), - "@sentry/junior-plugin-api": path.resolve( - pluginApiPackageRoot, - "src/index.ts", - ), - }, - // Vite 8 resolves tsconfig `paths` natively here: - // https://vite.dev/config/shared-options.html#resolve-tsconfigpaths - // The aliases above keep workspace package internals on source instead of package dist. - tsconfigPaths: true, - }, - test: { - environment: "node", - fileParallelism: false, - globalSetup: [path.resolve(__dirname, "global-setup.ts")], - // Strict system-correctness cases. Any failure fails the suite hard. - include: ["evals/integration/**/*.eval.ts"], - maxWorkers: 1, - setupFiles: [ - path.resolve(__dirname, "src/setup.ts"), - path.resolve(juniorPackageRoot, "tests/msw/setup.ts"), - path.resolve(juniorPackageRoot, "tests/fixtures/postgres/setup.ts"), - path.resolve(juniorPackageRoot, "tests/fixtures/experimental-setup.ts"), - ], - outputFile: { json: evalReportPath }, - reporters: [new DefaultEvalReporter(), "json"], - testTimeout: EVAL_TEST_TIMEOUT_MS, - }, +// Strict system-correctness cases. Any failure fails the suite hard. +export default createFullRuntimeEvalConfig({ + name: "integration", + include: ["evals/integration/**/*.eval.ts"], }); diff --git a/packages/junior-evals/vitest.evals.output-router.config.ts b/packages/junior-evals/vitest.evals.output-router.config.ts new file mode 100644 index 0000000000..ed68696612 --- /dev/null +++ b/packages/junior-evals/vitest.evals.output-router.config.ts @@ -0,0 +1,60 @@ +import { defineConfig } from "vitest/config"; +import { randomUUID } from "node:crypto"; +import DefaultEvalReporter from "vitest-evals/reporter"; +import path from "node:path"; +import { loadJuniorTestEnvFiles } from "../junior/tests/fixtures/env"; + +const juniorPackageRoot = path.resolve(__dirname, "../junior"); +const workspaceRoot = path.resolve(__dirname, "../.."); +const evalsPackageRoot = __dirname; +const pluginApiPackageRoot = path.resolve(__dirname, "../junior-plugin-api"); +const memoryPackageRoot = path.resolve(__dirname, "../junior-memory"); +// Leave room for provider retry inside the separate 60-second prepare budget. +const OUTPUT_ROUTER_EVAL_TEST_TIMEOUT_MS = 90_000; +const evalReportPath = path.resolve( + evalsPackageRoot, + process.env.VITEST_EVALS_OUTPUT_FILE ?? "output-router-results.json", +); + +loadJuniorTestEnvFiles({ + workspaceRoot, + packageRoots: [juniorPackageRoot, evalsPackageRoot], +}); + +process.env.JUNIOR_SECRET = "junior-test-secret"; +process.env.JUNIOR_BASE_URL ??= "https://junior.example.com"; +// Prepare cases do not touch Redis state, but keep a loopback default so any +// accidental shared import that reads REDIS_URL stays sandboxed. +process.env.JUNIOR_STATE_ADAPTER = "redis"; +process.env.JUNIOR_STATE_KEY_PREFIX ??= `junior:eval-output-router:${randomUUID()}`; +process.env.REDIS_URL = + process.env.JUNIOR_EVAL_REDIS_URL?.trim() || "redis://127.0.0.1:6382"; +// Prepare path uses the fast model on one assistant message. +process.env.AI_FAST_MODEL ??= "openai/gpt-5.6-luna"; + +export default defineConfig({ + resolve: { + alias: { + "@": path.resolve(juniorPackageRoot, "src"), + "@sentry/junior-memory": path.resolve(memoryPackageRoot, "src/index.ts"), + "@sentry/junior-plugin-api": path.resolve( + pluginApiPackageRoot, + "src/index.ts", + ), + }, + // Vite 8 resolves tsconfig `paths` natively here: + // https://vite.dev/config/shared-options.html#resolve-tsconfigpaths + tsconfigPaths: true, + }, + test: { + environment: "node", + fileParallelism: false, + globalSetup: [path.resolve(__dirname, "output-router-global-setup.ts")], + include: ["evals/output-router/**/*.eval.ts"], + maxWorkers: 1, + setupFiles: [path.resolve(__dirname, "src/output-router-setup.ts")], + outputFile: { json: evalReportPath }, + reporters: [new DefaultEvalReporter(), "json"], + testTimeout: OUTPUT_ROUTER_EVAL_TEST_TIMEOUT_MS, + }, +}); diff --git a/packages/junior/src/chat/README.md b/packages/junior/src/chat/README.md index 77e9c867d3..7bfc6b71c5 100644 --- a/packages/junior/src/chat/README.md +++ b/packages/junior/src/chat/README.md @@ -20,8 +20,11 @@ file. 6. `agent/` emits every completed, tool-free visible assistant message through one awaited delivery port with the completed Pi message that produced it; provider adapters deliver, then commit that agent message before the visible - reply in one transaction. Tool-bearing assistant text remains internal to - the agent loop. + reply in one transaction. When experimental `output-router` is enabled, a + fast-model pass may change only the visible reply text (silence or + shortening) while keeping the `SOUL.md` personality voice. The original agent + message stays in history. Tool-bearing assistant text remains internal to the + agent loop. 7. The completed run result supplies diagnostics and artifacts; successful delivery or intentional no-reply completion commits the durable turn outcome. diff --git a/packages/junior/src/chat/agent/index.ts b/packages/junior/src/chat/agent/index.ts index c01760bf50..230cb0d4f0 100644 --- a/packages/junior/src/chat/agent/index.ts +++ b/packages/junior/src/chat/agent/index.ts @@ -78,6 +78,8 @@ import { isTurnInputCommitLostError } from "@/chat/runtime/turn"; import type { AgentRunOutcome } from "@/chat/runtime/agent-run-outcome"; import { buildTurnResult } from "@/chat/services/turn-result"; import { decideReply } from "@/chat/services/assistant-reply"; +import { prepareAssistantMessage } from "@/chat/services/output-router"; +import { isExperimentalFeatureEnabled } from "@/chat/experimental"; import { findProviderError, getProviderErrorAttributes, @@ -1082,12 +1084,36 @@ async function executeAgentRunInPrivacyContext( const deliverAssistantMessage = async ( message: Parameters[0], ): Promise => { - const decision = decideReply(message); - if (decision.kind !== "deliver" || !delivery) { + if (!delivery) { return; } + + // Keep the original agent message for history. Only the visible reply text + // may change when output-router is on. + let visibleText: string | undefined; + if (isExperimentalFeatureEnabled("output-router")) { + const prepared = await prepareAssistantMessage({ + completeObject, + context: { + conversationId, + runId, + }, + fastModelId: botConfig.fastModelId, + message, + }); + if (prepared.kind === "skip" || prepared.kind === "silent") { + return; + } + visibleText = prepared.text; + } else { + const decision = decideReply(message); + if (decision.kind !== "deliver") { + return; + } + } + try { - await delivery(message); + await delivery(message, visibleText); acceptedToolFreeAssistant = true; } catch (error) { assistantMessageDeliveryError = new AssistantMessageDeliveryError( diff --git a/packages/junior/src/chat/agent/types.ts b/packages/junior/src/chat/agent/types.ts index a2fc233911..29e4e9b825 100644 --- a/packages/junior/src/chat/agent/types.ts +++ b/packages/junior/src/chat/agent/types.ts @@ -124,6 +124,9 @@ export type AgentRunState = { /** * Delivers completed tool-free assistant messages in model order. * + * `message` is the original agent message for history. `text` is the + * destination-visible reply when it differs from the agent message text. + * * The runner must commit the preceding agent boundary before invoking this * port; the accepted reply transaction appends only this message. * @@ -131,7 +134,10 @@ export type AgentRunState = { * implementations after the core Turn lifecycle stores each completed * assistant Message. */ -export type Delivery = (message: AssistantMessage) => void | Promise; +export type Delivery = ( + message: AssistantMessage, + text?: string, +) => void | Promise; /** Resume the agent turn after a transient or ambiguous delivery failure. */ export class RetryableDeliveryError extends Error { diff --git a/packages/junior/src/chat/experimental.ts b/packages/junior/src/chat/experimental.ts index 3096ce8145..6fa33eb09f 100644 --- a/packages/junior/src/chat/experimental.ts +++ b/packages/junior/src/chat/experimental.ts @@ -3,7 +3,11 @@ * Add new keys here as features graduate from private experiments; remove them * once they become stable defaults. */ -export const EXPERIMENTAL_FEATURES = ["passive-routing", "subagents"] as const; +export const EXPERIMENTAL_FEATURES = [ + "output-router", + "passive-routing", + "subagents", +] as const; /** One known experimental feature name. */ export type ExperimentalFeature = (typeof EXPERIMENTAL_FEATURES)[number]; diff --git a/packages/junior/src/chat/local/runner.ts b/packages/junior/src/chat/local/runner.ts index f91b23f3b4..58870e82ad 100644 --- a/packages/junior/src/chat/local/runner.ts +++ b/packages/junior/src/chat/local/runner.ts @@ -260,10 +260,12 @@ async function runLocalAgentTurnInContext( /** Print and record one completed assistant message in local conversation order. */ const deliverAssistantMessage = async ( reply: AssistantMessage | string, + visibleText?: string, ): Promise => { const message = typeof reply === "string" ? undefined : reply; const text = - typeof reply === "string" ? reply : getAssistantReplyText(reply); + visibleText ?? + (typeof reply === "string" ? reply : getAssistantReplyText(reply)); if (!text?.trim()) { return; } diff --git a/packages/junior/src/chat/providers/slack/resume.ts b/packages/junior/src/chat/providers/slack/resume.ts index c75735c65c..4ca5d25078 100644 --- a/packages/junior/src/chat/providers/slack/resume.ts +++ b/packages/junior/src/chat/providers/slack/resume.ts @@ -554,10 +554,12 @@ async function resumeSlackTurnInContext( /** Post and record one completed assistant message for the resumed turn. */ const deliverAssistantMessage = async ( reply: AssistantMessage | string, + visibleText?: string, ): Promise => { const message = typeof reply === "string" ? undefined : reply; const text = - typeof reply === "string" ? reply : getAssistantReplyText(reply); + visibleText ?? + (typeof reply === "string" ? reply : getAssistantReplyText(reply)); if (!text?.trim()) { return; } diff --git a/packages/junior/src/chat/providers/slack/turn.ts b/packages/junior/src/chat/providers/slack/turn.ts index 84602c925b..00d94ec7eb 100644 --- a/packages/junior/src/chat/providers/slack/turn.ts +++ b/packages/junior/src/chat/providers/slack/turn.ts @@ -810,11 +810,13 @@ export function createSlackTurn(deps: SlackTurnDeps) { /** Post and record one completed assistant message in the active thread. */ const deliverAssistantMessage = async ( reply: AssistantMessage | string, + visibleText?: string, terminalDispatchOutcome?: "blocked" | "failed", ): Promise => { const agentMessage = typeof reply === "string" ? undefined : reply; const text = - typeof reply === "string" ? reply : getAssistantReplyText(reply); + visibleText ?? + (typeof reply === "string" ? reply : getAssistantReplyText(reply)); if (!text?.trim()) { return; } diff --git a/packages/junior/src/chat/services/output-router.ts b/packages/junior/src/chat/services/output-router.ts new file mode 100644 index 0000000000..90ee9f1b70 --- /dev/null +++ b/packages/junior/src/chat/services/output-router.ts @@ -0,0 +1,316 @@ +import type { AssistantMessage } from "@earendil-works/pi-ai"; +import { z } from "zod"; +import { NO_REPLY_MARKER, isNoReplyMarker } from "@/chat/no-reply"; +import { + logInfo, + logWarn, + setSpanAttributes, + withSpan, + type LogContext, +} from "@/chat/logging"; +import { JUNIOR_PERSONALITY } from "@/chat/prompt"; +import { + decideReply, + sanitizeAssistantText, +} from "@/chat/services/assistant-reply"; + +/** Soft length target for visible replies. */ +export const OUTPUT_REPLY_SOFT_MAX_CHARS = 800; +/** Absolute max for a rewritten visible reply. */ +export const OUTPUT_REPLY_HARD_MAX_CHARS = 1_200; + +const OUTPUT_ROUTER_MAX_TOKENS = 1_200; +const OUTPUT_ROUTER_PROMPT_MAX_CHARS = 12_000; + +/** + * Model output is intentionally small: + * - text=null → no visible reply + * - text=string → that string is the visible reply + */ +const preparedReplySchema = z + .object({ + text: z.string().nullable(), + reason: z.string().min(1), + }) + .strict(); + +export type PreparedAssistantReply = + | { + kind: "silent"; + costUsd?: number; + reason: string; + } + | { + kind: "reply"; + costUsd?: number; + reason: string; + /** Visible reply text. May differ from the original agent text. */ + text: string; + }; + +type CompleteObject = (args: { + modelId: string; + schema: typeof preparedReplySchema; + maxTokens: number; + metadata: Record; + prompt: string; + thinkingLevel?: "low" | "medium" | "high" | "xhigh"; + system: string; + temperature: number; + promptName?: string; +}) => Promise<{ costUsd?: number; object: unknown }>; + +/** + * True when the last non-empty line is exactly the silence marker. + * That trailing line is a suppress-this-message signal. + */ +function hasTrailingNoReplyLine(text: string): boolean { + const lines = text + .replace(/\s+$/u, "") + .split("\n") + .map((line) => line.trimEnd()); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const line = lines[index]?.trim() ?? ""; + if (!line) { + continue; + } + return line === NO_REPLY_MARKER; + } + return false; +} + +/** + * Prompt shape follows current lab guidance: + * - put the task and rules first + * - keep rules short, specific, and direct + * - put the message body in the user turn, separate from instructions + * - let the JSON schema own the output shape; do not restate it at length + * - include SOUL personality so rewrites keep the bot voice + * + * Refs: OpenAI prompt engineering + structured outputs; Anthropic clear/direct. + */ +function buildSystemPrompt(personality: string = JUNIOR_PERSONALITY): string { + return [ + "Edit one assistant message into the final reply the user will see.", + "You receive only that message. No other conversation context.", + "Fields: text is the reply or null. reason is one short sentence.", + "", + "Rules:", + `- Set text to null for empty text or only ${NO_REPLY_MARKER}.`, + `- If the message answers the user, keep it as a reply even when it mentions ${NO_REPLY_MARKER}.`, + `- If the message explains how ${NO_REPLY_MARKER} or silence works, keep it as a reply and keep the marker text the user needs.`, + `- Keep short clear replies as-is (${OUTPUT_REPLY_SOFT_MAX_CHARS} characters or less).`, + `- If the reply is longer than ${OUTPUT_REPLY_SOFT_MAX_CHARS} characters, shorten it to at most ${OUTPUT_REPLY_SOFT_MAX_CHARS} characters in 1-5 short sentences. Keep the answer, key facts, links, and next steps. Do not add facts.`, + "- Do not add a preface or commentary about editing.", + "- These rules win over personality when they conflict.", + "", + "Personality", + "Match this voice and tone when you keep or rewrite text:", + personality.trim(), + ].join("\n"); +} + +function buildUserPrompt(text: string): string { + const body = + text.length <= OUTPUT_ROUTER_PROMPT_MAX_CHARS + ? text + : `${text.slice(0, OUTPUT_ROUTER_PROMPT_MAX_CHARS)}\n…[truncated]…`; + // Keep instructions in the system prompt. Put only the message body here. + return ["Message:", '"""', body, '"""'].join("\n"); +} + +function capVisibleText(text: string): string { + if (text.length <= OUTPUT_REPLY_HARD_MAX_CHARS) { + return text; + } + return `${text.slice(0, OUTPUT_REPLY_HARD_MAX_CHARS - 1).trimEnd()}…`; +} + +function silent(reason: string, costUsd?: number): PreparedAssistantReply { + return { + kind: "silent", + reason, + ...(costUsd !== undefined ? { costUsd } : undefined), + }; +} + +function reply( + text: string, + reason: string, + costUsd?: number, +): PreparedAssistantReply { + return { + kind: "reply", + text, + reason, + ...(costUsd !== undefined ? { costUsd } : undefined), + }; +} + +/** + * Cheap local checks before calling the model. + * Exact marker and trailing whole-line marker silence stay local. + * Inline marker mentions still need the model. + */ +export function prepareAssistantReplyLocal( + text: string, +): PreparedAssistantReply | null { + const trimmed = sanitizeAssistantText(text); + if (!trimmed) { + return silent("empty"); + } + if (isNoReplyMarker(trimmed)) { + return silent("no_reply"); + } + // A final line that is only the marker means suppress the whole message. + // Inline mentions of the marker still go to the model. + if (hasTrailingNoReplyLine(trimmed)) { + return silent("trailing_no_reply"); + } + return null; +} + +function finalizeModelResult( + object: unknown, + originalText: string, + costUsd?: number, +): PreparedAssistantReply { + const parsed = preparedReplySchema.parse(object); + const reason = parsed.reason.trim() || "prepared"; + + if (parsed.text === null) { + return silent(reason, costUsd); + } + + const text = sanitizeAssistantText(parsed.text); + if (!text) { + // Model returned blank text. Keep the original visible reply. + return reply(originalText, `empty_model_text:${reason}`, costUsd); + } + // Exact marker-only output is silence. Otherwise keep the model text, + // including answers that mention or explain the marker. + if (isNoReplyMarker(text)) { + return silent(`model_no_reply:${reason}`, costUsd); + } + return reply(capVisibleText(text), reason, costUsd); +} + +/** + * Prepare the visible reply for one assistant message. + * Does not change the original agent message text. + */ +export async function prepareAssistantReply(args: { + completeObject: CompleteObject; + context?: { + conversationId?: string; + runId?: string; + }; + fastModelId: string; + text: string; +}): Promise { + const originalText = sanitizeAssistantText(args.text); + const local = prepareAssistantReplyLocal(originalText); + if (local) { + return local; + } + + const logContext: LogContext = { + messageConversationId: args.context?.conversationId, + runId: args.context?.runId, + modelId: args.fastModelId, + }; + + return withSpan( + "chat.prepare_assistant_reply", + "chat.prepare_assistant_reply", + logContext, + async () => { + setSpanAttributes({ + "app.ai.output_router.input_char_count": originalText.length, + "app.ai.output_router.soft_max_chars": OUTPUT_REPLY_SOFT_MAX_CHARS, + }); + + try { + const result = await args.completeObject({ + modelId: args.fastModelId, + schema: preparedReplySchema, + maxTokens: OUTPUT_ROUTER_MAX_TOKENS, + metadata: { + modelId: args.fastModelId, + conversationId: args.context?.conversationId ?? "", + runId: args.context?.runId ?? "", + }, + prompt: buildUserPrompt(originalText), + thinkingLevel: "low", + system: buildSystemPrompt(), + temperature: 0, + promptName: "junior.prepare_assistant_reply", + }); + + const prepared = finalizeModelResult( + result.object, + originalText, + result.costUsd, + ); + setSpanAttributes({ + "app.ai.output_router.kind": prepared.kind, + "app.ai.output_router.reason": prepared.reason, + ...(prepared.kind === "reply" + ? { "app.ai.output_router.output_char_count": prepared.text.length } + : undefined), + }); + logInfo("ai.output_router.decided", { + "app.ai.output_router.kind": prepared.kind, + "app.ai.output_router.reason": prepared.reason, + "app.ai.output_router.input_char_count": originalText.length, + ...(prepared.kind === "reply" + ? { "app.ai.output_router.output_char_count": prepared.text.length } + : undefined), + }); + return prepared; + } catch (error) { + logWarn("ai.output_router.failed", { + "exception.message": + error instanceof Error ? error.message : String(error), + }); + // On failure, show the original text rather than dropping the reply. + return reply(originalText, "prepare_failed"); + } + }, + ); +} + +/** + * Decide the visible reply for a completed assistant message. + * Returns silent/skip without changing the agent message. + */ +export async function prepareAssistantMessage(args: { + completeObject: CompleteObject; + context?: { + conversationId?: string; + runId?: string; + }; + fastModelId: string; + message: AssistantMessage; +}): Promise< + | { kind: "skip" } + | { kind: "silent"; prepared: PreparedAssistantReply } + | { kind: "reply"; text: string; prepared: PreparedAssistantReply } +> { + const decision = decideReply(args.message); + if (decision.kind !== "deliver") { + return { kind: "skip" }; + } + + const prepared = await prepareAssistantReply({ + completeObject: args.completeObject, + context: args.context, + fastModelId: args.fastModelId, + text: decision.text, + }); + + if (prepared.kind === "silent") { + return { kind: "silent", prepared }; + } + return { kind: "reply", text: prepared.text, prepared }; +} diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index e72bffa49f..bd75acf796 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -353,10 +353,12 @@ export function createConversationTurnWorker( const deliverAssistantMessage = async ( value: AssistantMessage | string, + visibleText?: string, ): Promise => { const agentMessage = typeof value === "string" ? undefined : value; const replyText = - typeof value === "string" ? value : getAssistantReplyText(value); + visibleText ?? + (typeof value === "string" ? value : getAssistantReplyText(value)); if (!replyText?.trim()) { return; } diff --git a/packages/junior/tests/fixtures/experimental-setup.ts b/packages/junior/tests/fixtures/experimental-setup.ts index 5053d75cb3..adf1ddb4c5 100644 --- a/packages/junior/tests/fixtures/experimental-setup.ts +++ b/packages/junior/tests/fixtures/experimental-setup.ts @@ -4,8 +4,13 @@ import { setExperimentalFeatures } from "@/chat/experimental"; /** * Production leaves experimental features off. The suite opts in so coverage * exercises the real wiring path without an env flag. + * + * Isolated output-router evals call prepareAssistantReply directly and do not + * use this file. Delivery wiring stays covered by full-runtime suites with + * output-router left off unless a case opts in explicitly. */ export const SUITE_EXPERIMENTAL = { + "output-router": false, "passive-routing": true, subagents: true, } as const; diff --git a/packages/junior/tests/unit/experimental.test.ts b/packages/junior/tests/unit/experimental.test.ts index d79b7b758e..95fb9e3011 100644 --- a/packages/junior/tests/unit/experimental.test.ts +++ b/packages/junior/tests/unit/experimental.test.ts @@ -11,18 +11,29 @@ afterEach(() => { describe("experimental features", () => { it("defaults experimental features off", () => { setExperimentalFeatures(undefined); + expect(isExperimentalFeatureEnabled("output-router")).toBe(false); expect(isExperimentalFeatureEnabled("passive-routing")).toBe(false); expect(isExperimentalFeatureEnabled("subagents")).toBe(false); }); it("enables features from createApp-style config", () => { - setExperimentalFeatures({ "passive-routing": true, subagents: true }); + setExperimentalFeatures({ + "output-router": true, + "passive-routing": true, + subagents: true, + }); + expect(isExperimentalFeatureEnabled("output-router")).toBe(true); expect(isExperimentalFeatureEnabled("passive-routing")).toBe(true); expect(isExperimentalFeatureEnabled("subagents")).toBe(true); }); it("treats explicit false as disabled", () => { - setExperimentalFeatures({ "passive-routing": false, subagents: false }); + setExperimentalFeatures({ + "output-router": false, + "passive-routing": false, + subagents: false, + }); + expect(isExperimentalFeatureEnabled("output-router")).toBe(false); expect(isExperimentalFeatureEnabled("passive-routing")).toBe(false); expect(isExperimentalFeatureEnabled("subagents")).toBe(false); }); diff --git a/packages/junior/tests/unit/services/output-router.test.ts b/packages/junior/tests/unit/services/output-router.test.ts new file mode 100644 index 0000000000..16a143b7ee --- /dev/null +++ b/packages/junior/tests/unit/services/output-router.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it, vi } from "vitest"; +import type { AssistantMessage } from "@earendil-works/pi-ai"; +import { NO_REPLY_MARKER } from "@/chat/no-reply"; +import { + OUTPUT_REPLY_HARD_MAX_CHARS, + prepareAssistantMessage, + prepareAssistantReply, + prepareAssistantReplyLocal, +} from "@/chat/services/output-router"; + +function assistant(text: string, withToolCall = false): AssistantMessage { + return { + role: "assistant", + content: [ + { type: "text", text }, + ...(withToolCall + ? [ + { + type: "toolCall" as const, + id: "call-1", + name: "bash", + arguments: {}, + }, + ] + : []), + ], + api: "responses", + provider: "openai", + model: "test-model", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 1, + }; +} + +describe("prepare assistant reply", () => { + it("handles empty, exact, and trailing silence markers locally", () => { + expect(prepareAssistantReplyLocal("")).toEqual({ + kind: "silent", + reason: "empty", + }); + expect(prepareAssistantReplyLocal(NO_REPLY_MARKER)).toEqual({ + kind: "silent", + reason: "no_reply", + }); + // A final whole-line marker suppresses the whole message. + expect( + prepareAssistantReplyLocal( + [`status only note`, "", NO_REPLY_MARKER].join("\n"), + ), + ).toEqual({ + kind: "silent", + reason: "trailing_no_reply", + }); + // Inline marker mentions still need the model. + expect( + prepareAssistantReplyLocal(`shipped it ${NO_REPLY_MARKER}\nmore detail`), + ).toBeNull(); + }); + + it("skips the model for exact silence markers", async () => { + const completeObject = vi.fn(); + await expect( + prepareAssistantReply({ + completeObject, + fastModelId: "openai/gpt-5.6-luna", + text: NO_REPLY_MARKER, + }), + ).resolves.toEqual({ + kind: "silent", + reason: "no_reply", + }); + expect(completeObject).not.toHaveBeenCalled(); + }); + + it("keeps the original text when the model call fails", async () => { + const completeObject = vi.fn(async () => { + throw new Error("boom"); + }); + + await expect( + prepareAssistantReply({ + completeObject, + fastModelId: "openai/gpt-5.6-luna", + text: "Keep this answer.", + }), + ).resolves.toEqual({ + kind: "reply", + text: "Keep this answer.", + reason: "prepare_failed", + }); + }); + + it("caps oversized model text", async () => { + const completeObject = vi.fn(async () => ({ + object: { + text: "B".repeat(OUTPUT_REPLY_HARD_MAX_CHARS + 50), + reason: "still long", + }, + })); + + const prepared = await prepareAssistantReply({ + completeObject, + fastModelId: "openai/gpt-5.6-luna", + text: "A".repeat(900), + }); + + expect(prepared.kind).toBe("reply"); + if (prepared.kind !== "reply") return; + expect(prepared.text.length).toBe(OUTPUT_REPLY_HARD_MAX_CHARS); + expect(prepared.text.endsWith("…")).toBe(true); + }); + + it("returns visible text without changing the agent message", async () => { + const original = "A".repeat(900); + const message = assistant(original); + const completeObject = vi.fn(async () => ({ + object: { + text: "Condensed reply.", + reason: "too long", + }, + })); + + const prepared = await prepareAssistantMessage({ + completeObject, + fastModelId: "openai/gpt-5.6-luna", + message, + }); + + expect(prepared).toMatchObject({ + kind: "reply", + text: "Condensed reply.", + }); + expect(message.content).toEqual([{ type: "text", text: original }]); + }); + + it("skips tool-bearing assistant messages", async () => { + const completeObject = vi.fn(); + await expect( + prepareAssistantMessage({ + completeObject, + fastModelId: "openai/gpt-5.6-luna", + message: assistant("working", true), + }), + ).resolves.toEqual({ kind: "skip" }); + expect(completeObject).not.toHaveBeenCalled(); + }); +}); diff --git a/policies/evals.md b/policies/evals.md index 8d281d3d3e..f0c02a7cab 100644 --- a/policies/evals.md +++ b/policies/evals.md @@ -8,11 +8,13 @@ Suite policy: - **Integration** (`evals/integration/**`): full-runtime integration coverage that must never regress. Failures are hard pass/fail. -- **Behavioral** (domain folders under `evals/` except `integration/` and - `guardian/`): agent behavior with bounded variability. CI gates on the - aggregate suite floor, not a single weak case. -- **Guardian** (`evals/guardian/**`): isolated decision snapshots with exact - `allow` / `ask` / `deny` assertions. Failures are hard pass/fail. +- **Behavioral** (domain folders under `evals/` except `integration/`, + `guardian/`, and `output-router/`): agent behavior with bounded variability. + CI gates on the aggregate suite floor, not a single weak case. +- **Guardian** (`evals/guardian/**`): isolated action-review snapshots with + exact `allow` / `ask` / `deny` assertions. Failures are hard pass/fail. +- **Prepare reply** (`evals/output-router/**`): isolated prepare checks over + one assistant message (`silent` / `reply`). Failures are hard pass/fail. ## Policy @@ -20,6 +22,8 @@ Suite policy: - Assert behavior rules, not incidental wording or execution sequence. - Put never-break full-runtime integration coverage under `evals/integration/**`. Put agent-behavior measurement under behavioral domain folders. + Put isolated action-review snapshots under `evals/guardian/**`. + Put isolated prepare-reply cases under `evals/output-router/**`. - Do not patch product prompts with eval-shaped examples, fixture names, exact user messages, expected answers, or distinctive scenario phrases from eval files.