From 4714e8d904f6c15fbb07893e5ca77b8e7e57a735 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 12 Sep 2026 11:42:28 +0200 Subject: [PATCH 1/4] feat(agents): evaluate subscription routing policy explicitly --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- .../.claude-plugin/plugin.json | 2 +- plugins/agentic-engineering/README.md | 5 + .../agents/agent-improver.agent.md | 9 ++ .../agents/agentic-engineer.agent.md | 10 ++ plugins/agentic-engineering/plugin.json | 2 +- .../resources/inference-routing.md | 115 ++++++++++++++++++ .../provider-neutral.desired-state.json | 16 ++- .../classify-default-branch-ci-runs.test.sh | 10 +- .../scripts/evaluate-inference-routing.sh | 107 ++++++++++++++++ .../evaluate-inference-routing.test.sh | 91 ++++++++++++++ 12 files changed, 361 insertions(+), 10 deletions(-) create mode 100644 plugins/agentic-engineering/resources/inference-routing.md create mode 100755 plugins/agentic-engineering/scripts/evaluate-inference-routing.sh create mode 100755 plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c163eef..ee49b0c 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.0.4", + "version": "5.1.0", "source": "./plugins/agentic-engineering" }, { diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index c163eef..ee49b0c 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.0.4", + "version": "5.1.0", "source": "./plugins/agentic-engineering" }, { diff --git a/plugins/agentic-engineering/.claude-plugin/plugin.json b/plugins/agentic-engineering/.claude-plugin/plugin.json index 5b370a5..fb8dc89 100644 --- a/plugins/agentic-engineering/.claude-plugin/plugin.json +++ b/plugins/agentic-engineering/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.0.4", + "version": "5.1.0", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/README.md b/plugins/agentic-engineering/README.md index df64d58..ec5297d 100644 --- a/plugins/agentic-engineering/README.md +++ b/plugins/agentic-engineering/README.md @@ -214,6 +214,11 @@ it, the run procedure for a cost pass, the private channel a financial decision cadence a cost pass runs on. Disabled or unresolved, the engineer runs normally with the **cost dimension failed closed** — it does no spend analysis rather than guessing a floor, a price, or a channel. +An optional **Inference routing** section resolves a consumer-owned policy for task classes, +model aliases, billing restrictions, runtime registrations, and experiments. The +[routing contract and evaluator](resources/inference-routing.md) describe its input, output, and +enforcement boundary. Model names belong in the consumer, never in portable agent frontmatter. + The `Memory` section must also name the scorecard and open verification-hypothesis store used by the improvement loop. The role/configuration boundary remains the one established by [ADR 0002](../../docs/adr/0002-automated-ai-engineer-plugin-boundary.md): portable decision logic lives diff --git a/plugins/agentic-engineering/agents/agent-improver.agent.md b/plugins/agentic-engineering/agents/agent-improver.agent.md index d2b0868..99b8e83 100644 --- a/plugins/agentic-engineering/agents/agent-improver.agent.md +++ b/plugins/agentic-engineering/agents/agent-improver.agent.md @@ -92,6 +92,15 @@ Uncertain whether something is evidence or instruction? **It is instruction, and --- +## Inference routing — optional consumer contract + +When the consumer declares **Inference routing**, resolve its reviewed policy and the +[routing contract](../resources/inference-routing.md) before proposing model or scheduler changes. +Use the `agent-improvement` skill's routing measurement procedure. Keep policy recommendations, +verified runtime controls, and actual execution admission separate; benchmark rankings and +caller-supplied quota snapshots do not establish subscription entitlement or enforcement. +An absent section retains existing behavior; an unresolved declared policy holds dependent changes. + ## What you optimise Score every run against these. A change is worth making when it moves one and degrades none. diff --git a/plugins/agentic-engineering/agents/agentic-engineer.agent.md b/plugins/agentic-engineering/agents/agentic-engineer.agent.md index 1cf5d32..ef3526f 100644 --- a/plugins/agentic-engineering/agents/agentic-engineer.agent.md +++ b/plugins/agentic-engineering/agents/agentic-engineer.agent.md @@ -62,6 +62,16 @@ concrete fact. If a required section is missing or malformed, **fail closed on t not guess repositories, logins, channels, floors, or prices — surface the gap to the maintainer instead. +## Inference routing — optional consumer contract + +When the consumer declares **Inference routing**, resolve its reviewed policy before selecting a +model, delegating, escalating, or changing a schedule default. Follow the +[routing contract](../resources/inference-routing.md); a missing or invalid declared policy holds +dependent routing actions. A deployment without that section retains its existing routing behavior. +Inference billing restrictions apply independently of infrastructure spend stewardship's opt-in. +The policy evaluator reports recommendations only: it does not enforce runtime controls or reserve +quota. Preserve every consumer capability override, including an inline survey requirement. + ## How you operate 1. **Follow the run loop.** The bundled **`portfolio-maintenance`** skill is your procedure: diff --git a/plugins/agentic-engineering/plugin.json b/plugins/agentic-engineering/plugin.json index 5b370a5..fb8dc89 100644 --- a/plugins/agentic-engineering/plugin.json +++ b/plugins/agentic-engineering/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.0.4", + "version": "5.1.0", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/resources/inference-routing.md b/plugins/agentic-engineering/resources/inference-routing.md new file mode 100644 index 0000000..f9d10d2 --- /dev/null +++ b/plugins/agentic-engineering/resources/inference-routing.md @@ -0,0 +1,115 @@ +# Inference routing contract + +This optional contract separates task policy from runtime enforcement. The consumer's **Inference +routing** section names one reviewed policy file, its revision, runtime verification records, and +the private telemetry store. Without that section, existing model selection continues. If a +declared policy cannot resolve, hold dependent routing changes and report the gap; continue +independent authorized work. Infrastructure spend opt-in remains a separate contract. + +## Evaluation boundary + +`scripts/evaluate-inference-routing.sh` reads exactly one JSON object from stdin and writes a JSON +decision. It uses Bash and jq, makes no network request, reads no credentials, launches no model, +takes no lock, and writes no file. Run it from the reviewed plugin revision whose desired state +pins its digest. `--now ` supplies deterministic evaluation time for tests; production +uses the host clock. + +| Exit | Decision | Meaning | +|---|---|---| +| 0 | `RECOMMEND` | Policy and supplied observations permit the recommended route | +| 1 | `HOLD` | The request is valid but one or more policy/evidence conditions are unmet | +| 2 | `INVALID` | Malformed, unsupported, or prohibited configuration/input | + +**Every result has `executionAdmitted: false`. No exit code is a launch authorization.** The +`reportedControls` field describes caller-supplied data, not verified enforcement. Do not wire an +exit-zero shell condition directly to an inference launcher. An assistant calling this helper has +already incurred its own initial inference and instruction load. + +The JSON root requires `policy`, `task`, and `snapshot` only. Unknown fields, missing fields, extra +JSON documents, and invalid types fail validation. JSON producers must emit unique object keys; +jq parses repeated keys using its normal last-value semantics. The helper is a policy aid, not a +hostile-input authentication boundary. + +## Policy version 1 + +| Field | Contract | +|---|---| +| `version`, `revision`, `enabled` | `1`, immutable review identifier, and explicit boolean opt-in | +| `billingMode`, `paidFallback` | Exactly `subscription-only` and `false` | +| `deniedModelTerms` | Nonempty distinct case-insensitive literal substrings; every route is checked, including unused routes | +| `routes` | Exactly `support`, `workhorse`, `diagnosis`, `deepRefactor`; each has an exact `model`, registered `runtime`, and `effort` (`low`, `medium`, `high`, `xhigh`) | +| `runtimes` | Map of runtime registration IDs to `enabled`, `role` (`owner`, `builder`, `observer`), and `expiresAt` in Unix seconds | +| `limits` | `maxDepth` and `maxChildren` (0 or 1 initially), positive integer `repairHypotheses`, `activeMinutes`, `snapshotMaxAgeSeconds` (1–300), and `reservePercent` with positive `short` and `weekly` percentages | + +Deployment model names and runtime IDs never enter portable agent frontmatter. An exact string +in a policy is an intended model, not proof that the runtime supports it. Denial terms cover visible +IDs only; opaque aliases/defaults/substitutions need native verification before execution. + +## Task and observation input + +The task supplies `class`, nonnegative integer `depth`, `children`, `distinctFailedHypotheses`, and +`activeMinutes`; `failureKind` (`none`, `reasoning`, `environment`, `quota`, `authority`); and boolean +`contractClear`, `checksDefined`, `reversible`, `sensitiveInvariants`, `writeRequired`. +`depth` is the proposed target depth (the owner is zero); `children` is the total active child count +after the proposed dispatch. They are prospective counts, not the counts before adding a child. + +Support and workhorse tasks move to diagnosis when the stated contract/check/reversibility criteria +fail, sensitive invariants remain, or either repair/active-time threshold is reached. Active time +excludes waits. Count distinct tested hypotheses, not repeated commands. Environment, quota, and +authority failures hold the request; they never justify buying more reasoning. An unclear contract +also holds writes, as do missing checks and irreversible scope; diagnosis may investigate these +read-only but cannot manufacture missing authority. +`deepRefactor` is +an explicit task classification supported by a demonstrated runtime advantage, not the next +automatic rung after diagnosis. The owner must also check task scope and previous handoffs. + +The snapshot requires: + +- `observedAt` (Unix seconds), `runtime`, `runtimeVersion`, intended exact `model` or `null`; +- `billing` (`included`, `unknown`, `paygo`), `controls` (`verified`, `unverified`), `evidenceRef` + (a private verification record reference or `null`); +- `buckets.short` and `buckets.weekly`, each `null` or an object with `remainingPercent`, + `estimatedChainPercent`, `reservedPercent`, and `unsettledPercent`. Each value is a percentage + in 0–100 or `null`. Unknown is never zero. + +Snapshots must refer to the selected runtime and exact model. Future timestamps, expired runtime +registrations, stale observations, observer writes, and excess delegation produce `HOLD`. Quota +headroom must cover the estimated whole attempt chain, existing reservations, unsettled debits, +and the reserve in **each** bucket. This is arithmetic over reported values, not a reservation. +Provider/model-specific extra buckets require a verified adapter extension before that route can +run; never discard a bucket to fit this two-window version. Percentage units are local to each +provider bucket and cannot be added across accounts or compared as inference prices. + +See the adjacent hermetic test for complete synthetic requests and independently expected results. + +## Runtime controls before unattended activation + +Maintain an expiring verification record for each concrete deployed surface and version. Verify +the included billing path, disabled paid fallback/overages, exact model resolution, inherited tool +permissions, workspace isolation, and account admission. The trusted launcher/native control must +apply before inference; a prompt, model preference, or caller-supplied `verified` flag is insufficient. +Exercise launch, resume, child overrides, advisors, defaults, and fallback. Intercept forbidden +requests before inference in negative tests; never invoke a prohibited model as a test. If the +runtime cannot provide the needed control, leave its affected unattended route disabled. + +Use the native subscription harness; never add an inference broker, API credential, purchased +credits, or pay-as-you-go fallback. Native schemas differ: do not copy generic tool names into a +provider manifest and assume enforcement. A deployment's explicit inline survey override takes +precedence over generic delegation advice. + +Start with serialized scheduled admission per account and one bounded child at depth one. The +reservation covers parent, children, integration, retries, and handoff. Serialize the check and +reservation in a trusted external admission mechanism before enabling fan-out. A worktree lock +does not reserve model quota. Expiring a writer lease does not stop an old session; retain its +unsettled debit until consumption is reconciled or termination is verified. Resume with the same +reservation identity. Include interactive and other-device usage in coverage; missing coverage +prevents an exact per-task attribution claim. + +## Ownership and measurement + +The engineering skill owns bounded task packets and one delivery owner. A helper performing lint, +tests, or a commit is a procedure/tool call, not an automatic new agent. The improvement skill owns +complete attempt-chain accounting, context hydration, matched task cohorts, quality floors, +canary promotion, and rollback. Consumer governance supplies experimental thresholds and its single +policy publisher; overlapping runs of that publisher still require fencing. Runtime validation and +longitudinal outcomes are separate acceptance gates from passing these offline tests. diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index ab3e175..88b9f5e 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -25,9 +25,14 @@ "path": "scripts/surveyor-forge-readonly.sh", "sha256": "ce5a3b71d5ae28b0787651aa191b9c7df510be1db7ab81211a0e0fb1e66ca2a2", "executable": true + }, + { + "path": "scripts/evaluate-inference-routing.sh", + "sha256": "d8a239f7e75d2e0d1ec457f4bbc280c7918a0f5d3db7a189d99ae2491fd83fa4", + "executable": true } ], - "entrypointSha256": "a3d01aa54b2af445aa4a6d07eb406d401824be7db08d5ec4c2f914db7fc9bf39", + "entrypointSha256": "f9d9599f5d85cab11be7b09a9a26c4b515460f258bb38aeac1094133260fbf5f", "updatePolicy": "latest-reviewed-default-branch", "providerPolicy": "neutral", "refreshTiming": "before-starting-each-run", @@ -66,7 +71,7 @@ "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", "mode": "separate-schedule-or-on-demand", - "definitionSha256": "4e5d1dac3307ac7151d40dababae48b1da7a68f927a11026f1f5f87424b336aa", + "definitionSha256": "39e0e70bcdffd20f796160bf5136709aa64a9a567c25e829d4ac430ebf7c7cbe", "skillSha256": "93b0c9551ce4d5360cb8a398af91bbafc85468a9cfff99bb78b506bfcd27287f" } }, @@ -97,9 +102,9 @@ "approvalMode": "no-unattended-step-may-depend-on-an-interactive-approval" }, "model": { - "selectionPolicy": "best-available-agentic-coding-model", - "upgradePolicy": "follow-the-runtime-default-unless-the-consumer-contract-pins-a-reviewed-model", - "reasoningPolicy": "highest-practical-effort-within-the-consumer-cadence-budget" + "selectionPolicy": "Resolve the optional AGENTS.md Inference routing contract first; otherwise retain best-available-agentic-coding-model.", + "upgradePolicy": "When Inference routing is declared, change aliases only through its reviewed governance; otherwise follow the runtime default unless the consumer pins a reviewed model.", + "reasoningPolicy": "Use the reviewed task-class effort when Inference routing is declared; otherwise highest practical effort within the consumer cadence budget." }, "memory": { "backendPolicy": "provider-native-preferred", @@ -116,6 +121,7 @@ "Read AGENTS.md and verify every required consumer contract section. Enable agent-improver only when both additional sections are present. Preserve spec.roles[\"agentic-engineer\"].spendStewardshipEnabled from the single effective desired-state document declared in Spend contract, or the shipped false default when none is declared. Apply the engineer entrypoint's explicit opt-in contract before spend work; onboarding never infers or grants maintainer opt-in. Report the effective source, flag value, and unresolved prerequisites while continuing ordinary operate and advance engineering.", "Map the declared roles onto native agent capabilities, preserve portfolio-surveyor as read-only, and grant least privilege for each role. Export a disabling GH_TELEMETRY (0 or false) in the environment the surveyor's shell inherits: the read-only forge guard treats a missing value as unproven and refuses every command, including reads, so a runtime that enforces the guard without this variable leaves the surveyor unable to run any forge query at all. It cannot be supplied inside the command string, which the guard also refuses.", "Allocate a unique branch namespace for every deployed writer instance and record it in the consumer contract before enabling writes.", + "If AGENTS.md declares Inference routing, resolve its reviewed policy and scripts/evaluate-inference-routing.sh before configuring models or delegation. Read resources/inference-routing.md. A policy recommendation does not authenticate reported controls, admit execution, or reserve quota; verify native billing, model, tool and admission controls before enabling the affected unattended route. Preserve consumer capability overrides.", "Create or update a native schedule only for each entry in runtime.scheduler.schedules whose corresponding role's enabled or enabledWhen condition is satisfied, using AGENTS.md#Cadence and that entry's thin bootstrap prompt.", "Load native persistent memory and perform a read-only preflight or dry run before allowing unattended writes.", "Report the installed source revision, plugin version, cadence, timezone, branch namespace, memory backend, permission boundary, and every unsupported capability." diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh index 1f58d81..4745fd3 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh @@ -206,11 +206,14 @@ hash_file() { classifier_sha=$(hash_file "$CLASSIFIER") guard_sha=$(hash_file "$HERE/forge-readonly-guard.sh") wrapper_sha=$(hash_file "$HERE/surveyor-forge-readonly.sh") +routing_sha=$(hash_file "$HERE/evaluate-inference-routing.sh") if grep -Fq 'referenced runtime assets' "$DESIRED_STATE" && jq -e \ --arg classifier_sha "$classifier_sha" \ --arg guard_sha "$guard_sha" \ - --arg wrapper_sha "$wrapper_sha" ' + --arg wrapper_sha "$wrapper_sha" \ + --arg routing_sha "$routing_sha" ' + # The routing helper is independent of the surveyor, but shares the runtime asset pin set. .spec.source.requiredRuntimeAssets == [ { path: "scripts/classify-default-branch-ci-runs.sh", @@ -226,6 +229,11 @@ if grep -Fq 'referenced runtime assets' "$DESIRED_STATE" && path: "scripts/surveyor-forge-readonly.sh", sha256: $wrapper_sha, executable: true + }, + { + path: "scripts/evaluate-inference-routing.sh", + sha256: $routing_sha, + executable: true } ] ' "$DESIRED_STATE" > /dev/null; then diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh new file mode 100755 index 0000000..efa49fb --- /dev/null +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Pure, offline policy evaluation. Reads one {policy,task,snapshot} JSON object. +# Exit 0 RECOMMEND, 1 HOLD, 2 INVALID. NO exit status authorizes a launch. +# Observations are caller reports, not authenticated runtime or billing evidence. +# This command never launches inference, takes a lock, or reserves allowance. +set -euo pipefail +invalid() { + printf '%s\n' '{"decision":"INVALID","executionAdmitted":false,"reasons":["INVALID_INPUT"]}' + exit 2 +} +if [[ $# == 0 ]]; then + now="$(date +%s)" +elif [[ $# == 2 && "$1" == --now && "$2" =~ ^[0-9]{1,12}$ ]]; then + now="$2" +else + invalid +fi +command -v jq > /dev/null || invalid +result=$(jq -sce --argjson now "$now" ' + def exact($allowed): type == "object" and (keys | sort) == ($allowed | sort); + def text: type == "string" and length > 0 and length <= 256; + def identifier: text and test("^[A-Za-z0-9][A-Za-z0-9._/-]*$"); + def int: type == "number" and floor == . and . >= 0 and . <= 1000000000000; + def pct: type == "number" and . >= 0 and . <= 100; + def nullable_pct: . == null or pct; + def boolean: type == "boolean"; + def bucket: + . == null or + (exact(["remainingPercent","estimatedChainPercent","reservedPercent","unsettledPercent"]) + and all(.[]; nullable_pct)); + def valid: + exact(["policy","task","snapshot"]) + and (.policy | + exact(["version","revision","enabled","billingMode","paidFallback","deniedModelTerms","routes","runtimes","limits"]) + and .version == 1 and (.revision | identifier) and (.enabled | boolean) + and .billingMode == "subscription-only" and .paidFallback == false + and (.deniedModelTerms | type == "array" and length > 0 + and all(.[]; identifier) and (map(ascii_downcase) | unique | length) == length) + and (.routes | exact(["support","workhorse","diagnosis","deepRefactor"]) + and all(.[]; exact(["model","runtime","effort"]) + and (.model | identifier and (ascii_downcase | IN("default","inherit","latest","auto") | not)) + and (.runtime | identifier) + and (.effort | IN("low","medium","high","xhigh")))) + and (.runtimes | type == "object" and length > 0 + and all(keys[]; identifier) + and all(.[]; exact(["enabled","role","expiresAt"]) + and (.enabled | boolean) and (.role | IN("owner","builder","observer")) + and (.expiresAt | int))) + and (.limits | exact(["maxDepth","maxChildren","repairHypotheses","activeMinutes","snapshotMaxAgeSeconds","reservePercent"]) + and (.maxDepth | int and . <= 1) and (.maxChildren | int and . <= 1) + and (.repairHypotheses | int and . >= 1) and (.activeMinutes | int and . >= 1) + and (.snapshotMaxAgeSeconds | int and . >= 1 and . <= 300) + and (.reservePercent | exact(["short","weekly"]) and all(.[]; pct and . > 0))) + and (. as $p | all(.routes[]; . as $route | + ($p.runtimes | has($route.runtime)) + and all($p.deniedModelTerms[]; . as $term | + $route.model | ascii_downcase | contains($term | ascii_downcase) | not)))) + and (.task | + exact(["class","depth","children","distinctFailedHypotheses","activeMinutes","failureKind","contractClear","checksDefined","reversible","sensitiveInvariants","writeRequired"]) + and (.class | IN("support","workhorse","diagnosis","deepRefactor")) + and all(.depth,.children,.distinctFailedHypotheses,.activeMinutes; int) + and (.failureKind | IN("none","reasoning","environment","quota","authority")) + and all(.contractClear,.checksDefined,.reversible,.sensitiveInvariants,.writeRequired; boolean)) + and (.snapshot | + exact(["observedAt","runtime","runtimeVersion","model","billing","controls","evidenceRef","buckets"]) + and (.observedAt | int) and (.runtime | identifier) and (.runtimeVersion | text) + and (.model | . == null or identifier) + and (.billing | IN("included","unknown","paygo")) + and (.controls | IN("verified","unverified")) + and (.evidenceRef | . == null or text) + and (.buckets | exact(["short","weekly"]) and all(.[]; bucket))); + if length != 1 or (.[0] | valid | not) then error("invalid request") else .[0] end + | .policy as $p | .task as $t | .snapshot as $s + | (if ($t.class | IN("workhorse","support")) and + (($t.contractClear and $t.checksDefined and $t.reversible and ($t.sensitiveInvariants | not) | not) + or $t.distinctFailedHypotheses >= $p.limits.repairHypotheses + or $t.activeMinutes >= $p.limits.activeMinutes) + then "diagnosis" else $t.class end) as $class + | $p.routes[$class] as $route + | $p.runtimes[$route.runtime] as $runtime + | [ + if $p.enabled then empty else "POLICY_DISABLED" end, + if $t.failureKind | IN("environment","quota","authority") then "NON_REASONING_FAILURE" else empty end, + if $t.writeRequired and ($t.contractClear | not) then "CONTRACT_UNCLEAR" else empty end, + if $t.writeRequired and ($t.checksDefined | not) then "CHECKS_UNDEFINED" else empty end, + if $t.writeRequired and ($t.reversible | not) then "IRREVERSIBLE_WRITE" else empty end, + if $runtime.enabled then empty else "RUNTIME_DISABLED" end, + if $runtime.expiresAt <= $now then "RUNTIME_EXPIRED" else empty end, + if $runtime.role == "observer" and $t.writeRequired then "OBSERVER_WRITE" else empty end, + if $t.depth > $p.limits.maxDepth or $t.children > $p.limits.maxChildren then "DELEGATION_LIMIT" else empty end, + if $s.runtime != $route.runtime then "RUNTIME_MISMATCH" else empty end, + if $s.model != $route.model then "MODEL_MISMATCH" else empty end, + if $s.observedAt > $now or ($now - $s.observedAt) > $p.limits.snapshotMaxAgeSeconds then "SNAPSHOT_STALE" else empty end, + if $s.billing != "included" then "BILLING_UNPROVEN" else empty end, + if $s.controls != "verified" or $s.evidenceRef == null then "CONTROLS_UNVERIFIED" else empty end, + (["short","weekly"][] as $name | $s.buckets[$name] as $b | + if $b == null or any($b[]; . == null) then "QUOTA_UNKNOWN" + elif ($b.estimatedChainPercent + $b.reservedPercent + $b.unsettledPercent + $p.limits.reservePercent[$name]) > $b.remainingPercent + then "QUOTA_RESERVE" else empty end) + ] | unique as $reasons + | {decision: (if $reasons | length == 0 then "RECOMMEND" else "HOLD" end), + policyRevision: $p.revision, taskClass: $class, route: $route, + reportedControls: $s.controls, executionAdmitted: false, reasons: $reasons, + boundary: "Caller reports are unauthenticated. No inference launch, billing enforcement, or quota reservation occurred."} +' 2>/dev/null) || invalid +printf '%s\n' "$result" +[[ "$(jq -r '.decision' <<< "$result")" == RECOMMEND ]] || exit 1 diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh new file mode 100755 index 0000000..56c1220 --- /dev/null +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Real evaluator behavior; all observations are synthetic, no model is launched. +set -euo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +cat > "$TMP/request.json" <<'JSON' +{ + "policy": { + "version": 1, "revision": "test-1", "enabled": true, + "billingMode": "subscription-only", "paidFallback": false, + "deniedModelTerms": ["forbidden-family"], + "routes": { + "support": {"model": "small-v1", "runtime": "local", "effort": "low"}, + "workhorse": {"model": "standard-v1", "runtime": "local", "effort": "medium"}, + "diagnosis": {"model": "reasoner-v1", "runtime": "local", "effort": "high"}, + "deepRefactor": {"model": "native-v1", "runtime": "native", "effort": "high"} + }, + "runtimes": { + "local": {"enabled": true, "role": "owner", "expiresAt": 2000003600}, + "native": {"enabled": true, "role": "builder", "expiresAt": 2000003600} + }, + "limits": {"maxDepth": 1, "maxChildren": 1, "repairHypotheses": 2, + "activeMinutes": 20, "snapshotMaxAgeSeconds": 300, + "reservePercent": {"short": 20, "weekly": 15}} + }, + "task": {"class": "workhorse", "depth": 0, "children": 0, + "distinctFailedHypotheses": 0, "activeMinutes": 0, "failureKind": "none", + "contractClear": true, "checksDefined": true, "reversible": true, + "sensitiveInvariants": false, "writeRequired": true}, + "snapshot": {"observedAt": 2000000000, "runtime": "local", "runtimeVersion": "test", + "model": "standard-v1", "billing": "included", "controls": "verified", + "evidenceRef": "test-fixture-only", + "buckets": { + "short": {"remainingPercent": 70, "estimatedChainPercent": 5, "reservedPercent": 10, "unsettledPercent": 5}, + "weekly": {"remainingPercent": 80, "estimatedChainPercent": 2, "reservedPercent": 3, "unsettledPercent": 1} + }} +} +JSON +run_case() { + local name="$1" change="$2" expected_exit="$3" assertion="$4" status=0 + jq "$change" "$TMP/request.json" > "$TMP/input.json" + bash "$HERE/evaluate-inference-routing.sh" --now 2000000000 < "$TMP/input.json" > "$TMP/output.json" || status=$? + if [[ "$status" != "$expected_exit" ]] || ! jq -e "$assertion" "$TMP/output.json" > /dev/null; then + printf 'FAIL %s: exit=%s expected=%s\n' "$name" "$status" "$expected_exit" >&2 + cat "$TMP/output.json" >&2 + exit 1 + fi + printf 'PASS %s\n' "$name" +} +run_case nominal '.' 0 '.decision == "RECOMMEND" and .route.model == "standard-v1" and .executionAdmitted == false' +run_case escalation '.task.distinctFailedHypotheses=2 | .snapshot.model="reasoner-v1"' 0 '.taskClass == "diagnosis"' +run_case active-time '.task.activeMinutes=20 | .snapshot.model="reasoner-v1"' 0 '.taskClass == "diagnosis"' +run_case hard-task '.task.sensitiveInvariants=true | .snapshot.model="reasoner-v1"' 0 '.taskClass == "diagnosis"' +run_case unclear-write '.task.contractClear=false | .snapshot.model="reasoner-v1"' 1 '.reasons | index("CONTRACT_UNCLEAR") != null' +run_case unclear-investigation '.task.contractClear=false | .task.writeRequired=false | .snapshot.model="reasoner-v1"' 0 '.taskClass == "diagnosis"' +run_case missing-checks '.task.checksDefined=false | .snapshot.model="reasoner-v1"' 1 '.reasons | index("CHECKS_UNDEFINED") != null' +run_case irreversible-write '.task.reversible=false | .snapshot.model="reasoner-v1"' 1 '.reasons | index("IRREVERSIBLE_WRITE") != null' +run_case environment '.task.failureKind="environment"' 1 '.reasons | index("NON_REASONING_FAILURE") != null' +run_case policy-off '.policy.enabled=false' 1 '.reasons | index("POLICY_DISABLED") != null' +run_case no-model-override '.snapshot.model="other-v2"' 1 '.reasons | index("MODEL_MISMATCH") != null' +run_case no-paid-fallback '.policy.paidFallback=true' 2 '.decision == "INVALID"' +run_case denied-case '.policy.routes.workhorse.model="FORBIDDEN-FAMILY-v2"' 2 '.decision == "INVALID"' +run_case denied-unused-route '.policy.routes.support.model="forbidden-family-v1"' 2 '.decision == "INVALID"' +run_case opaque-default '.policy.routes.workhorse.model="default" | .snapshot.model="default"' 2 '.decision == "INVALID"' +run_case duplicated-denial '.policy.deniedModelTerms=["prohibited","PROHIBITED"]' 2 '.decision == "INVALID"' +run_case missing-policy '.policy=null' 2 '.decision == "INVALID"' +run_case malformed-class '.task.class="latest"' 2 '.decision == "INVALID"' +run_case unknown-field '.snapshot.admitted=true' 2 '.decision == "INVALID"' +run_case unknown-quota '.snapshot.buckets.short=null' 1 '.reasons | index("QUOTA_UNKNOWN") != null' +run_case unsettled-debit '.snapshot.buckets.short.unsettledPercent=40' 1 '.reasons | index("QUOTA_RESERVE") != null' +run_case weekly-floor '.snapshot.buckets.weekly.remainingPercent=20' 1 '.reasons | index("QUOTA_RESERVE") != null' +run_case exact-floor '.snapshot.buckets.weekly.remainingPercent=21' 0 '.decision == "RECOMMEND"' +run_case unknown-estimate '.snapshot.buckets.short.estimatedChainPercent=null' 1 '.reasons | index("QUOTA_UNKNOWN") != null' +run_case stale '.snapshot.observedAt=1999999600' 1 '.reasons | index("SNAPSHOT_STALE") != null' +run_case future '.snapshot.observedAt=2000000001' 1 '.reasons | index("SNAPSHOT_STALE") != null' +run_case expired-runtime '.policy.runtimes.local.expiresAt=2000000000' 1 '.reasons | index("RUNTIME_EXPIRED") != null' +run_case disabled-runtime '.policy.runtimes.local.enabled=false' 1 '.reasons | index("RUNTIME_DISABLED") != null' +run_case observer-write '.policy.runtimes.local.role="observer"' 1 '.reasons | index("OBSERVER_WRITE") != null' +run_case child-limit '.task.children=2' 1 '.reasons | index("DELEGATION_LIMIT") != null' +run_case depth-limit '.task.depth=2' 1 '.reasons | index("DELEGATION_LIMIT") != null' +run_case unverified-controls '.snapshot.controls="unverified"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case no-evidence '.snapshot.evidenceRef=null' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case paygo '.snapshot.billing="paygo"' 1 '.reasons | index("BILLING_UNPROVEN") != null' +run_case negative-quota '.snapshot.buckets.weekly.remainingPercent=-1' 2 '.decision == "INVALID"' +run_case no-cross-runtime-reuse '.task.class="deepRefactor" | .snapshot.model="native-v1"' 1 '.reasons | index("RUNTIME_MISMATCH") != null' +printf '{broken' > "$TMP/broken.json" +status=0 +bash "$HERE/evaluate-inference-routing.sh" --now 2000000000 < "$TMP/broken.json" > "$TMP/output.json" 2>/dev/null || status=$? +[[ "$status" == 2 ]] && jq -e '.decision == "INVALID" and .executionAdmitted == false' "$TMP/output.json" > /dev/null +printf 'PASS malformed JSON\n' From 3b2be869700039df2f86a3f48d1d55a06e345038 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 12 Sep 2026 12:13:43 +0200 Subject: [PATCH 2/4] docs(routing): document evaluator helper functions --- .../resources/provider-neutral.desired-state.json | 2 +- .../agentic-engineering/scripts/evaluate-inference-routing.sh | 1 + .../scripts/evaluate-inference-routing.test.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 88b9f5e..8e4b0fa 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -28,7 +28,7 @@ }, { "path": "scripts/evaluate-inference-routing.sh", - "sha256": "d8a239f7e75d2e0d1ec457f4bbc280c7918a0f5d3db7a189d99ae2491fd83fa4", + "sha256": "4ff266055d8b96b84448fe843f1890baff2f5e716bb428b1e3b6fb4c60b4e49c", "executable": true } ], diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh index efa49fb..4a5c1b2 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh @@ -4,6 +4,7 @@ # Observations are caller reports, not authenticated runtime or billing evidence. # This command never launches inference, takes a lock, or reserves allowance. set -euo pipefail +# Emit a fixed, non-sensitive invalid-input result and terminate with exit status 2. invalid() { printf '%s\n' '{"decision":"INVALID","executionAdmitted":false,"reasons":["INVALID_INPUT"]}' exit 2 diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh index 56c1220..98bd1b5 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh @@ -37,6 +37,8 @@ cat > "$TMP/request.json" <<'JSON' }} } JSON +# Apply a fixture change, run the real evaluator, and assert its exit code and JSON result. +# Arguments: case name, jq fixture transformation, expected exit status, jq assertion. run_case() { local name="$1" change="$2" expected_exit="$3" assertion="$4" status=0 jq "$change" "$TMP/request.json" > "$TMP/input.json" From a5abc2bf74a4fba7d21aa7785235d92dedb175d3 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 12 Sep 2026 12:29:15 +0200 Subject: [PATCH 3/4] fix: reject opaque best model alias in routing policy --- plugins/agentic-engineering/resources/inference-routing.md | 3 ++- .../resources/provider-neutral.desired-state.json | 2 +- .../agentic-engineering/scripts/evaluate-inference-routing.sh | 2 +- .../scripts/evaluate-inference-routing.test.sh | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/agentic-engineering/resources/inference-routing.md b/plugins/agentic-engineering/resources/inference-routing.md index f9d10d2..8e503fa 100644 --- a/plugins/agentic-engineering/resources/inference-routing.md +++ b/plugins/agentic-engineering/resources/inference-routing.md @@ -65,7 +65,8 @@ automatic rung after diagnosis. The owner must also check task scope and previou The snapshot requires: -- `observedAt` (Unix seconds), `runtime`, `runtimeVersion`, intended exact `model` or `null`; +- `observedAt` (Unix seconds), `runtime`, `runtimeVersion`, runtime-reported resolved exact `model` + or `null` (never populate it by copying the policy's intended model); - `billing` (`included`, `unknown`, `paygo`), `controls` (`verified`, `unverified`), `evidenceRef` (a private verification record reference or `null`); - `buckets.short` and `buckets.weekly`, each `null` or an object with `remainingPercent`, diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 8e4b0fa..cc64e50 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -28,7 +28,7 @@ }, { "path": "scripts/evaluate-inference-routing.sh", - "sha256": "4ff266055d8b96b84448fe843f1890baff2f5e716bb428b1e3b6fb4c60b4e49c", + "sha256": "9861b64b6caff842025fab51ad55791e9101cfdf5bdcf837ba08d8c4aec2c809", "executable": true } ], diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh index 4a5c1b2..3fd4382 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh @@ -39,7 +39,7 @@ result=$(jq -sce --argjson now "$now" ' and all(.[]; identifier) and (map(ascii_downcase) | unique | length) == length) and (.routes | exact(["support","workhorse","diagnosis","deepRefactor"]) and all(.[]; exact(["model","runtime","effort"]) - and (.model | identifier and (ascii_downcase | IN("default","inherit","latest","auto") | not)) + and (.model | identifier and (ascii_downcase | IN("default","inherit","latest","auto","best") | not)) and (.runtime | identifier) and (.effort | IN("low","medium","high","xhigh")))) and (.runtimes | type == "object" and length > 0 diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh index 98bd1b5..a0f2658 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh @@ -65,6 +65,7 @@ run_case no-paid-fallback '.policy.paidFallback=true' 2 '.decision == "INVALID"' run_case denied-case '.policy.routes.workhorse.model="FORBIDDEN-FAMILY-v2"' 2 '.decision == "INVALID"' run_case denied-unused-route '.policy.routes.support.model="forbidden-family-v1"' 2 '.decision == "INVALID"' run_case opaque-default '.policy.routes.workhorse.model="default" | .snapshot.model="default"' 2 '.decision == "INVALID"' +run_case opaque-best '.policy.routes.workhorse.model="best" | .snapshot.model="best"' 2 '.decision == "INVALID"' run_case duplicated-denial '.policy.deniedModelTerms=["prohibited","PROHIBITED"]' 2 '.decision == "INVALID"' run_case missing-policy '.policy=null' 2 '.decision == "INVALID"' run_case malformed-class '.task.class="latest"' 2 '.decision == "INVALID"' From 1ddcdbc702dd2cf5e42f1d42a2314ac26d9c2be9 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 12 Sep 2026 12:42:59 +0200 Subject: [PATCH 4/4] fix: make invalid-input regression assertion fail reliably --- plugins/agentic-engineering/resources/inference-routing.md | 5 +++++ .../scripts/evaluate-inference-routing.test.sh | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/agentic-engineering/resources/inference-routing.md b/plugins/agentic-engineering/resources/inference-routing.md index 8e503fa..d4d4d09 100644 --- a/plugins/agentic-engineering/resources/inference-routing.md +++ b/plugins/agentic-engineering/resources/inference-routing.md @@ -44,6 +44,11 @@ hostile-input authentication boundary. Deployment model names and runtime IDs never enter portable agent frontmatter. An exact string in a policy is an intended model, not proof that the runtime supports it. Denial terms cover visible IDs only; opaque aliases/defaults/substitutions need native verification before execution. +The four task-class route keys are the stable logical aliases. Each route's `model` must already be +the reviewed resolved exact model ID; it is not a provider alias such as `reasoner-stable`. A native +adapter must resolve and verify any provider alias before this comparison, then select a policy +revision pinning that exact ID. Do not replace the observed model with the intended value or relax +equality to make an unresolved alias pass; a mismatch deliberately holds execution. ## Task and observation input diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh index a0f2658..e052cbb 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh @@ -61,6 +61,7 @@ run_case irreversible-write '.task.reversible=false | .snapshot.model="reasoner- run_case environment '.task.failureKind="environment"' 1 '.reasons | index("NON_REASONING_FAILURE") != null' run_case policy-off '.policy.enabled=false' 1 '.reasons | index("POLICY_DISABLED") != null' run_case no-model-override '.snapshot.model="other-v2"' 1 '.reasons | index("MODEL_MISMATCH") != null' +run_case unresolved-provider-alias '.policy.routes.workhorse.model="reasoner-stable" | .snapshot.model="reasoner-2026-09"' 1 '.reasons | index("MODEL_MISMATCH") != null' run_case no-paid-fallback '.policy.paidFallback=true' 2 '.decision == "INVALID"' run_case denied-case '.policy.routes.workhorse.model="FORBIDDEN-FAMILY-v2"' 2 '.decision == "INVALID"' run_case denied-unused-route '.policy.routes.support.model="forbidden-family-v1"' 2 '.decision == "INVALID"' @@ -90,5 +91,8 @@ run_case no-cross-runtime-reuse '.task.class="deepRefactor" | .snapshot.model="n printf '{broken' > "$TMP/broken.json" status=0 bash "$HERE/evaluate-inference-routing.sh" --now 2000000000 < "$TMP/broken.json" > "$TMP/output.json" 2>/dev/null || status=$? -[[ "$status" == 2 ]] && jq -e '.decision == "INVALID" and .executionAdmitted == false' "$TMP/output.json" > /dev/null +if [[ "$status" != 2 ]] || ! jq -e '.decision == "INVALID" and .executionAdmitted == false' "$TMP/output.json" > /dev/null; then + printf 'FAIL malformed JSON\n' >&2 + exit 1 +fi printf 'PASS malformed JSON\n'