From dc7810a7f158fd20f07807a57b782097ccd440d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=AD=D0=BC=D0=B8=D0=BB=D1=8C?= Date: Sat, 22 Aug 2026 02:30:15 +0400 Subject: [PATCH] Close the execution loop: facet-fct agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run` compiles one request and stops, so §16's Runtime Guard and Appendix F's provenance chain existed only in simulation: `ToolExecutor` was reachable from `test_runner` and its mocks, and from nowhere else. Deterministic agent execution was specified but never executed. `facet-fct agent` drives the turn cycle — the model answers, the guard decides, tools execute, results enter the next turn. The model sits behind a `ModelClient` trait so this stays a runtime rather than a vendor wrapper; `ScriptedClient` replays recorded turns, which makes guard behaviour testable without a network and gives item 3 a seam to plug a real provider into. Three properties now hold in fact rather than on paper: - A `tool_call` guard decision is taken before the call is initiated (§16.6.1a), against the effect class declared on the `@interface` function, and lands in the artifact with its input hash. - A denial stops the run with F454 *and still writes the artifact*. A refused run is exactly the case where provenance matters, so the denied decision is recorded before the error is reported (§18.1.3). This is why a denial is an outcome in `AgentOutcome` rather than an early return. - One hash chain spans the whole run, with contiguous `seq` across turns, and replaying the same script reproduces the head byte for byte — the property that makes an attestation over `hash_chain.head` worth anything. Tools are registered from the contract's own `@interface` blocks, so a call to something the contract never declared has nowhere to land. `run` and `agent` now share one compile path (`compile_contract`), so a turn built by the loop is byte-identical to the same contract compiled by `run`. Coverage rises to 57 of 206: the loop unblocks §16.6 and Appendix F statements that could not previously be exercised at all. What is still out of reach is recorded in ROADMAP.md and in docs/19-agent-loop.md — F455 needs a condition that fails at evaluation time rather than a policy that says no, and tool results are appended to the rendered payload rather than fed back through the document, so a turn is still a patched payload rather than a recompiled contract. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 19 ++ ROADMAP.md | 26 +- docs/19-agent-loop.md | 95 +++++++ docs/conformance/coverage-report.md | 6 +- docs/conformance/coverage.json | 39 +++ src/commands/agent.rs | 401 ++++++++++++++++++++++++++++ src/commands/mod.rs | 40 +++ src/commands/run.rs | 105 +++++--- src/main.rs | 21 ++ tests/agent_loop_tests.rs | 247 +++++++++++++++++ 11 files changed, 956 insertions(+), 45 deletions(-) create mode 100644 docs/19-agent-loop.md create mode 100644 src/commands/agent.rs create mode 100644 tests/agent_loop_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b438ebb..2f9aa60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: diff -q /tmp/requirements.json docs/conformance/requirements.json \ || (echo "Inventory is stale: re-run scripts/extract_requirements.py" && exit 1) - name: Coverage map is valid and has not regressed - run: python3 scripts/conformance_coverage.py --baseline 51 --markdown docs/conformance/coverage-report.md + run: python3 scripts/conformance_coverage.py --baseline 57 --markdown docs/conformance/coverage-report.md doc-examples: name: Doc Examples Compile diff --git a/CHANGELOG.md b/CHANGELOG.md index 44c31c2..60690ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/SemVer ## [Unreleased] +### Added +- **`facet-fct agent`: the host execution loop.** `run` compiles one request and + stops; `agent` drives the turn cycle — model answers, guard decides, tools + execute, results enter the next turn. The model sits behind a `ModelClient` + trait, so this is a runtime rather than a vendor wrapper; `ScriptedClient` + replays recorded turns. This makes §16 guard behaviour and Appendix F + provenance observable for the first time outside mocks: a real `tool_call` + decision, `F454` from a real denial, and one hash chain with contiguous `seq` + across every turn. A denied run still writes its artifact (§18.1.3). +- `docs/19-agent-loop.md`, and `tests/agent_loop_tests.rs` covering the allowed + path, the denied path, multi-turn chaining, replay reproducibility, and a call + to a tool the contract never declared. +- Specification coverage rises to 57 of 206 as the loop unblocks §16.6 and + Appendix F requirements that could not previously be exercised. + +### Changed +- `run` and `agent` share one compile path (`compile_contract`), so a turn built + by the loop is byte-identical to the same contract compiled by `run`. + ## [0.1.4] - 2026-08-22 ### Added diff --git a/ROADMAP.md b/ROADMAP.md index a1417e8..29d87bf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,9 +27,8 @@ that is a smoke test, not a conformance suite. language chapters are now largely attributed: §12 9/17, §13 6/9, §14 9/13, §15 3/3 - [ ] Fill gaps by risk order: §5 syntax → §8 FTS → §11 layout → §16 policy. - §16 sits at 12/65 and most of the remainder is guard *behaviour*, which - cannot be observed until item 2 closes the loop; §17 (5), §2 profiles (10) - and Appendix F (10) are likewise blocked on execution + §16 sits at 15/65; the loop from item 2 unblocked guard behaviour, so the + rest is now writable rather than blocked - [x] Publish the coverage report next to the compliance report (`docs/conformance/coverage-report.md`, regenerated by CI) @@ -43,14 +42,23 @@ recorded as untestable-here with a reason. effect classes, fail-closed denial — and Appendix F's hash chain are exercised only in simulation. Deterministic agent execution is specified, not executed. -- [ ] Host runtime: canonical JSON → provider → parse tool calls → guard → - `ToolExecutor` → next turn -- [ ] Multi-turn Execution Artifact: one hash chain across the whole run -- [ ] Replay: re-run a recorded artifact and assert an identical chain head -- [ ] `F454` / `F455` raised from a real denied call, not a mock +- [x] Host runtime: canonical JSON → model → parse tool calls → guard → + `ToolExecutor` → next turn (`facet-fct agent`, `src/commands/agent.rs`). + The model sits behind a `ModelClient` trait so this stays a runtime, not a + vendor wrapper; `ScriptedClient` replays recorded turns +- [x] Multi-turn Execution Artifact: one hash chain across the whole run, with + contiguous `seq` over every turn +- [x] Replay: the same script reproduces the chain head byte for byte +- [x] `F454` raised from a real denied call, and the denied decision is still + written to the artifact — a refused run is exactly when provenance matters +- [ ] `F455` from a genuinely undecidable guard state (needs a condition that + fails at evaluation time, not a policy that says no) +- [ ] Feed tool results back through the contract rather than as appended + messages, so a turn is a compiled artifact rather than a patched payload **Done when** an agent run that calls a tool produces a verifiable provenance -chain, and replaying it reproduces the chain head byte for byte. +chain, and replaying it reproduces the chain head byte for byte. *(Reached for +the scripted client; a real provider arrives with item 3.)* ## 3. Ship one real adapter diff --git a/docs/19-agent-loop.md b/docs/19-agent-loop.md new file mode 100644 index 0000000..1c645a9 --- /dev/null +++ b/docs/19-agent-loop.md @@ -0,0 +1,95 @@ +--- +permalink: /19-agent-loop.html +title: Agent Loop +--- + +# 19. The agent loop + +`run` compiles one request and stops. An agent is the loop around it: the model +answers, asks for tools, the guard decides, tools execute, and their results +enter the next turn. + +FACET does not standardize that loop — the host owns it. What FACET standardizes +is what must hold *while* it runs: the Runtime Guard (§16.6), effect classes +(§16.5), and the provenance record (Appendix F). None of it is observable in a +single compile-and-stop invocation, which is why the loop lives here. + +## The model is behind a trait + +```rust +pub trait ModelClient { + fn complete(&mut self, turn: usize, request: &CanonicalPayload) -> Result; +} +``` + +This keeps the compiler a runtime rather than a vendor wrapper. `ScriptedClient` +replays recorded turns, so guard behaviour is testable without a network; a +provider adapter is the same trait with a socket behind it. + +## Running a scripted conversation + +```bash +facet-fct agent --input contract.facet --script turns.json --artifact execution.json --exec +``` + +`turns.json` records what the model returns and what the tools produce: + +```json +{ + "turns": [ + { "tool_calls": [ { "id": "c1", "name": "WeatherAPI.get_current", "arguments": { "city": "Minsk" } } ] }, + { "text": "It is raining in Minsk." } + ], + "tool_results": { "WeatherAPI.get_current": "Rain, 11C" } +} +``` + +Each turn: + +1. the client returns an answer for the current canonical payload +2. no tool calls → the run completes with that text +3. otherwise, for each call: the declared `effect` is looked up from + `@interface`, the guard decides, and only then does the tool execute +4. the result enters the next turn's context + +Tools are registered from the contract's own `@interface` blocks, so a call to +something the contract never declared has nowhere to land. + +## What lands in the artifact + +One hash chain covers the whole run, with `seq` contiguous across turns +(Appendix F.3–F.4): + +``` +seq=1 message_emit system#1 allowed +seq=2 message_emit user#1 allowed +seq=3 tool_expose WeatherAPI.get_current allowed rule=expose-weather +seq=4 tool_call WeatherAPI.get_current allowed rule=call-weather +``` + +Remove the `tool_call` rule from `@policy` and the same script produces: + +``` +seq=4 tool_call WeatherAPI.get_current denied +``` + +The run stops with `F454` and the tool never executes — **and the artifact is +still written**. A refused run is exactly the case where the provenance record +matters, so the denial is recorded before the error is reported (§18.1.3). + +## Replay + +The same contract and the same script reproduce the same chain head byte for +byte. That is the property that makes an artifact worth signing: an attestation +over `hash_chain.head` is only meaningful if the head is reproducible. + +## Current limits + +- `F455` (undecidable guard state) is not yet reachable from the loop; it needs + a condition that fails at evaluation time rather than a policy that says no. +- Tool results are appended to the rendered payload as messages. A turn is + therefore a patched payload rather than a recompiled contract — acceptable for + provenance, but the next step is to feed results back through the document so + every turn is a compiled artifact in its own right. +- The only client is `ScriptedClient`. A real provider adapter is the next + roadmap item. diff --git a/docs/conformance/coverage-report.md b/docs/conformance/coverage-report.md index 3f36f09..ffc65aa 100644 --- a/docs/conformance/coverage-report.md +++ b/docs/conformance/coverage-report.md @@ -1,6 +1,6 @@ # Specification coverage -51 of 206 normative statements are exercised by a named test (24%); 0 are recorded as not testable here. +57 of 206 normative statements are exercised by a named test (27%); 0 are recorded as not testable here. | Chapter | Requirements | Covered | Waived | |---------|-------------:|--------:|-------:| @@ -18,7 +18,7 @@ | §13 | 9 | 6 | 0 | | §14 | 13 | 9 | 0 | | §15 | 3 | 3 | 0 | -| §16 | 65 | 12 | 0 | +| §16 | 65 | 15 | 0 | | §17 | 5 | 0 | 0 | | §18 | 11 | 1 | 0 | | §19 | 1 | 0 | 0 | @@ -26,4 +26,4 @@ | §Appendix A | 1 | 0 | 0 | | §Appendix C | 2 | 0 | 0 | | §Appendix D | 1 | 0 | 0 | -| §Appendix F | 10 | 0 | 0 | +| §Appendix F | 10 | 3 | 0 | diff --git a/docs/conformance/coverage.json b/docs/conformance/coverage.json index 8618d1f..819f0a1 100644 --- a/docs/conformance/coverage.json +++ b/docs/conformance/coverage.json @@ -346,5 +346,44 @@ "policy_validation_tests::policy_accepts_valid_rule_and_condition" ], "note": "§16.3 PolicyCond forms" + }, + "9494d30f50b323dd": { + "tests": [ + "agent_loop_tests::allowed_tool_call_is_executed_and_recorded", + "agent_loop_tests::denied_tool_call_stops_the_run_and_is_still_recorded" + ], + "note": "§16.6.1 the guard runs on every tool_call, allowed and denied alike" + }, + "d937a7d07bc662de": { + "tests": [ + "agent_loop_tests::denied_tool_call_stops_the_run_and_is_still_recorded", + "agent_loop_tests::an_undeclared_tool_cannot_be_called" + ], + "note": "§16.6.1a the decision precedes execution: a denied call never runs" + }, + "997c6920fd1fc224": { + "tests": [ + "agent_loop_tests::denied_tool_call_stops_the_run_and_is_still_recorded" + ], + "note": "§16.6.6 a deterministic deny is F454" + }, + "2bea6c78116dc1ac": { + "tests": [ + "agent_loop_tests::allowed_tool_call_is_executed_and_recorded" + ], + "note": "Appendix F.3 GuardDecision shape: op, name, effect_class, decision, input_hash" + }, + "632a37ff15ef05d7": { + "tests": [ + "agent_loop_tests::every_turn_shares_one_hash_chain", + "agent_loop_tests::replaying_the_same_script_reproduces_the_chain_head" + ], + "note": "Appendix F.2 artifact shape; contiguous seq and a chain over the whole run" + }, + "3a81c34d065c7024": { + "tests": [ + "agent_loop_tests::replaying_the_same_script_reproduces_the_chain_head" + ], + "note": "Appendix F.1 identical inputs reproduce the artifact" } } diff --git a/src/commands/agent.rs b/src/commands/agent.rs new file mode 100644 index 0000000..dc45af9 --- /dev/null +++ b/src/commands/agent.rs @@ -0,0 +1,401 @@ +//! The host execution loop. +//! +//! `run` compiles one request and stops. An agent is the loop around it: the +//! model answers, asks for tools, the guard decides, tools execute, and their +//! results enter the next turn. FACET does not standardize that loop — but it +//! does standardize what must hold while it runs (§16 guard, Appendix F +//! provenance), and none of that is observable until the loop exists. +//! +//! The model is behind a trait, so this stays a runtime rather than a vendor +//! wrapper. `ScriptedClient` replays recorded turns, which is what makes guard +//! behaviour testable without a network. + +use anyhow::{Context, Result}; +use fct_ast::{FacetNode, OrderedMap, ScalarValue, ValueNode}; +use fct_engine::{ + ExecutionGuardDecision, ExecutionMode, ToolDefinition, ToolExecutor, ToolInvocation, +}; +use fct_render::{CanonicalMessage, CanonicalPayload, Content, GuardDecision}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use tracing::info; + +use crate::commands::artifact::build_execution_artifact; +use crate::commands::canonical::canonicalize_json; +use crate::commands::mode_profile::resolve_execution_mode; +use crate::commands::run::compile_contract; + +/// One tool call the model asked for. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ToolCallRequest { + /// Canonical `Interface.fn` name (§16.2.6). + pub name: String, + #[serde(default)] + pub arguments: serde_json::Map, + #[serde(default)] + pub id: Option, +} + +/// What the model returned for one turn. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct ModelTurn { + #[serde(default)] + pub text: Option, + #[serde(default)] + pub tool_calls: Vec, +} + +/// A model, from the runtime's point of view. +pub trait ModelClient { + fn complete(&mut self, turn: usize, request: &CanonicalPayload) -> Result; +} + +/// A recorded conversation: turns the model "returns" and results tools produce. +#[derive(Debug, Deserialize)] +pub struct Script { + pub turns: Vec, + /// Tool results by canonical `Interface.fn` name. + #[serde(default)] + pub tool_results: HashMap, +} + +pub struct ScriptedClient { + turns: Vec, +} + +impl ScriptedClient { + pub fn new(turns: Vec) -> Self { + Self { turns } + } +} + +impl ModelClient for ScriptedClient { + fn complete(&mut self, turn: usize, _request: &CanonicalPayload) -> Result { + self.turns + .get(turn) + .cloned() + .ok_or_else(|| anyhow::anyhow!("script ran out of turns at turn {}", turn + 1)) + } +} + +/// Why the loop stopped. +#[derive(Debug, Clone, PartialEq)] +pub enum AgentStop { + /// The model answered without asking for more tools. + Completed, + /// The guard denied a call (§16.6.6, F454). + PolicyDenied { name: String }, + /// The guard could not reach a deterministic decision (F455). + GuardUndecidable { name: String }, + /// The turn limit was reached first. + MaxTurns, +} + +/// What one agent run produced. +/// +/// A denial is an outcome, not an early return: the run stops, but the decision +/// that stopped it belongs in the provenance record (§18.1.3), so the caller +/// still receives every event and can write the artifact before reporting the +/// error. +pub struct AgentOutcome { + pub turns: usize, + pub final_text: Option, + pub guard_decisions: Vec, + pub payload: CanonicalPayload, + pub stop: AgentStop, +} + +/// Look up the declared effect class for `Interface.fn` (§13.1, §16.5.2). +/// +/// The guard needs it before the call is initiated; an unknown tool has no +/// declared effect and therefore no non-null class, which the guard treats as +/// unsafe (§16.4.3). +fn declared_effect(document: &fct_ast::FacetDocument, canonical_name: &str) -> Option { + let (interface_name, fn_name) = canonical_name.split_once('.')?; + for node in &document.blocks { + if let FacetNode::Interface(interface) = node { + if interface.name != interface_name { + continue; + } + for function in &interface.functions { + if function.name == fn_name { + return function.effect.clone(); + } + } + } + } + None +} + +fn json_to_value_node(value: &serde_json::Value) -> ValueNode { + match value { + serde_json::Value::Null => ValueNode::Scalar(ScalarValue::Null), + serde_json::Value::Bool(b) => ValueNode::Scalar(ScalarValue::Bool(*b)), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + ValueNode::Scalar(ScalarValue::Int(i)) + } else { + ValueNode::Scalar(ScalarValue::Float(n.as_f64().unwrap_or_default())) + } + } + serde_json::Value::String(s) => ValueNode::String(s.clone()), + serde_json::Value::Array(items) => { + ValueNode::List(items.iter().map(json_to_value_node).collect()) + } + serde_json::Value::Object(map) => { + let mut out = OrderedMap::new(); + for (key, item) in map { + out.insert(key.clone(), json_to_value_node(item)); + } + ValueNode::Map(out) + } + } +} + +fn engine_decision_to_render(decision: ExecutionGuardDecision) -> GuardDecision { + GuardDecision { + seq: decision.seq, + op: decision.op, + name: decision.name, + effect_class: decision.effect_class, + mode: decision.mode, + decision: decision.decision, + policy_rule_id: decision.policy_rule_id, + input_hash: decision.input_hash, + error_code: decision.error_code, + } +} + +/// Drive a contract through as many turns as the model asks for. +/// +/// Every guarded operation across every turn lands in one decision list, so the +/// artifact carries a single hash chain over the whole run (Appendix F.4). +#[allow(clippy::too_many_arguments)] +pub fn drive( + input: &Path, + runtime_input: Option<&Path>, + budget: usize, + context_budget: usize, + execution_mode: ExecutionMode, + mode: &str, + client: &mut dyn ModelClient, + tool_results: &HashMap, + max_turns: usize, +) -> Result { + let compiled = compile_contract( + input, + runtime_input, + budget, + context_budget, + execution_mode, + mode, + )?; + + let mut payload = compiled.payload; + let mut decisions = compiled.guard_decisions; + let document = compiled.resolved; + let policy = compiled.effective_policy; + let computed_vars = compiled.computed_vars; + + let mut executor = ToolExecutor::new(); + + // Register every tool the contract declares, so a call to something the + // contract never declared fails as an unknown tool rather than silently + // finding a handler. + for node in &document.blocks { + if let FacetNode::Interface(interface) = node { + for function in &interface.functions { + executor + .register_tool(ToolDefinition { + name: format!("{}.{}", interface.name, function.name), + description: format!("Declared by @interface {}", interface.name), + input_schema: serde_json::json!({ "type": "object" }), + output_schema: None, + }) + .map_err(|e| anyhow::anyhow!("failed to register tool: {}", e))?; + } + } + } + + for (name, result) in tool_results { + let value = json_to_value_node(result); + executor + .register_handler(name.clone(), move |_| Ok(value.clone())) + .map_err(|e| anyhow::anyhow!("failed to register tool result for {}: {}", name, e))?; + } + + let mut final_text = None; + let mut turn = 0; + let mut stop = AgentStop::MaxTurns; + + while turn < max_turns { + let answer = client.complete(turn, &payload)?; + turn += 1; + + if answer.tool_calls.is_empty() { + final_text = answer.text; + stop = AgentStop::Completed; + break; + } + + for call in &answer.tool_calls { + let effect = declared_effect(&document, &call.name); + let mut arguments = HashMap::new(); + for (key, value) in &call.arguments { + arguments.insert(key.clone(), json_to_value_node(value)); + } + + let invocation = ToolInvocation { + tool_name: call.name.clone(), + arguments, + invocation_id: call.id.clone(), + }; + + // The guard decides before anything external happens (§16.6.1a). + let decision = executor.evaluate_tool_call_guard( + &invocation, + policy.as_ref(), + Some(&computed_vars), + mode, + &payload.metadata.host_profile_id, + effect.as_deref(), + )?; + let denied = decision.decision == "denied"; + let undecidable = decision.error_code.as_deref() == Some("F455"); + decisions.push(engine_decision_to_render(decision)); + + if undecidable { + stop = AgentStop::GuardUndecidable { + name: call.name.clone(), + }; + break; + } + if denied { + stop = AgentStop::PolicyDenied { + name: call.name.clone(), + }; + break; + } + + let result = executor + .execute(invocation) + .with_context(|| format!("tool {} failed", call.name))?; + + info!("turn {}: {} -> ok", turn, call.name); + + // Tool output enters the next turn as context. Canonical roles are + // system/user/assistant (§12.3), so a result is carried as a user + // message; the loop shape is host-defined, the roles are not. + payload.messages.push(CanonicalMessage { + role: "assistant".to_string(), + content: Content::Text(format!("[tool_call] {}", call.name)), + }); + payload.messages.push(CanonicalMessage { + role: "user".to_string(), + content: Content::Text(format!( + "[tool_result] {} {}", + call.name, + value_preview(&result.result) + )), + }); + } + + if stop != AgentStop::MaxTurns { + break; + } + } + + Ok(AgentOutcome { + turns: turn, + final_text, + guard_decisions: decisions, + payload, + stop, + }) +} + +fn value_preview(value: &ValueNode) -> String { + match value { + ValueNode::String(s) => s.clone(), + other => format!("{:?}", other), + } +} + +/// `facet-fct agent` — run a contract through the loop against a recorded script. +#[allow(clippy::too_many_arguments)] +pub fn execute_agent( + input: std::path::PathBuf, + runtime_input: Option, + script: std::path::PathBuf, + artifact: Option, + budget: usize, + context_budget: usize, + max_turns: usize, + pure: bool, + exec: bool, +) -> Result<()> { + let (execution_mode, mode) = resolve_execution_mode(pure, exec)?; + + let script_source = fs::read_to_string(&script) + .with_context(|| format!("Failed to read script: {:?}", script))?; + let script: Script = + serde_json::from_str(&script_source).context("Failed to parse turn script")?; + + let mut client = ScriptedClient::new(script.turns); + let outcome = drive( + &input, + runtime_input.as_deref(), + budget, + context_budget, + execution_mode, + mode, + &mut client, + &script.tool_results, + max_turns, + )?; + + let execution_artifact = build_execution_artifact(&outcome.payload, &outcome.guard_decisions)?; + let artifact_path = artifact.unwrap_or_else(|| { + input + .parent() + .map(|p| p.join("execution.json")) + .unwrap_or_else(|| std::path::PathBuf::from("execution.json")) + }); + fs::write(&artifact_path, canonicalize_json(&execution_artifact)?) + .with_context(|| format!("Failed to write execution artifact: {:?}", artifact_path))?; + + let head = execution_artifact + .pointer("/provenance/hash_chain/head") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + let stop = match &outcome.stop { + AgentStop::Completed => "completed".to_string(), + AgentStop::MaxTurns => "max_turns".to_string(), + AgentStop::PolicyDenied { name } => format!("policy_denied:{name}"), + AgentStop::GuardUndecidable { name } => format!("guard_undecidable:{name}"), + }; + + println!( + "{}", + serde_json::to_string(&serde_json::json!({ + "turns": outcome.turns, + "stop": stop, + "final_text": outcome.final_text, + "guard_events": outcome.guard_decisions.len(), + "hash_chain_head": head, + }))? + ); + + // The artifact is written first: a denied run is exactly the case where the + // provenance record matters most. + match outcome.stop { + AgentStop::PolicyDenied { name } => Err(anyhow::anyhow!("F454: policy denied {}", name)), + AgentStop::GuardUndecidable { name } => { + Err(anyhow::anyhow!("F455: guard could not decide on {}", name)) + } + _ => Ok(()), + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index dbacdc3..3b6054f 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; pub type DefaultRateLimiter = RateLimiter; +pub mod agent; pub mod artifact; pub mod build; pub mod canonical; @@ -149,6 +150,45 @@ pub enum Commands { exec: bool, }, + /// Drive a contract through the host execution loop against a recorded script + Agent { + /// Input FACET file path + #[arg(short, long)] + input: PathBuf, + + /// Runtime input values for @input variables + #[arg(long)] + runtime_input: Option, + + /// Recorded model turns and tool results + #[arg(long)] + script: PathBuf, + + /// Where to write the execution artifact + #[arg(long)] + artifact: Option, + + /// Layout budget when the document does not set one + #[arg(long, default_value = "4096")] + budget: usize, + + /// Gas limit for the reactive compute phase + #[arg(long, default_value = "10000")] + context_budget: usize, + + /// Stop after this many turns + #[arg(long, default_value = "8")] + max_turns: usize, + + /// Pure mode + #[arg(long)] + pure: bool, + + /// Exec mode (default) + #[arg(long)] + exec: bool, + }, + /// Generate SDK from FACET interfaces Codegen { /// Input FACET file path diff --git a/src/commands/run.rs b/src/commands/run.rs index 6aca75b..6669e3f 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -25,33 +25,28 @@ use std::collections::HashMap; use std::fs; use tracing::info; -/// Run command handler -#[allow(clippy::too_many_arguments)] -pub fn execute_run( - input: std::path::PathBuf, - runtime_input: Option, +/// The compiled result of one contract: everything the host needs for a turn. +pub(crate) struct CompiledContract { + pub payload: CanonicalPayload, + pub guard_decisions: Vec, + pub resolved: FacetDocument, + pub computed_vars: std::collections::HashMap, + pub effective_policy: Option>, +} + +/// Run phases 1-5 for a document: resolve, type check, compute, lay out, render. +/// +/// `run` and `agent` share this so a turn built by the agent loop is byte-identical +/// to the same contract compiled by `run`. +pub(crate) fn compile_contract( + input: &std::path::Path, + runtime_input: Option<&std::path::Path>, budget: usize, context_budget: usize, - format: String, - pure: bool, - exec: bool, - _no_progress: bool, - rate_limiter: &crate::commands::DefaultRateLimiter, -) -> Result<()> { - // Check rate limit - if rate_limiter.check().is_err() { - eprintln!( - "{}", - style("L Rate limit exceeded. Please wait before running another command.").red() - ); - std::process::exit(1); - } - - info!("Starting full pipeline for file: {:?}", input); - - let (execution_mode, mode) = resolve_execution_mode(pure, exec)?; - - let source = fs::read_to_string(&input) + execution_mode: fct_engine::ExecutionMode, + mode: &str, +) -> Result { + let source = fs::read_to_string(input) .with_context(|| format!("Failed to read input file: {:?}", input))?; let parsed = parse_document(&source).map_err(|e| anyhow::anyhow!("Parse error: {}", e))?; @@ -81,17 +76,12 @@ pub fn execute_run( engine.validate()?; let mut exec_ctx = ExecutionContext::new_with_mode(context_budget, execution_mode); if let Some(runtime_input_path) = runtime_input { - let runtime_inputs = load_runtime_inputs(&runtime_input_path)?; + let runtime_inputs = load_runtime_inputs(runtime_input_path)?; exec_ctx.set_inputs(runtime_inputs); } engine.execute(&mut exec_ctx)?; - // Report the budget actually used: `@context budget` wins over the host default. let effective_budget = effective_layout_budget(&resolved, budget); - info!( - "Layout budget: {} facet units (host default {}), gas limit: {}", - effective_budget, budget, context_budget - ); let lens_registry = LensRegistry::new(); let sections = doc_to_sections(&resolved, &exec_ctx.variables, &lens_registry)?; let box_model = TokenBoxModel::new(effective_budget); @@ -112,10 +102,61 @@ pub fn execute_run( computed_vars: Some(exec_ctx.variables.clone()), }, )?; - let payload = render_output.payload; let guard_decisions = merge_guard_decisions(&exec_ctx.guard_decisions, &render_output.guard_decisions); + + Ok(CompiledContract { + payload: render_output.payload, + guard_decisions, + resolved, + computed_vars: exec_ctx.variables.clone(), + effective_policy: exec_ctx.effective_policy.clone(), + }) +} + +/// Run command handler +#[allow(clippy::too_many_arguments)] +pub fn execute_run( + input: std::path::PathBuf, + runtime_input: Option, + budget: usize, + context_budget: usize, + format: String, + pure: bool, + exec: bool, + _no_progress: bool, + rate_limiter: &crate::commands::DefaultRateLimiter, +) -> Result<()> { + // Check rate limit + if rate_limiter.check().is_err() { + eprintln!( + "{}", + style("L Rate limit exceeded. Please wait before running another command.").red() + ); + std::process::exit(1); + } + + info!("Starting full pipeline for file: {:?}", input); + + let (execution_mode, mode) = resolve_execution_mode(pure, exec)?; + + let compiled = compile_contract( + &input, + runtime_input.as_deref(), + budget, + context_budget, + execution_mode, + mode, + )?; + let payload = compiled.payload; + let guard_decisions = compiled.guard_decisions; + + info!( + "Layout budget: {} facet units (host default {}), gas limit: {}", + payload.metadata.budget_units, budget, context_budget + ); + let execution_artifact = build_execution_artifact(&payload, &guard_decisions)?; let execution_json = canonicalize_json(&execution_artifact)?; let execution_path = input diff --git a/src/main.rs b/src/main.rs index 1b855aa..e326960 100644 --- a/src/main.rs +++ b/src/main.rs @@ -84,6 +84,27 @@ fn main() -> anyhow::Result<()> { exec, &rate_limiter, ), + Commands::Agent { + input, + runtime_input, + script, + artifact, + budget, + context_budget, + max_turns, + pure, + exec, + } => commands::agent::execute_agent( + input, + runtime_input, + script, + artifact, + budget, + context_budget, + max_turns, + pure, + exec, + ), Commands::Codegen { input, output, diff --git a/tests/agent_loop_tests.rs b/tests/agent_loop_tests.rs new file mode 100644 index 0000000..8cab2a1 --- /dev/null +++ b/tests/agent_loop_tests.rs @@ -0,0 +1,247 @@ +//! The execution loop, end to end. +//! +//! These exercise what could previously only be simulated: a guard decision on +//! a real `tool_call`, taken before the call is initiated, recorded in a +//! provenance chain that spans every turn of the run. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +const CONTRACT: &str = r#"@context + budget: 32000 + +@interface WeatherAPI + fn get_current(city: string) -> string (effect="read") + +@policy + allow: + - id: "expose-weather" + op: "tool_expose" + name: "WeatherAPI.get_current" + effect: "read" + - id: "call-weather" + op: "tool_call" + name: "WeatherAPI.get_current" + effect: "read" + +@system + tools: [$WeatherAPI] + content: "Use WeatherAPI.get_current when asked about weather." + +@user + content: "What is the weather in Minsk?" +"#; + +/// Same contract, but nothing allows the call itself. +const CONTRACT_NO_CALL_RULE: &str = r#"@context + budget: 32000 + +@interface WeatherAPI + fn get_current(city: string) -> string (effect="read") + +@policy + allow: + - id: "expose-weather" + op: "tool_expose" + name: "WeatherAPI.get_current" + effect: "read" + +@system + tools: [$WeatherAPI] + content: "Use WeatherAPI.get_current when asked about weather." + +@user + content: "What is the weather in Minsk?" +"#; + +const SCRIPT: &str = r#"{ + "turns": [ + { "tool_calls": [ { "id": "c1", "name": "WeatherAPI.get_current", "arguments": { "city": "Minsk" } } ] }, + { "text": "It is raining in Minsk." } + ], + "tool_results": { "WeatherAPI.get_current": "Rain, 11C" } +} +"#; + +const SCRIPT_TWO_CALLS: &str = r#"{ + "turns": [ + { "tool_calls": [ { "id": "c1", "name": "WeatherAPI.get_current", "arguments": { "city": "Minsk" } } ] }, + { "tool_calls": [ { "id": "c2", "name": "WeatherAPI.get_current", "arguments": { "city": "Vilnius" } } ] }, + { "text": "Both cities are wet." } + ], + "tool_results": { "WeatherAPI.get_current": "Rain, 11C" } +} +"#; + +struct Case { + dir: PathBuf, +} + +impl Case { + fn new(name: &str, contract: &str, script: &str) -> Self { + let dir = std::env::temp_dir().join(format!("facet_agent_{name}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp dir"); + std::fs::write(dir.join("contract.facet"), contract).expect("write contract"); + std::fs::write(dir.join("script.json"), script).expect("write script"); + Self { dir } + } + + fn run(&self, artifact: &str) -> (bool, String, serde_json::Value) { + let artifact_path = self.dir.join(artifact); + let output = Command::new(env!("CARGO_BIN_EXE_facet-fct")) + .arg("agent") + .arg("--input") + .arg(self.dir.join("contract.facet")) + .arg("--script") + .arg(self.dir.join("script.json")) + .arg("--artifact") + .arg(&artifact_path) + .arg("--exec") + .output() + .expect("run agent"); + + let stdout = String::from_utf8(output.stdout).expect("utf-8"); + let artifact = read_artifact(&artifact_path); + (output.status.success(), stdout, artifact) + } +} + +impl Drop for Case { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn read_artifact(path: &Path) -> serde_json::Value { + let text = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!("artifact {} must exist: {e}", path.display())); + serde_json::from_str(&text).expect("artifact is JSON") +} + +fn events(artifact: &serde_json::Value) -> Vec<&serde_json::Value> { + artifact["provenance"]["events"] + .as_array() + .expect("events array") + .iter() + .collect() +} + +fn head(artifact: &serde_json::Value) -> String { + artifact["provenance"]["hash_chain"]["head"] + .as_str() + .expect("chain head") + .to_string() +} + +#[test] +fn allowed_tool_call_is_executed_and_recorded() { + let case = Case::new("allowed", CONTRACT, SCRIPT); + let (ok, stdout, artifact) = case.run("execution.json"); + + assert!(ok, "an allowed run must succeed: {stdout}"); + assert!(stdout.contains("It is raining in Minsk.")); + + let calls: Vec<_> = events(&artifact) + .into_iter() + .filter(|e| e["op"] == "tool_call") + .collect(); + + assert_eq!(calls.len(), 1, "one tool_call event expected"); + assert_eq!(calls[0]["decision"], "allowed"); + assert_eq!(calls[0]["effect_class"], "read"); + assert_eq!(calls[0]["name"], "WeatherAPI.get_current"); + assert!( + calls[0]["input_hash"] + .as_str() + .is_some_and(|h| h.starts_with("sha256:")), + "§F.3 requires an input hash on every decision" + ); +} + +#[test] +fn denied_tool_call_stops_the_run_and_is_still_recorded() { + let case = Case::new("denied", CONTRACT_NO_CALL_RULE, SCRIPT); + let (ok, stdout, artifact) = case.run("execution.json"); + + assert!(!ok, "a denied run must fail"); + assert!( + stdout.contains("policy_denied:WeatherAPI.get_current"), + "the stop reason belongs in the report: {stdout}" + ); + + let calls: Vec<_> = events(&artifact) + .into_iter() + .filter(|e| e["op"] == "tool_call") + .collect(); + + assert_eq!( + calls.len(), + 1, + "the denied call is still an event (§18.1.3)" + ); + assert_eq!(calls[0]["decision"], "denied"); +} + +#[test] +fn every_turn_shares_one_hash_chain() { + let case = Case::new("multiturn", CONTRACT, SCRIPT_TWO_CALLS); + let (ok, _stdout, artifact) = case.run("execution.json"); + assert!(ok); + + let all = events(&artifact); + let calls = all.iter().filter(|e| e["op"] == "tool_call").count(); + assert_eq!(calls, 2, "both turns' calls land in the same artifact"); + + // §F.3: seq starts at 1 and increments with no gaps, across turns. + for (index, event) in all.iter().enumerate() { + assert_eq!( + event["seq"].as_u64().expect("seq"), + index as u64 + 1, + "sequence numbers must be contiguous across the whole run" + ); + } +} + +#[test] +fn replaying_the_same_script_reproduces_the_chain_head() { + let case = Case::new("replay", CONTRACT, SCRIPT); + let (ok_first, _, first) = case.run("first.json"); + let (ok_second, _, second) = case.run("second.json"); + + assert!(ok_first && ok_second); + assert_eq!( + head(&first), + head(&second), + "identical inputs must reproduce the provenance chain byte for byte" + ); +} + +#[test] +fn an_undeclared_tool_cannot_be_called() { + let script = r#"{ + "turns": [ + { "tool_calls": [ { "name": "Payments.charge", "arguments": {} } ] } + ], + "tool_results": { "Payments.charge": "ok" } +} +"#; + let case = Case::new("undeclared", CONTRACT, script); + let artifact_path = case.dir.join("execution.json"); + let output = Command::new(env!("CARGO_BIN_EXE_facet-fct")) + .arg("agent") + .arg("--input") + .arg(case.dir.join("contract.facet")) + .arg("--script") + .arg(case.dir.join("script.json")) + .arg("--artifact") + .arg(&artifact_path) + .arg("--exec") + .output() + .expect("run agent"); + + assert!( + !output.status.success(), + "a tool the contract never declared must not execute" + ); +}