From 2a08d16f90991745f157a8850527dc1787f9359c Mon Sep 17 00:00:00 2001 From: BALAJI BASHYAM Date: Sun, 27 Sep 2026 15:06:16 +0000 Subject: [PATCH 1/3] Add evidence-first Agent Framework incident triage sample --- .../.gitignore | 87 ++ .../AgentFrameworkIncidentTriage.slnx | 8 + .../agent-framework-incident-triage/README.md | 149 +++ .../docs/architecture.md | 805 ++++++++++++ .../docs/evidence-first-contract.md | 651 +++++++++ .../docs/sample-run.md | 1170 +++++++++++++++++ .../docs/tool-risk-ladder.md | 491 +++++++ .../docs/verified-environment.md | 643 +++++++++ .../global.json | 7 + .../Agents/TriageAgentFactory.cs | 136 ++ .../IncidentTriageAgent/Data/deployments.json | 26 + .../IncidentTriageAgent/Data/runbooks.json | 26 + .../IncidentTriageAgent/Data/services.json | 25 + .../Evidence/EvidenceItem.cs | 26 + .../Evidence/EvidenceStore.cs | 152 +++ .../Guidance/RunbookItem.cs | 14 + .../Guidance/RunbookStore.cs | 90 ++ .../IncidentTriageAgent.csproj | 20 + .../src/IncidentTriageAgent/Program.cs | 209 +++ .../Tools/DeploymentTool.cs | 63 + .../IncidentTriageAgent/Tools/RunbookTool.cs | 77 ++ .../Tools/ServiceHealthTool.cs | 62 + .../EvidenceStoreTests.cs | 101 ++ .../IncidentTriageAgent.Tests.csproj | 25 + .../RunbookStoreTests.cs | 60 + .../IncidentTriageAgent.Tests/ToolTests.cs | 127 ++ .../verified-environment.json | 40 + 27 files changed, 5290 insertions(+) create mode 100644 dotnet-ai/agent-framework-incident-triage/.gitignore create mode 100644 dotnet-ai/agent-framework-incident-triage/AgentFrameworkIncidentTriage.slnx create mode 100644 dotnet-ai/agent-framework-incident-triage/README.md create mode 100644 dotnet-ai/agent-framework-incident-triage/docs/architecture.md create mode 100644 dotnet-ai/agent-framework-incident-triage/docs/evidence-first-contract.md create mode 100644 dotnet-ai/agent-framework-incident-triage/docs/sample-run.md create mode 100644 dotnet-ai/agent-framework-incident-triage/docs/tool-risk-ladder.md create mode 100644 dotnet-ai/agent-framework-incident-triage/docs/verified-environment.md create mode 100644 dotnet-ai/agent-framework-incident-triage/global.json create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Agents/TriageAgentFactory.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/deployments.json create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/runbooks.json create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/services.json create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceItem.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceStore.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookItem.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookStore.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/IncidentTriageAgent.csproj create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Program.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/DeploymentTool.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/RunbookTool.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/ServiceHealthTool.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/EvidenceStoreTests.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/IncidentTriageAgent.Tests.csproj create mode 100644 dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/RunbookStoreTests.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/ToolTests.cs create mode 100644 dotnet-ai/agent-framework-incident-triage/verified-environment.json diff --git a/dotnet-ai/agent-framework-incident-triage/.gitignore b/dotnet-ai/agent-framework-incident-triage/.gitignore new file mode 100644 index 0000000..a6b6374 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/.gitignore @@ -0,0 +1,87 @@ +# ============================================================ +# .NET build output +# ============================================================ + +bin/ +obj/ + +# Test output +TestResults/ +*.trx + +# ============================================================ +# Visual Studio / IDE files +# ============================================================ + +.vs/ +.vscode/ +.idea/ + +*.user +*.suo +*.userosscache +*.sln.docstates + +# ============================================================ +# Build / package artifacts +# ============================================================ + +artifacts/ +packages/ + +*.nupkg +*.snupkg + +# ============================================================ +# Local environment files and secrets +# ============================================================ + +.env +.env.* +!.env.example + +# Local secret/config variants +appsettings.Local.json +appsettings.local.json +appsettings.Development.local.json + +# ============================================================ +# API keys / secret files +# ============================================================ + +secrets.json +*.secrets.json +credentials.json +*.credentials.json + +# ============================================================ +# OS-generated files +# ============================================================ + +.DS_Store +Thumbs.db +Desktop.ini + +# ============================================================ +# Temporary/editor files +# ============================================================ + +*.tmp +*.temp +*.bak +*.swp +*~ + +# ============================================================ +# Logs +# ============================================================ + +*.log +logs/ + +# ============================================================ +# Local live-run captures +# ============================================================ + +live-output/ +live-run.txt \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/AgentFrameworkIncidentTriage.slnx b/dotnet-ai/agent-framework-incident-triage/AgentFrameworkIncidentTriage.slnx new file mode 100644 index 0000000..d498e82 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/AgentFrameworkIncidentTriage.slnx @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/dotnet-ai/agent-framework-incident-triage/README.md b/dotnet-ai/agent-framework-incident-triage/README.md new file mode 100644 index 0000000..6f814c0 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/README.md @@ -0,0 +1,149 @@ +# Microsoft Agent Framework in .NET — Evidence-First Incident Triage Agent + +Full tutorial: [Microsoft Agent Framework Tutorial in C#: Build a Tool-Using AI Agent](https://www.dotnet-guide.com/tutorials/dotnet-ai/agent-framework-incident-triage/) + +A minimal .NET 10 console companion sample for the Microsoft Agent Framework tutorial. It builds a tool-using AI agent that triages a sample incident using deterministic operational evidence, read-only function tools, and a shared agent session — with no live model required to build or test it. + +> This sample is an independent educational reference and is not an official Microsoft sample or specification. + +## What this sample demonstrates + +- **`IChatClient` provider boundary** — provider-specific OpenRouter configuration stays in `Program.cs`; everything else works against the neutral `IChatClient` abstraction. +- **`AIAgent`** — agent creation, instructions, and tool registration are centralized in a factory. +- **`AgentSession`** — one session is created for the incident and reused for the same-session follow-up question. +- **Streaming** — model responses are produced with `RunStreamingAsync` and consumed as `AgentResponseUpdate`. +- **Three read-only function tools** — `get_service_health`, `get_recent_deployment`, and `get_runbook`, created with `AIFunctionFactory.Create`. +- **Deterministic evidence** — observed operational facts are loaded from embedded JSON (`services.json`, `deployments.json`). +- **Separate runbook guidance** — reference investigation guidance is loaded independently from evidence (`runbooks.json`). +- **Missing-field behavior** — absent evidence stays unknown; the agent is instructed never to invent missing facts. +- **Correlation vs causation** — the sample shows that temporal correlation does not establish root cause. +- **Deterministic tests without an API key** — the evidence, guidance, and tool layers are fully unit-tested offline. + +## Architecture + +```text +Program.cs → IChatClient → TriageAgentFactory → AIAgent → AgentSession + │ + ┌───────────────┬───────────────┬────────┘ + ▼ ▼ ▼ + ServiceHealthTool DeploymentTool RunbookTool + │ │ │ + EvidenceStore EvidenceStore RunbookStore + │ │ │ + services.json deployments.json runbooks.json +``` + +See [docs/architecture.md](docs/architecture.md) for the full design discussion. + +## Project structure + +```text +AgentFrameworkIncidentTriage.slnx +global.json +verified-environment.json +docs/ +├── architecture.md +├── evidence-first-contract.md +├── sample-run.md +├── tool-risk-ladder.md +└── verified-environment.md +src/ +└── IncidentTriageAgent/ + ├── Program.cs + ├── Agents/TriageAgentFactory.cs + ├── Data/ (embedded JSON evidence + runbooks) + ├── Evidence/ (EvidenceItem, EvidenceStore) + ├── Guidance/ (RunbookItem, RunbookStore) + └── Tools/ (ServiceHealthTool, DeploymentTool, RunbookTool) +tests/ +└── IncidentTriageAgent.Tests/ + ├── EvidenceStoreTests.cs + ├── RunbookStoreTests.cs + └── ToolTests.cs +``` + +## Prerequisites + +- .NET 10 SDK +- An API key is required **only** for live execution. Building and testing require no API key and no internet connection to a model provider. + +## Restore, build, and test + +```bash +dotnet restore AgentFrameworkIncidentTriage.slnx +dotnet build AgentFrameworkIncidentTriage.slnx --configuration Release --no-restore +dotnet test AgentFrameworkIncidentTriage.slnx --configuration Release --no-build +``` + +Expected: build with 0 errors, and **14/14 deterministic tests passing**. + +## Run without an API key + +With no `OPENROUTER_API_KEY` set, the application prints a short notice and exits safely — it makes no live model call: + +```bash +dotnet run --project src/IncidentTriageAgent +``` + +``` +OPENROUTER_API_KEY is not configured. + +The project can build and test without an API key. +Run 'dotnet test' to verify the deterministic evidence and tool layers. + +Set OPENROUTER_API_KEY only when you want to run the live agent. +``` + +## Run the live sample + +The live path uses OpenRouter's OpenAI-compatible endpoint with model `openai/gpt-4o-mini` (override with `OPENROUTER_MODEL`). + +PowerShell: + +```powershell +$env:OPENROUTER_API_KEY="YOUR_KEY" +dotnet run --project src/IncidentTriageAgent +``` + +Bash: + +```bash +export OPENROUTER_API_KEY="YOUR_KEY" +dotnet run --project src/IncidentTriageAgent +``` + +`.env` files are not automatically loaded. Replace `YOUR_KEY` with a real key only when you intend to make a live model request — never commit it. + +## Evidence-first behavior + +The agent retrieves operational evidence (for example `E-101`, `E-201`) and separate runbook guidance (`RB-CHECKOUT-03`), and distinguishes what is *known* from what is *inferred*. Missing fields remain unknown, guidance is not evidence, and correlation is not treated as proof of causation. This is the repository-specific **Evidence-First Agent Contract** — see [docs/evidence-first-contract.md](docs/evidence-first-contract.md). + +## Tool risk boundary + +All three tools in this sample are **read-only** Level 0 / Level 1 tools: they query deterministic stores and perform no operational write. The agent can recommend investigation steps but cannot restart services, roll back deployments, or modify configuration. See [docs/tool-risk-ladder.md](docs/tool-risk-ladder.md). + +## Verified sample run + +A controlled live run was captured while the deterministic tests were passing — see [docs/sample-run.md](docs/sample-run.md). Note that LLM wording is probabilistic and may vary between runs; the deterministic layer is stable and fully tested. + +## Important boundary / deliberate exclusions + +This first Agent Framework sample deliberately does **not** implement: MCP, RAG, vector search, persistent long-term memory, multi-agent workflows, automatic remediation, write-capable tools, service restart/rollback tools, configuration mutation, human approval workflows, durable orchestration, production hosting, A2A, or AG-UI. The tutorial focuses on provider abstraction, tool calling, session state, deterministic evidence, and evidence-aware reasoning. See [docs/architecture.md](docs/architecture.md) and the full tutorial for the follow-on topics. + +## Verification table + +| Item | Value | +| --- | --- | +| Target framework | net10.0 | +| SDK baseline | 10.0.401 | +| Microsoft.Agents.AI.OpenAI | 1.22.0 | +| Deterministic tests | 14/14 passing | +| Live provider used for verification | OpenRouter | +| Live model used for verification | openai/gpt-4o-mini | +| Live run verification date | 26 September 2026 | + +Environment details: [docs/verified-environment.md](docs/verified-environment.md) · [verified-environment.json](verified-environment.json) + +## License + +Repository-owned sample code is available under the [MIT License](../../LICENSE). \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/docs/architecture.md b/dotnet-ai/agent-framework-incident-triage/docs/architecture.md new file mode 100644 index 0000000..cfd351c --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/docs/architecture.md @@ -0,0 +1,805 @@ +# Evidence-First Incident Triage Agent — Architecture + +This document describes the architecture of the +**Evidence-First Incident Triage Agent** reference implementation. + +The sample demonstrates how to combine: + +- .NET 10; +- Microsoft Agent Framework; +- `IChatClient`; +- `AIAgent`; +- `AgentSession`; +- deterministic operational evidence; +- read-only function tools; +- reference runbook guidance; +- streaming model responses. + +The central design goal is simple: + +> Keep deterministic facts separate from probabilistic model reasoning. + +--- + +# Architecture at a Glance + +```text + ┌──────────────────────────┐ + │ Program.cs │ + │ │ + │ Provider configuration │ + │ OpenRouter endpoint │ + │ model selection │ + └────────────┬─────────────┘ + │ + ▼ + ┌───────────────┐ + │ IChatClient │ + └───────┬───────┘ + │ + ▼ + ┌──────────────────────────┐ + │ TriageAgentFactory │ + │ │ + │ Agent instructions │ + │ Tool registration │ + └────────────┬─────────────┘ + │ + ▼ + ┌────────────┐ + │ AIAgent │ + └─────┬──────┘ + │ + ▼ + ┌──────────────┐ + │ AgentSession │ + └──────┬───────┘ + │ + ┌───────────────┼────────────────┐ + │ │ │ + ▼ ▼ ▼ + ┌──────────────────┐ ┌──────────────┐ ┌────────────────┐ + │ ServiceHealthTool│ │DeploymentTool│ │ RunbookTool │ + └─────────┬────────┘ └──────┬───────┘ └───────┬────────┘ + │ │ │ + └──────────┬──────┘ │ + ▼ ▼ + ┌─────────────────┐ ┌─────────────────┐ + │ EvidenceStore │ │ RunbookStore │ + └────────┬────────┘ └────────┬────────┘ + │ │ + ┌─────────┴─────────┐ │ + ▼ ▼ ▼ + services.json deployments.json runbooks.json +``` + +--- + +# Design Principle 1 — Provider-specific code stays at the edge + +The application currently uses OpenRouter through an +OpenAI-compatible endpoint. + +That configuration exists only in: + +```text +Program.cs +``` + +The provider-specific path is: + +```text +OpenRouter + ↓ +OpenAI-compatible endpoint + ↓ +OpenAIClient + ↓ +AsIChatClient() + ↓ +IChatClient +``` + +Everything after `IChatClient` is independent of the OpenRouter +configuration. + +This means the core agent factory does not need to know: + +- the API endpoint; +- the API key; +- the model identifier; +- how the provider SDK was configured. + +The architectural boundary is: + +```text +provider-specific code + ↓ + IChatClient + ↓ +provider-neutral agent code +``` + +--- + +# Design Principle 2 — Agent creation is centralized + +The agent is constructed by: + +```text +Agents/TriageAgentFactory.cs +``` + +The factory receives: + +```text +IChatClient +EvidenceStore +RunbookStore +``` + +and creates the configured: + +```text +AIAgent +``` + +The factory is responsible for: + +- agent name; +- agent instructions; +- evidence-first rules; +- tool registration. + +It is not responsible for: + +- API keys; +- provider endpoints; +- model selection; +- reading JSON directly; +- console interaction. + +This keeps agent policy separate from application startup code. + +--- + +# Design Principle 3 — Evidence exists independently of the model + +Observed operational facts are stored in: + +```text +Data/services.json +Data/deployments.json +``` + +They are loaded through: + +```text +EvidenceStore +``` + +For example: + +```text +E-101 +service=checkout-api +status=degraded +error_rate=18.7% +database_pool_utilization=96% +``` + +exists whether or not an LLM is available. + +The model does not create this evidence. + +It only receives the evidence through controlled tools. + +This allows the deterministic layer to be tested without: + +- an API key; +- an Internet connection; +- a model request; +- probabilistic output. + +--- + +# Design Principle 4 — Missing evidence remains missing + +The evidence model does not invent values for fields that are absent. + +For example: + +```text +E-201 +deployment=deploy-1842 +previous_deployment=deploy-1839 +``` + +contains the identity of the previous deployment. + +It does not contain: + +```text +previous_deployment_completed_at +``` + +Therefore: + +```text +completion time of deploy-1839 = UNKNOWN +``` + +The deterministic layer returns `null` for a missing fact. + +The agent instructions then reinforce the same boundary: + +> Never invent or fill in a missing field. + +This is one of the core behaviors tested by the sample. + +--- + +# Design Principle 5 — Evidence and guidance use separate stores + +The project deliberately separates: + +```text +EvidenceStore +``` + +from: + +```text +RunbookStore +``` + +because they represent different information classes. + +## EvidenceStore + +Contains observed records such as: + +```text +service health +deployment history +timestamps +error rates +resource utilization +``` + +Example IDs: + +```text +E-101 +E-201 +``` + +## RunbookStore + +Contains reference guidance such as: + +```text +what to inspect +which metrics to compare +which traces to review +``` + +Example ID: + +```text +RB-CHECKOUT-03 +``` + +This distinction prevents architecture like: + +```text +runbook recommendation + ↓ +treated as observed fact +``` + +The desired relationship is: + +```text +evidence + ↓ +what is known + +guidance + ↓ +what should be checked next +``` + +--- + +# Design Principle 6 — Tools are thin adapters + +The three tools are: + +```text +ServiceHealthTool +DeploymentTool +RunbookTool +``` + +Their job is intentionally small. + +They: + +1. receive model-selected arguments; +2. query a deterministic store; +3. return structured text; +4. perform no operational write. + +They do not contain model reasoning. + +For example: + +```text +ServiceHealthTool + ↓ +EvidenceStore + ↓ +E-101 +``` + +The model decides how to interpret E-101. + +The tool decides only which record is returned. + +--- + +# Tool Boundary + +The current reference implementation exposes: + +| Tool | Information class | Risk level | Writes state? | +|---|---|---:|---| +| `get_runbook` | Reference guidance | 0 | No | +| `get_service_health` | Observed evidence | 1 | No | +| `get_recent_deployment` | Observed evidence | 1 | No | + +The risk levels correspond to the +[AI Agent Tool Risk Ladder](tool-risk-ladder.md). + +No tool in this sample can: + +```text +restart a service +rollback a deployment +modify configuration +deploy code +delete data +``` + +The agent can recommend investigation steps. + +It cannot perform remediation. + +--- + +# Agent Execution Flow + +A live execution follows this sequence: + +```text +User incident prompt + │ + ▼ + AIAgent + │ + │ decides which information is needed + │ + ├──────────────► get_service_health + │ │ + │ ▼ + │ E-101 + │ + ├──────────────► get_recent_deployment + │ │ + │ ▼ + │ E-201 + │ + └──────────────► get_runbook + │ + ▼ + RB-CHECKOUT-03 + │ + ▼ + model interpretation + │ + ▼ + streamed assessment +``` + +The model is allowed to interpret the retrieved information. + +It is not allowed to redefine the underlying evidence. + +--- + +# Session Flow + +The application creates: + +```text +AgentSession +``` + +once for the incident. + +The same session is then used for: + +```text +initial incident assessment + ↓ +same-session follow-up +``` + +The reference scenario asks: + +```text +Can I truthfully write: + +"deploy-1842 caused the checkout-api outage"? +``` + +Because the previous evidence is already part of the session context, +the follow-up can be answered without reconstructing the entire incident +prompt. + +The verified run rejected the causal statement because the evidence +showed timing correlation but did not establish root cause. + +--- + +# Streaming + +Live responses are produced through: + +```text +RunStreamingAsync(...) +``` + +and consumed as: + +```text +AgentResponseUpdate +``` + +The console therefore receives model output progressively rather than +waiting for one complete response object. + +Streaming affects delivery. + +It does not change the evidence model or evidence-first rules. + +--- + +# Deterministic vs Probabilistic Components + +The architecture intentionally contains both. + +## Deterministic + +```text +services.json +deployments.json +runbooks.json +EvidenceStore +RunbookStore +tool lookup behavior +missing-field behavior +tool construction +``` + +These components are tested with xUnit. + +At the verified baseline: + +```text +14 tests +14 passed +0 failed +``` + +## Probabilistic + +```text +tool selection by the model +wording of the response +reasoning phrasing +ordering of explanations +whether identical information is requested more than once +``` + +These behaviors may vary between model runs. + +That is why the repository does not make live LLM output a required CI +assertion. + +--- + +# Test Boundary + +The normal test path is: + +```text +dotnet test +``` + +It does not require: + +```text +OPENROUTER_API_KEY +``` + +This is deliberate. + +CI should be able to verify: + +```text +evidence +stores +tools +missing fields +function construction +``` + +without paying for or depending on a model provider. + +The live-model layer is verified separately. + +--- + +# Secret Boundary + +The repository contains no real API key. + +For a live run the application reads: + +```text +OPENROUTER_API_KEY +``` + +from the environment. + +Optional model override: + +```text +OPENROUTER_MODEL +``` + +The default model identifier used during verification was: + +```text +openai/gpt-4o-mini +``` + +Secrets belong outside: + +```text +source files +JSON sample data +README examples +tests +committed configuration +``` + +--- + +# Repository Layers + +The application is organized into the following responsibilities: + +```text +src/IncidentTriageAgent/ + +├── Agents/ +│ └── TriageAgentFactory.cs +│ +│ Agent policy and tool composition +│ +├── Evidence/ +│ ├── EvidenceItem.cs +│ └── EvidenceStore.cs +│ +│ Deterministic observed facts +│ +├── Guidance/ +│ ├── RunbookItem.cs +│ └── RunbookStore.cs +│ +│ Deterministic reference guidance +│ +├── Tools/ +│ ├── ServiceHealthTool.cs +│ ├── DeploymentTool.cs +│ └── RunbookTool.cs +│ +│ Read-only model-callable adapters +│ +├── Data/ +│ ├── services.json +│ ├── deployments.json +│ └── runbooks.json +│ +│ Reproducible sample data +│ +└── Program.cs + + Provider configuration and application execution +``` + +Tests live separately: + +```text +tests/IncidentTriageAgent.Tests/ +``` + +Reference documentation lives under: + +```text +docs/ +``` + +--- + +# Why the sample does not call production monitoring APIs + +The sample deliberately uses deterministic JSON instead of: + +```text +Azure Monitor +Application Insights +Datadog +Grafana +Kubernetes +production databases +``` + +This keeps the tutorial reproducible. + +Every reader starts with the same incident: + +```text +14:02 deploy-1842 completed +14:09 checkout-api error spike began +14:12 checkout-api observed degraded + error rate = 18.7% + database pool = 96% +``` + +Only the model interpretation is probabilistic. + +The evidence remains fixed. + +A production implementation could later replace the stores with adapters +for live telemetry systems without changing the core evidence-first +design. + +--- + +# What This Architecture Deliberately Excludes + +The first reference sample does not implement: + +```text +MCP +RAG +vector search +persistent long-term memory +multi-agent workflows +automatic remediation +write-capable tools +human approval workflows +durable orchestration +production hosting +A2A +AG-UI +``` + +These are intentionally separate topics. + +The goal of Agent Framework #1 is to establish: + +```text +provider abstraction + + +tool calling + + +session state + + +deterministic evidence + + +evidence-aware reasoning +``` + +before introducing more advanced capabilities. + +--- + +# Extension Points + +The architecture can later evolve without discarding the core pattern. + +For example: + +```text +EvidenceStore + ↓ +replace with +Azure Monitor adapter +``` + +or: + +```text +RunbookStore + ↓ +replace with +MCP-backed knowledge tool +``` + +or: + +```text +read-only agent + ↓ +add +approval-controlled operational tool +``` + +The important boundary remains: + +```text +source of truth + ↓ +controlled tool + ↓ +agent interpretation +``` + +--- + +# Related Reference Documents + +See: + +```text +evidence-first-contract.md +``` + +for the reasoning contract. + +See: + +```text +tool-risk-ladder.md +``` + +for the tool-capability classification. + +See: + +```text +verified-environment.md +``` + +for the exact environment in which the implementation was tested. + +--- + +## Status + +**Implementation:** verified reference sample + +**Target framework:** .NET 10 + +**Agent framework package:** Microsoft.Agents.AI.OpenAI 1.22.0 + +**Automated tests:** 14 passing + +**Live Agent Framework run:** verified + +**Last verified:** 26 September 2026 \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/docs/evidence-first-contract.md b/dotnet-ai/agent-framework-incident-triage/docs/evidence-first-contract.md new file mode 100644 index 0000000..3118dc1 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/docs/evidence-first-contract.md @@ -0,0 +1,651 @@ +# Evidence-First Agent Contract + +The **Evidence-First Agent Contract** is a practical design pattern for +tool-using AI agents that need to reason about operational facts without +silently inventing missing information. + +It was created for the +**Evidence-First Incident Triage Agent** reference sample from +dotnet-guide.com. + +This is **not a Microsoft Agent Framework specification** and is not a +replacement for framework documentation. It is an application-level +design contract that can be implemented with Microsoft Agent Framework +or adapted to other tool-using agent systems. + +--- + +# Why this contract exists + +Giving an AI agent access to tools does not automatically make its +answers evidence-grounded. + +An agent can: + +1. retrieve correct data from a tool; +2. accurately repeat some of that data; +3. and still introduce unsupported facts while connecting the pieces. + +For example, suppose a deployment tool returns: + +```text +E-201 +deployment=deploy-1842 +completed_at=2026-09-24T14:02:00Z +previous_deployment=deploy-1839 +``` + +The evidence establishes that: + +- `deploy-1842` completed at 14:02 UTC; +- `deploy-1839` was the previous deployment. + +It does **not** establish when `deploy-1839` occurred. + +An evidence-first agent should therefore say: + +> The completion time of deploy-1839 is unknown from the available evidence. + +It should not fill the missing field with a plausible time. + +This distinction is the foundation of the contract. + +--- + +# The Five Evidence-First Rules + +## Rule 1 — Retrieve before asserting + +Claims about current system state should originate from an appropriate +evidence source. + +If the agent needs to state: + +> checkout-api has an 18.7% error rate + +that value should come from a retrieved evidence record rather than from +model memory, assumption, or prompt context that was never verified. + +In the reference sample: + +```text +get_service_health + ↓ +E-101 + ↓ +error_rate=18.7% +``` + +The model may explain the value, but the value itself belongs to the +evidence layer. + +--- + +## Rule 2 — Never fill a missing evidence field + +Missing information must remain missing. + +If the evidence contains: + +```text +previous_deployment=deploy-1839 +``` + +but no timestamp, the agent may state: + +> deploy-1839 was the previous deployment. + +It must not state: + +> deploy-1839 occurred shortly before deploy-1842. + +unless another evidence source actually establishes that timing. + +A useful implementation rule is: + +```text +missing field + ↓ +UNKNOWN +``` + +not: + +```text +missing field + ↓ +model-generated completion +``` + +This applies even when the inferred value would appear highly plausible. + +--- + +## Rule 3 — Separate evidence from inference + +Observed facts and model interpretation should not be presented as if +they have the same evidentiary status. + +Consider: + +```text +E-101 +database_pool_utilization=96% +``` + +The following is an observed fact: + +> Database pool utilization was 96%. + +The following is an inference: + +> High pool utilization may indicate resource pressure. + +The following would be an unsupported causal claim: + +> Database pool exhaustion caused the outage. + +An evidence-first response should make these levels distinguishable. + +A practical response structure is: + +```text +OBSERVED EVIDENCE +REASONABLE INFERENCES +RUNBOOK GUIDANCE +UNKNOWNS +UNPROVEN CLAIMS +``` + +--- + +## Rule 4 — Correlation is not causation + +Timing can make an event worth investigating without proving that the +event caused the incident. + +The sample contains: + +```text +deploy-1842 completed 14:02 UTC +checkout-api error spike 14:09 UTC +``` + +This establishes a seven-minute temporal relationship. + +It supports the statement: + +> deploy-1842 is a reasonable investigation target. + +It does not by itself support: + +> deploy-1842 caused the outage. + +Additional evidence could include: + +- traces showing failures introduced by changed code; +- connection counts before and after deployment; +- configuration differences; +- rollback results; +- comparison with the previous known-good release; +- dependency or database telemetry; +- reproducible behavior tied to the deployed change. + +An evidence-first agent should preserve this distinction explicitly. + +--- + +## Rule 5 — Guidance is not evidence + +Reference material can tell an agent what to investigate without proving +that a particular condition exists. + +For example, the sample runbook contains: + +```text +RB-CHECKOUT-03 + +Inspect database connection-pool utilization. +Review traces for timeout or connection-acquisition failures. +Compare configuration with the previous known-good release. +``` + +Those instructions are **guidance**. + +They do not establish that: + +- a connection pool was exhausted; +- a timeout occurred; +- the configuration changed incorrectly. + +The runbook helps determine the next investigation step. + +The evidence layer determines what is currently known. + +--- + +# Evidence Categories + +The reference sample intentionally separates information into different +categories. + +## Observed evidence + +Examples: + +```text +E-101 +E-201 +``` + +Observed evidence represents deterministic operational records such as: + +- service health; +- error rates; +- deployment identifiers; +- timestamps; +- resource utilization. + +These records should be traceable to their source. + +--- + +## Reference guidance + +Example: + +```text +RB-CHECKOUT-03 +``` + +Reference guidance represents material such as: + +- runbooks; +- troubleshooting procedures; +- operational checklists; +- recommended diagnostic steps. + +Guidance can help determine what to inspect next. + +It should not be treated as proof that a condition occurred. + +--- + +## Inference + +Inference is the agent's interpretation of retrieved information. + +For example: + +> 96% database pool utilization suggests resource pressure. + +This may be useful and reasonable, but it is still an interpretation. + +Inference should be presented as such. + +--- + +## Unknown + +An unknown is information that the available evidence does not contain. + +For example: + +```text +previous_deployment=deploy-1839 +``` + +does not provide: + +```text +previous_deployment_completed_at +``` + +Therefore the completion time is: + +```text +UNKNOWN +``` + +Unknowns are an expected and useful output of an evidence-first system. + +--- + +## Unproven claim + +An unproven claim is a conclusion that available evidence does not +establish. + +For example: + +> deploy-1842 caused the checkout-api outage. + +The reference evidence shows temporal correlation but does not establish +that causal relationship. + +The appropriate output is therefore: + +```text +UNPROVEN +``` + +rather than a confident yes. + +--- + +# Evidence IDs + +Where practical, evidence-first systems should attach stable identifiers +to retrieved records. + +The sample uses identifiers such as: + +```text +E-101 +E-201 +RB-CHECKOUT-03 +``` + +This provides several benefits: + +- users can see where a factual statement came from; +- tests can verify stable evidence; +- debugging becomes easier; +- model output can be compared with the retrieved source; +- downstream systems can trace claims back to records. + +An evidence ID does not make the underlying information correct by +itself. It provides traceability. + +--- + +# Deterministic Evidence vs Probabilistic Interpretation + +One of the most important boundaries in an agent system is: + +```text +deterministic evidence + ↓ +probabilistic interpretation +``` + +The sample's evidence layer is deterministic. + +For example: + +```text +E-101 +error_rate=18.7% +database_pool_utilization=96% +``` + +Automated tests can verify those values exactly. + +The model's interpretation may vary between runs: + +> 96% utilization may indicate resource pressure. + +Another run might phrase this differently. + +Therefore: + +- evidence retrieval should be tested deterministically; +- tool contracts should be tested deterministically; +- missing-field behavior should be tested deterministically; +- model-generated wording should not normally be treated as a stable CI assertion. + +A successful live model run demonstrates observed behavior. + +It does not guarantee that every future model invocation will produce +identical reasoning or wording. + +--- + +# Read-Only First + +The reference implementation intentionally gives the first agent only +read-only tools. + +It can: + +```text +read service health +read deployment information +read runbook guidance +``` + +It cannot: + +```text +restart a service +roll back a deployment +modify configuration +delete data +deploy code +``` + +This separates: + +```text +investigation +``` + +from: + +```text +action +``` + +An agent that can recommend an action is not automatically authorized to +perform that action. + +Write-capable and destructive tools require additional controls such as +explicit approval, authorization, validation, audit logging, and +appropriate operational safeguards. + +These controls are outside the scope of the first reference sample. + +--- + +# Example Evidence-First Assessment + +Given: + +```text +E-101 +service=checkout-api +status=degraded +error_rate=18.7% +error_spike_started=2026-09-24T14:09:00Z +database_pool_utilization=96% +``` + +and: + +```text +E-201 +deployment=deploy-1842 +completed_at=2026-09-24T14:02:00Z +previous_deployment=deploy-1839 +``` + +an evidence-first assessment could state: + +### Observed evidence + +- E-101 shows that checkout-api is degraded. +- E-101 reports an error rate of 18.7%. +- E-101 reports database pool utilization of 96%. +- E-201 shows that deploy-1842 completed at 14:02 UTC. +- E-101 shows that the error spike began at 14:09 UTC. + +### Reasonable inference + +The timing makes deploy-1842 a reasonable investigation target. + +The 96% pool utilization makes resource pressure another reasonable +investigation lead. + +### Unknown + +The completion time of deploy-1839 is not available. + +### Unproven + +The evidence does not establish that deploy-1842 caused the outage. + +The evidence does not establish that database pool pressure caused the +outage. + +This is the behavior the contract is designed to encourage. + +--- + +# Anti-Patterns + +An agent violates the Evidence-First Agent Contract when it does things +such as: + +## Inventing missing facts + +```text +Evidence: +previous_deployment=deploy-1839 + +Agent: +deploy-1839 completed shortly before deploy-1842 +``` + +No timing evidence exists. + +--- + +## Turning guidance into fact + +```text +Runbook: +Inspect connection-pool exhaustion. + +Agent: +The connection pool was exhausted. +``` + +The runbook is a diagnostic instruction, not incident evidence. + +--- + +## Turning correlation into causation + +```text +Deployment completed: 14:02 +Error spike began: 14:09 + +Agent: +The deployment caused the outage. +``` + +The chronology supports investigation, not proof of causation. + +--- + +## Hiding uncertainty + +```text +Evidence is insufficient. + +Agent: +The most likely root cause is definitely... +``` + +An evidence-first agent should expose important uncertainty rather than +mask it with confident language. + +--- + +## Claiming actions that were never executed + +```text +Available tools: +read-only + +Agent: +I rolled back the deployment. +``` + +The model must not claim an operational action that no tool performed. + +--- + +# Implementation Checklist + +An implementation following this pattern should aim to answer yes to +these questions: + +- Does every operational fact have a retrievable source? +- Can missing fields remain explicitly unknown? +- Are evidence and guidance represented separately? +- Are inference and fact distinguishable in the response? +- Are evidence records traceable with stable IDs? +- Are deterministic components testable without an LLM? +- Can the system build and test without production secrets? +- Are read-only and write-capable tools clearly separated? +- Does the system avoid claiming actions that were not executed? +- Are causal conclusions withheld when evidence establishes only + correlation? + +--- + +# Scope + +The Evidence-First Agent Contract is intentionally narrow. + +It does not solve: + +- factual correctness of the underlying data source; +- authentication or authorization; +- malicious or compromised tools; +- prompt injection; +- model evaluation; +- long-term memory; +- multi-agent coordination; +- human approval workflows; +- production observability; +- regulatory or compliance requirements. + +Those concerns require additional controls. + +The contract addresses one specific problem: + +> **How should a tool-using agent distinguish retrieved facts, +> interpretation, guidance, missing information, and unproven claims?** + +--- + +# Reference Implementation + +The accompanying .NET sample demonstrates this contract using: + +- .NET 10; +- Microsoft Agent Framework; +- `IChatClient`; +- `AIAgent`; +- `AgentSession`; +- read-only function tools; +- deterministic JSON evidence; +- deterministic automated tests; +- a live model only for the probabilistic reasoning layer. + +See `verified-environment.md` for the exact environment in which the +reference implementation was built and tested. + +--- + +## Status + +**Pattern status:** tutorial/reference design pattern + +**Specification status:** not an official Microsoft specification + +**Reference implementation:** Evidence-First Incident Triage Agent + +**Last verified:** 26 September 2026 \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/docs/sample-run.md b/dotnet-ai/agent-framework-incident-triage/docs/sample-run.md new file mode 100644 index 0000000..a3bd6db --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/docs/sample-run.md @@ -0,0 +1,1170 @@ +\# Evidence-First Incident Triage Agent — Verified Sample Run + + + +This document records one successful live execution of the + +\*\*Evidence-First Incident Triage Agent\*\* reference sample. + + + +The run demonstrates: + + + +\- Microsoft Agent Framework agent creation; + +\- read-only function-tool use; + +\- deterministic operational evidence retrieval; + +\- runbook guidance retrieval; + +\- `AgentSession` reuse; + +\- streaming output; + +\- missing-field handling; + +\- evidence-versus-inference separation; + +\- correlation-versus-causation handling. + + + +> This is a captured example, not a golden-output test. + +> Model-generated wording is probabilistic and may differ between runs. + + + +\--- + + + +\# Verification Date + + + +\*\*26 September 2026\*\* + + + +\--- + + + +\# Verified Environment + + + +The captured run used: + + + +```text + +Target framework: net10.0 + +.NET SDK: 10.0.401 + +Agent package: Microsoft.Agents.AI.OpenAI 1.22.0 + +Provider: OpenRouter + +Model: openai/gpt-4o-mini + +``` + + + +See: + + + +```text + +verified-environment.md + +``` + + + +for the complete verification record. + + + +\--- + + + +\# Scenario + + + +The sample simulates an incident involving: + + + +```text + +checkout-api + +``` + + + +The agent can retrieve information from three read-only tools: + + + +```text + +get\_service\_health + +get\_recent\_deployment + +get\_runbook + +``` + + + +These tools expose deterministic evidence and reference guidance stored + +with the repository. + + + +\--- + + + +\# Deterministic Evidence Available to the Agent + + + +\## Service-health evidence + + + +```text + +E-101 + +service=checkout-api + +observed\_at=2026-09-24T14:12:00Z + +status=degraded + +error\_rate=18.7% + +error\_spike\_started=2026-09-24T14:09:00Z + +database\_pool\_utilization=96% + +``` + + + +\## Deployment evidence + + + +```text + +E-201 + +service=checkout-api + +deployment=deploy-1842 + +deployment\_status=succeeded + +completed\_at=2026-09-24T14:02:00Z + +previous\_deployment=deploy-1839 + +change\_summary=pricing-rule refresh and structured logging update + +rollback\_performed=false + +``` + + + +Notice that the evidence does \*\*not\*\* contain: + + + +```text + +previous\_deployment\_completed\_at + +``` + + + +for: + + + +```text + +deploy-1839 + +``` + + + +That omission is intentional. + + + +The agent should leave the missing timestamp unknown rather than invent + +one. + + + +\--- + + + +\# Reference Guidance + + + +The runbook tool exposes: + + + +```text + +RB-CHECKOUT-03 + +``` + + + +with investigation guidance including: + + + +```text + +Compare error-rate onset with recent deployments. + +Inspect database connection-pool utilization. + +Check database connection counts before and after deployment. + +Review traces for timeout or connection-acquisition failures. + +Compare configuration with the previous known-good release. + +``` + + + +This is \*\*reference guidance\*\*, not observed incident evidence. + + + +\--- + + + +\# Initial Prompt + + + +The application sends the following incident request: + + + +```text + +Investigate the checkout-api incident. + + + +Using only the available operational evidence and reference guidance: + + + +1\. state what the observed evidence establishes, + +2\. identify reasonable inferences, + +3\. list the relevant runbook guidance, + +4\. identify unknown information, + +5\. and state which root-cause claims remain unproven. + + + +Do not invent missing facts. + +Do not treat correlation as proof of causation. + +``` + + + +\--- + + + +\# Captured Live Output + + + +The following output was observed during the verified live run: + + + +```text + +============================================ + + Evidence-First Incident Triage Agent + +============================================ + + + +Model: openai/gpt-4o-mini + +Agent: ChatClientAgent + +Session: ChatClientAgentSession + + + +=== INCIDENT ASSESSMENT === + + + +\### OBSERVED EVIDENCE + +1\. The \*\*checkout-api\*\* service has a status of \*\*degraded\*\* with an \*\*error rate of 18.7%\*\* observed on September 24, 2026, at 14:12 UTC (Evidence ID: E-101). + +2\. A \*\*deployment labeled deploy-1842\*\* was completed successfully on September 24, 2026, at 14:02 UTC, with a previous deployment labeled \*\*deploy-1839\*\* (Evidence ID: E-201). + +3\. The \*\*error spike\*\* began at \*\*14:09 UTC\*\*, which is after the completion of the last deployment (Evidence ID: E-101). + +4\. The \*\*database pool utilization\*\* was at \*\*96%\*\* (Evidence ID: E-101). + + + +\### REASONABLE INFERENCES + +1\. The error rate increased after the most recent deployment (deploy-1842), suggesting a potential connection that requires further investigation. + +2\. The high database pool utilization (96%) may be contributing to the degraded service status and high error rate. + +3\. The spike in errors beginning shortly after the latest deployment may indicate issues that arose as a result of changes made in that deployment. + + + +\### RUNBOOK GUIDANCE + +The relevant runbook guidance (RB-CHECKOUT-03) suggests the following checks: + +1\. Compare the onset of the error rate with recent deployments. + +2\. Inspect database connection-pool utilization. + +3\. Check database connection counts before and after the deployment. + +4\. Review traces for timeout or connection-acquisition failures. + +5\. Compare the current configuration with the previous known-good release. + + + +It is noted that the presence of these checks does not establish root cause. + + + +\### UNKNOWNS + +1\. The timestamp of the previous deployment (deploy-1839) is UNKNOWN. + +2\. The specific reasons for the increase in error rate remain UNKNOWN, as no root cause has been identified. + +3\. The connection counts for the database before and after the deployment are UNKNOWN. + + + +\### UNPROVEN CLAIMS + +1\. There is no established root cause for the degraded status and increased error rate, as the current evidence does not support a definitive conclusion regarding causation between the deployment and the errors. + +2\. It remains unproven whether the high database pool utilization is solely responsible for the degraded status. + +``` + + + +\--- + + + +\# What the Initial Run Demonstrated + + + +Several behaviors are important. + + + +\## 1. Evidence IDs were preserved + + + +The generated assessment referred to: + + + +```text + +E-101 + +E-201 + +RB-CHECKOUT-03 + +``` + + + +This makes factual claims easier to trace back to deterministic sources. + + + +\--- + + + +\## 2. The missing deployment timestamp stayed unknown + + + +The agent stated: + + + +```text + +The timestamp of the previous deployment (deploy-1839) is UNKNOWN. + +``` + + + +This is important because the evidence contains the identifier: + + + +```text + +previous\_deployment=deploy-1839 + +``` + + + +but contains no corresponding timestamp. + + + +The model did not fill that missing field with a plausible value. + + + +That behavior demonstrates Rule 2 of the + +\*\*Evidence-First Agent Contract\*\*: + + + +> Never fill a missing evidence field. + + + +\--- + + + +\## 3. Runbook guidance remained guidance + + + +The agent retrieved: + + + +```text + +RB-CHECKOUT-03 + +``` + + + +but did not claim that the runbook itself proved a root cause. + + + +It explicitly stated: + + + +```text + +the presence of these checks does not establish root cause + +``` + + + +This demonstrates the intended distinction: + + + +```text + +guidance + + ≠ + +evidence + +``` + + + +\--- + + + +\## 4. High database utilization was treated as an inference + + + +The observed fact was: + + + +```text + +database\_pool\_utilization=96% + +``` + + + +The generated interpretation used qualified wording: + + + +```text + +may be contributing + +``` + + + +rather than claiming that database pool pressure was definitively the + +root cause. + + + +\--- + + + +\# A Subtle Probabilistic Reasoning Issue + + + +One generated sentence was: + + + +```text + +The spike in errors beginning shortly after the latest deployment may + +indicate issues that arose as a result of changes made in that deployment. + +``` + + + +This is still qualified with: + + + +```text + +may + +``` + + + +and the later `UNPROVEN CLAIMS` section explicitly rejects a definitive + +causal conclusion. + + + +However, this sentence shows why prompt instructions alone should not be + +treated as a mathematical guarantee. + + + +A model may occasionally phrase an inference more strongly than an + +engineer would prefer. + + + +The deterministic evidence layer remains unchanged: + + + +```text + +deployment completed: 14:02 UTC + +error spike began: 14:09 UTC + +``` + + + +Those facts establish chronology. + + + +They do not establish causation. + + + +This is one reason the repository separates: + + + +```text + +deterministic evidence + + ↓ + +probabilistic interpretation + +``` + + + +and does not make exact live-model wording a normal CI assertion. + + + +\--- + + + +\# Same-Session Follow-Up + + + +After the initial assessment, the application reused the same: + + + +```text + +AgentSession + +``` + + + +and sent this follow-up: + + + +```text + +Can I truthfully write in the incident report: + + + +"deploy-1842 caused the checkout-api outage"? + + + +Give a concise evidence-based answer. + +``` + + + +\--- + + + +\# Captured Same-Session Response + + + +The verified live response was: + + + +```text + +No, you cannot truthfully write that "deploy-1842 caused the checkout-api outage." The evidence indicates a correlation between the deployment and the increase in error rate, but correlation does not establish causation. The specific root cause of the outage has not been confirmed, and the evidence does not definitively support the claim that the deployment was the cause of the outage. + +``` + + + +\--- + + + +\# Why the Follow-Up Matters + + + +The evidence establishes this sequence: + + + +```text + +14:02 UTC + +deploy-1842 completed + + + + ↓ 7 minutes + + + +14:09 UTC + +checkout-api error spike began + +``` + + + +The chronology makes the deployment a reasonable investigation target. + + + +It does not prove: + + + +```text + +deploy-1842 + + ↓ + +caused + + ↓ + +checkout-api outage + +``` + + + +The same-session response correctly preserved the distinction: + + + +```text + +correlation + + ≠ + +causation + +``` + + + +This is one of the central behaviors of the reference sample. + + + +\--- + + + +\# Session Reuse + + + +The application creates one: + + + +```text + +AgentSession + +``` + + + +and reuses it for: + + + +```text + +incident assessment + + ↓ + +follow-up question + +``` + + + +The second prompt does not restate: + + + +```text + +E-101 + +E-201 + +RB-CHECKOUT-03 + +``` + + + +yet the agent can answer in the context of the incident already discussed + +within the session. + + + +This demonstrates conversational session continuity without adding a + +separate persistent-memory system. + + + +\--- + + + +\# What This Run Does Not Prove + + + +A successful live run does not prove that: + + + +\- every future invocation will produce identical wording; + +\- every model will select tools in exactly the same order; + +\- every model will make equally cautious inferences; + +\- prompt instructions eliminate hallucination; + +\- tool access alone guarantees grounded reasoning; + +\- the sample is ready for unsupervised production remediation. + + + +The captured run demonstrates observed behavior in the verified + +environment. + + + +The deterministic components are covered separately by automated tests. + + + +\--- + + + +\# Automated Verification Boundary + + + +The repository uses automated tests for behavior that should be + +deterministic. + + + +At the time of this run: + + + +```text + +total: 14 + +failed: 0 + +succeeded: 14 + +skipped: 0 + +``` + + + +The automated suite verifies areas such as: + + + +```text + +evidence loading + +runbook loading + +service lookup + +missing-field behavior + +tool output + +unknown-service behavior + +AIFunction construction + +``` + + + +The test suite requires no model-provider API key. + + + +Live model reasoning is verified separately. + + + +\--- + + + +\# Evidence-First Interpretation of the Incident + + + +The safest summary of the sample incident is: + + + +```text + +KNOWN + +\----- + +checkout-api was degraded + +error rate was 18.7% + +database pool utilization was 96% + +deploy-1842 completed at 14:02 UTC + +the error spike began at 14:09 UTC + +deploy-1839 was the previous deployment + + + +REASONABLE TO INVESTIGATE + +\------------------------- + +deploy-1842 + +database connection-pool pressure + + + +UNKNOWN + +\------- + +deploy-1839 completion time + +database connection counts before and after deployment + +trace evidence for timeout or connection-acquisition failures + +confirmed root cause + + + +UNPROVEN + +\-------- + +deploy-1842 caused the outage + +database pool pressure caused the outage + +``` + + + +That separation is the core behavior this sample is designed to teach. + + + +\--- + + + +\# Related Documents + + + +See: + + + +```text + +evidence-first-contract.md + +``` + + + +for the five evidence-first reasoning rules. + + + +See: + + + +```text + +tool-risk-ladder.md + +``` + + + +for the classification of read-only and higher-risk agent tools. + + + +See: + + + +```text + +architecture.md + +``` + + + +for the implementation architecture. + + + +See: + + + +```text + +verified-environment.md + +``` + + + +for package versions, SDK information, tests, and the complete + +verification record. + + + +\--- + + + +\## Status + + + +\*\*Sample:\*\* Evidence-First Incident Triage Agent + + + +\*\*Run type:\*\* controlled live verification + + + +\*\*Provider:\*\* OpenRouter + + + +\*\*Model:\*\* `openai/gpt-4o-mini` + + + +\*\*Agent:\*\* `ChatClientAgent` + + + +\*\*Session:\*\* `ChatClientAgentSession` + + + +\*\*Deterministic tests at verification:\*\* 14/14 passing + + + +\*\*Live run:\*\* PASS + + + +\*\*Last verified:\*\* 26 September 2026 + diff --git a/dotnet-ai/agent-framework-incident-triage/docs/tool-risk-ladder.md b/dotnet-ai/agent-framework-incident-triage/docs/tool-risk-ladder.md new file mode 100644 index 0000000..e9ce5d5 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/docs/tool-risk-ladder.md @@ -0,0 +1,491 @@ +# AI Agent Tool Risk Ladder + +The **AI Agent Tool Risk Ladder** is a practical way to classify tools +based on what they are allowed to do and the potential impact of an +incorrect or unauthorized tool call. + +It was created for the +**Evidence-First Incident Triage Agent** reference sample from +dotnet-guide.com. + +This is not an official Microsoft Agent Framework classification. +It is an application-level design model that can be adapted to other +tool-using AI systems. + +--- + +# Why classify agent tools? + +Not all tools carry the same risk. + +Consider these two functions: + +```text +get_service_health() +restart_service() +``` + +Both might be callable by an AI agent. + +But their consequences are very different. + +The first reads information. + +The second changes a running system. + +Treating both as equivalent "tools" hides an important security and +operational distinction. + +The Tool Risk Ladder makes that distinction explicit. + +--- + +# Level 0 — Reference + +## Purpose + +Returns static or reference information. + +Examples: + +```text +read_runbook +get_documentation +lookup_error_code +read_policy +``` + +Typical characteristics: + +- no system-state changes; +- normally low operational impact; +- output may still contain incorrect or outdated information; +- source quality still matters. + +Example from this repository: + +```text +get_runbook +``` + +The tool returns: + +```text +RB-CHECKOUT-03 +``` + +which contains diagnostic guidance. + +The agent can read it, but nothing is changed. + +### Typical controls + +- validate tool inputs; +- restrict accessible resources; +- identify the source; +- distinguish reference guidance from observed evidence. + +--- + +# Level 1 — Observational + +## Purpose + +Reads current or historical system state. + +Examples: + +```text +get_service_health +get_recent_deployment +get_database_metrics +read_logs +query_traces +``` + +Example from this repository: + +```text +get_service_health +get_recent_deployment +``` + +These tools expose evidence such as: + +```text +E-101 +error_rate=18.7% +database_pool_utilization=96% +``` + +and: + +```text +E-201 +deployment=deploy-1842 +``` + +The tool still does not modify the system. + +However, Level 1 can be more sensitive than Level 0 because operational +data may include confidential or security-relevant information. + +### Typical controls + +- authentication; +- authorization; +- input validation; +- data minimization; +- audit logging; +- evidence/source identifiers; +- access restrictions for sensitive telemetry. + +--- + +# Level 2 — Reversible Write + +## Purpose + +Changes system state in a way that is normally easy to review or undo. + +Examples: + +```text +create_incident_note +add_ticket_comment +set_case_label +create_draft +update_noncritical_metadata +``` + +The agent is no longer simply observing. + +It is creating or modifying information. + +Although the action may be reversible, incorrect tool calls can create +confusion, unwanted records, or workflow problems. + +### Typical controls + +In addition to Level 0 and Level 1 controls: + +- validate proposed changes; +- show the intended action before execution where appropriate; +- enforce user or service permissions; +- record who or what initiated the action; +- support rollback or correction. + +--- + +# Level 3 — Operational Action + +## Purpose + +Changes a running system or operational environment. + +Examples: + +```text +restart_service +scale_application +disable_feature +invalidate_cache +rollback_deployment +rotate_runtime_configuration +``` + +An incorrect call can cause downtime or materially change production +behavior. + +A model recommendation and authority to execute that recommendation +should be treated as separate decisions. + +For example: + +```text +Agent conclusion: +"Rolling back deploy-1842 may be worth considering." + +does not imply: + +Agent permission: +rollback_deployment("deploy-1842") +``` + +### Typical controls + +Level 3 tools should normally introduce additional safeguards such as: + +- explicit authorization; +- bounded parameters; +- environment restrictions; +- human approval for consequential actions; +- idempotency where possible; +- audit logs; +- dry-run or preview modes; +- rollback planning; +- rate limits. + +--- + +# Level 4 — Destructive or High-Impact External Action + +## Purpose + +Performs actions that may be difficult to reverse, affect external +parties, or create substantial operational consequences. + +Examples: + +```text +delete_production_data +drop_database +deploy_to_all_regions +send_customer_notification +revoke_all_credentials +terminate_resource +execute_financial_transaction +``` + +These tools should not be treated as ordinary extensions of model +reasoning. + +The consequence of an incorrect invocation can be much larger than the +cost of an incorrect model response. + +### Typical controls + +Possible controls include: + +- strong authentication; +- narrow authorization; +- explicit human approval; +- multi-party approval for high-impact operations; +- deterministic validation outside the model; +- transaction limits; +- environment restrictions; +- immutable audit logs; +- rollback or recovery mechanisms where possible; +- policy enforcement independent of the LLM. + +Some Level 4 operations may be inappropriate for autonomous execution +altogether. + +--- + +# The Risk Ladder + +A simplified view is: + +```text +Level 0 +REFERENCE +Read runbook + │ + ▼ +Level 1 +OBSERVATIONAL +Read service health + │ + ▼ +Level 2 +REVERSIBLE WRITE +Create incident note + │ + ▼ +Level 3 +OPERATIONAL ACTION +Restart service + │ + ▼ +Level 4 +DESTRUCTIVE / HIGH IMPACT +Delete data or perform irreversible external action +``` + +As the level increases: + +```text +potential consequence ↑ +authorization needs ↑ +validation needs ↑ +approval requirements ↑ +audit requirements ↑ +``` + +--- + +# Why the reference agent stops at Levels 0 and 1 + +The Evidence-First Incident Triage Agent intentionally exposes only: + +```text +get_runbook Level 0 +get_service_health Level 1 +get_recent_deployment Level 1 +``` + +It can investigate. + +It cannot remediate. + +This is deliberate. + +The first tutorial focuses on: + +```text +retrieve +↓ +evaluate +↓ +explain +``` + +rather than: + +```text +retrieve +↓ +decide +↓ +change production +``` + +This keeps the core Agent Framework example understandable while also +providing a safer boundary for a first tool-using agent. + +--- + +# Recommendation Is Not Authorization + +One of the most important distinctions in agent design is: + +```text +model recommends an action + ≠ +system authorizes the action +``` + +For example, an agent may reasonably conclude: + +> Consider rolling back deploy-1842 if additional evidence confirms +> that the deployment introduced the failure. + +That conclusion should not automatically grant access to: + +```text +rollback_deployment() +``` + +Authorization should be enforced outside the language model. + +--- + +# Tool Risk Can Depend on Context + +A tool does not necessarily have one universal risk level. + +For example: + +```text +restart_service +``` + +might be: + +- relatively low risk in a developer sandbox; +- high risk in a production payment system. + +Likewise: + +```text +create_ticket +``` + +may be low impact internally but more consequential if it automatically +triggers an external workflow. + +The ladder should therefore be treated as a classification aid rather +than an absolute universal rating. + +--- + +# Design Questions for Every Tool + +Before exposing a function to an agent, ask: + +1. Does the tool only read information or can it change state? +2. Can the action be reversed? +3. What happens if the model calls it with the wrong arguments? +4. What happens if the model calls it repeatedly? +5. Does the user or service actually have permission to perform it? +6. Should a human approve execution? +7. Can deterministic code validate the request before execution? +8. Is the action recorded in an audit log? +9. Can its scope be reduced? +10. Is exposing this operation to an AI agent necessary at all? + +--- + +# Relationship to the Evidence-First Agent Contract + +The two patterns address different concerns. + +The **Evidence-First Agent Contract** asks: + +> Is the agent making claims that are actually supported by retrieved +> information? + +The **Tool Risk Ladder** asks: + +> What is the potential consequence if the agent invokes this tool? + +Together: + +```text +Evidence-First Contract + ↓ +controls claims and reasoning + +Tool Risk Ladder + ↓ +controls capability and consequence +``` + +A well-grounded agent can still be dangerous if it has excessive tool +permissions. + +Likewise, a read-only agent can still produce poor conclusions if it +misrepresents evidence. + +Both boundaries matter. + +--- + +# Reference Implementation + +The accompanying .NET reference sample currently uses only Level 0 and +Level 1 tools: + +| Tool | Level | Capability | +|---|---|---| +| `get_runbook` | 0 | Reference guidance | +| `get_service_health` | 1 | Read operational health evidence | +| `get_recent_deployment` | 1 | Read deployment evidence | + +There are intentionally no write-capable or destructive tools. + +Future examples may demonstrate stronger controls around higher-risk +tools, including explicit approval boundaries. + +--- + +## Status + +**Pattern status:** tutorial/reference design model + +**Specification status:** not an official Microsoft specification + +**Reference implementation:** Evidence-First Incident Triage Agent + +**Last verified:** 26 September 2026 \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/docs/verified-environment.md b/dotnet-ai/agent-framework-incident-triage/docs/verified-environment.md new file mode 100644 index 0000000..886ed72 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/docs/verified-environment.md @@ -0,0 +1,643 @@ +# Verified Environment + +This document records the environment in which the +**Evidence-First Incident Triage Agent** reference sample was built, +tested, and manually exercised. + +It is a verification record, not a claim that these are the only +compatible versions. + +--- + +# Verification Date + +**26 September 2026** + +--- + +# Platform + +- Target framework: `net10.0` +- .NET SDK used: `10.0.401` +- Development environment: Windows +- Solution format: `.slnx` + +The SDK version was verified with: + +```text +dotnet --version +``` + +Observed result: + +```text +10.0.401 +``` + +--- + +# Direct Package Dependency + +The application project has one direct Agent Framework integration +package: + +| Package | Requested | Resolved | +|---|---:|---:| +| `Microsoft.Agents.AI.OpenAI` | `1.22.0` | `1.22.0` | + +This was installed with: + +```text +dotnet add src/IncidentTriageAgent/IncidentTriageAgent.csproj package Microsoft.Agents.AI.OpenAI --version 1.22.0 +``` + +--- + +# Relevant Resolved Dependencies + +The resolved package graph was inspected with: + +```text +dotnet list src/IncidentTriageAgent/IncidentTriageAgent.csproj package --include-transitive +``` + +The following relevant packages were observed: + +| Package | Resolved version | +|---|---:| +| `Microsoft.Agents.AI` | `1.22.0` | +| `Microsoft.Agents.AI.Abstractions` | `1.22.0` | +| `Microsoft.Extensions.AI` | `10.10.0` | +| `Microsoft.Extensions.AI.Abstractions` | `10.10.0` | +| `Microsoft.Extensions.AI.Evaluation` | `10.10.0` | +| `Microsoft.Extensions.AI.OpenAI` | `10.10.0` | +| `Microsoft.Extensions.VectorData.Abstractions` | `10.10.0` | +| `OpenAI` | `2.13.0` | +| `System.ClientModel` | `1.15.0` | + +Additional transitive dependencies were also resolved by NuGet. + +The versions above are recorded because they are directly relevant to +the Agent Framework, Microsoft.Extensions.AI, and OpenAI-compatible +provider path used by this sample. + +--- + +# SDK Selection + +The repository includes a `global.json` baseline using: + +```json +{ + "sdk": { + "version": "10.0.401", + "rollForward": "latestFeature", + "allowPrerelease": false + } +} +``` + +This records the SDK baseline while allowing an appropriate later +installed .NET 10 feature band according to the configured roll-forward +policy. + +--- + +# Build Verification + +Command: + +```text +dotnet build +``` + +Observed result: + +```text +Build succeeded. +``` + +Both projects compiled successfully: + +```text +src/IncidentTriageAgent/IncidentTriageAgent.csproj +tests/IncidentTriageAgent.Tests/IncidentTriageAgent.Tests.csproj +``` + +Verification status: + +```text +PASS +``` + +--- + +# Deterministic Test Verification + +Command: + +```text +dotnet test +``` + +Observed test summary: + +```text +total: 14 +failed: 0 +succeeded: 14 +skipped: 0 +``` + +Verification status: + +```text +PASS +``` + +The automated test suite does not require: + +```text +OPENROUTER_API_KEY +``` + +and does not require a live model request. + +The deterministic tests cover the sample's evidence stores, runbook +store, tool behavior, missing-field behavior, and Agent Framework +function construction. + +--- + +# No-Key Execution Verification + +The application was also run without an API key: + +```text +dotnet run --project src/IncidentTriageAgent +``` + +Observed behavior: + +```text +OPENROUTER_API_KEY is not configured. + +The project can build and test without an API key. +Run 'dotnet test' to verify the deterministic evidence and tool layers. + +Set OPENROUTER_API_KEY only when you want to run the live agent. +``` + +This verifies that cloning, building, and testing the sample does not +require a model-provider secret. + +--- + +# Live Agent Framework Verification + +A separate controlled live run was performed after the deterministic +build and tests passed. + +Provider: + +```text +OpenRouter +``` + +OpenAI-compatible endpoint: + +```text +https://openrouter.ai/api/v1 +``` + +Model identifier used: + +```text +openai/gpt-4o-mini +``` + +Verification status: + +```text +PASS +``` + +The live run verified the following application path: + +```text +OpenRouter + ↓ +OpenAI-compatible client + ↓ +AsIChatClient() + ↓ +IChatClient + ↓ +TriageAgentFactory + ↓ +ChatClientAgent + ↓ +AgentSession + ↓ +read-only function tools + ↓ +streamed response +``` + +--- + +# Verified Agent Framework Runtime Types + +During the compile spike and live verification, the following concrete +runtime types were observed: + +```text +Agent type: ChatClientAgent +Session type: ChatClientAgentSession +``` + +The application code depends on the public abstractions: + +```text +AIAgent +AgentSession +IChatClient +``` + +rather than on the concrete runtime types. + +--- + +# Verified Function Tools + +The reference implementation exposes three read-only tools: + +```text +get_service_health +get_recent_deployment +get_runbook +``` + +The live Agent Framework run successfully used the sample's deterministic +information sources: + +```text +E-101 +E-201 +RB-CHECKOUT-03 +``` + +These correspond to: + +```text +E-101 service-health evidence +E-201 deployment evidence +RB-CHECKOUT-03 runbook guidance +``` + +--- + +# Verified Incident Evidence + +The deterministic sample includes: + +```text +E-101 +evidence_type=service_health +service=checkout-api +observed_at=2026-09-24T14:12:00Z +status=degraded +error_rate=18.7% +error_spike_started=2026-09-24T14:09:00Z +database_pool_utilization=96% +``` + +and: + +```text +E-201 +evidence_type=deployment +service=checkout-api +deployment=deploy-1842 +deployment_status=succeeded +completed_at=2026-09-24T14:02:00Z +previous_deployment=deploy-1839 +change_summary=pricing-rule refresh and structured logging update +rollback_performed=false +``` + +The evidence intentionally does **not** include a completion timestamp +for: + +```text +deploy-1839 +``` + +That omission is used to test whether missing evidence remains unknown. + +--- + +# Missing-Field Verification + +During the experimental validation, the agent was explicitly challenged +with: + +```text +At exactly what time did deploy-1839 complete? +``` + +The verified response correctly treated that value as unknown because the +retrieved evidence did not contain the timestamp. + +The final live sample also preserved this boundary and reported the +timestamp of `deploy-1839` as unknown. + +This verifies the intended behavior: + +```text +known identifier + ≠ +known timestamp +``` + +--- + +# Correlation vs Causation Verification + +The deterministic evidence establishes: + +```text +14:02 UTC deploy-1842 completed +14:09 UTC checkout-api error spike began +14:12 UTC checkout-api observed degraded +``` + +This establishes temporal correlation. + +It does not establish that: + +```text +deploy-1842 caused the checkout-api outage +``` + +In the verified same-session follow-up, the agent rejected that causal +statement and explained that the available evidence supports correlation +but not causation. + +This behavior is a central part of the +**Evidence-First Agent Contract**. + +--- + +# Guidance vs Evidence Verification + +The live sample retrieved: + +```text +RB-CHECKOUT-03 +``` + +which recommends checks such as: + +```text +inspect database connection-pool utilization +review traces for timeout or connection-acquisition failures +compare configuration with the previous known-good release +``` + +The verified agent treated these items as investigation guidance rather +than as proof that any one of those conditions caused the incident. + +This preserves the intended distinction: + +```text +runbook guidance + ≠ +observed incident evidence +``` + +--- + +# AgentSession Verification + +The same `AgentSession` was reused for: + +```text +initial incident assessment + ↓ +same-session follow-up +``` + +The follow-up asked whether it was truthful to state: + +```text +deploy-1842 caused the checkout-api outage +``` + +The agent answered using the existing session context rather than +requiring the complete incident evidence to be restated by the user. + +--- + +# Streaming Verification + +The live sample uses: + +```text +RunStreamingAsync(...) +``` + +and consumes: + +```text +AgentResponseUpdate +``` + +The live run successfully streamed the incident assessment and the +same-session follow-up. + +--- + +# Security and Secret Handling + +No real API key is stored in the source code, JSON data files, tests, or +repository documentation. + +The application reads the live provider key from: + +```text +OPENROUTER_API_KEY +``` + +An optional model override can be supplied with: + +```text +OPENROUTER_MODEL +``` + +The temporary OpenRouter key used for manual live verification was not +committed to the repository and was intended to be revoked after the +verification run. + +--- + +# Deterministic vs Probabilistic Verification Boundary + +The repository deliberately separates deterministic verification from +probabilistic model behavior. + +## Deterministically verified + +```text +JSON evidence loading +runbook loading +evidence lookup +missing-field behavior +tool output +unknown-service behavior +AIFunction construction +build +unit tests +``` + +These behaviors are suitable for automated CI verification. + +## Manually observed live behavior + +```text +model tool selection +model-generated wording +reasoning phrasing +same-session interpretation +correlation-vs-causation response +``` + +These behaviors are probabilistic and may vary between model runs. + +For that reason, live LLM output is not treated as a deterministic CI +assertion. + +--- + +# Current Verification Summary + +| Item | Verified | +|---|---| +| .NET SDK `10.0.401` | Yes | +| Target framework `net10.0` | Yes | +| `.slnx` solution | Yes | +| `Microsoft.Agents.AI.OpenAI 1.22.0` | Yes | +| Build | PASS | +| Automated tests | 14/14 PASS | +| Build without API key | Yes | +| Tests without API key | Yes | +| Safe no-key application exit | Yes | +| `IChatClient` provider abstraction | Yes | +| `ChatClientAgent` creation | Yes | +| `AgentSession` creation and reuse | Yes | +| Function tools | Yes | +| Streaming | Yes | +| Service-health evidence retrieval | Yes | +| Deployment evidence retrieval | Yes | +| Runbook guidance retrieval | Yes | +| Missing-field handling | Yes | +| Correlation vs causation handling | Yes | +| Guidance vs evidence separation | Yes | +| Live OpenRouter run | PASS | + +--- + +# Important Limitations + +This verification record should not be interpreted as proof that: + +- every future package version will behave identically; +- every .NET 10 SDK build is compatible; +- every OpenRouter model will produce the same reasoning; +- every future model response will follow the same wording; +- the sample is a complete production incident-response system; +- the underlying synthetic evidence represents a real production system. + +The sample uses fixed synthetic data so that the deterministic layer is +reproducible. + +A successful live model run demonstrates observed behavior for the +recorded environment. It does not make probabilistic output deterministic. + +--- + +# Reproducing the Deterministic Verification + +From the repository root: + +```text +dotnet --version +dotnet build +dotnet test +``` + +The build and tests require no live model API key. + +To inspect resolved application dependencies: + +```text +dotnet list src/IncidentTriageAgent/IncidentTriageAgent.csproj package --include-transitive +``` + +A live run additionally requires: + +```text +OPENROUTER_API_KEY +``` + +and can be started with: + +```text +dotnet run --project src/IncidentTriageAgent +``` + +--- + +# Related Documents + +See: + +```text +evidence-first-contract.md +``` + +for the reasoning and evidence-grounding design contract. + +See: + +```text +tool-risk-ladder.md +``` + +for the tool capability and consequence model. + +See: + +```text +architecture.md +``` + +for the implementation architecture. + +--- + +## Status + +**Reference implementation:** Evidence-First Incident Triage Agent + +**Verification date:** 26 September 2026 + +**Build:** PASS + +**Automated tests:** 14/14 PASS + +**Live Agent Framework run:** PASS + +**Target framework:** .NET 10 + +**Direct Agent Framework package:** `Microsoft.Agents.AI.OpenAI 1.22.0` \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/global.json b/dotnet-ai/agent-framework-incident-triage/global.json new file mode 100644 index 0000000..30c4ed3 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "10.0.401", + "rollForward": "latestFeature", + "allowPrerelease": false + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Agents/TriageAgentFactory.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Agents/TriageAgentFactory.cs new file mode 100644 index 0000000..72c3df0 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Agents/TriageAgentFactory.cs @@ -0,0 +1,136 @@ +using IncidentTriageAgent.Guidance; +using IncidentTriageAgent.Evidence; +using IncidentTriageAgent.Tools; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace IncidentTriageAgent.Agents; + +/// +/// Builds the evidence-first incident-triage agent. +/// +/// Provider configuration is deliberately kept outside this factory. +/// The factory depends only on IChatClient and the deterministic +/// evidence/tool layer. +/// +public sealed class TriageAgentFactory +{ + private readonly EvidenceStore _evidenceStore; +private readonly RunbookStore _runbookStore; + + public TriageAgentFactory( + EvidenceStore evidenceStore, + RunbookStore runbookStore) +{ + _evidenceStore = evidenceStore; + _runbookStore = runbookStore; +} + + + /// + /// Creates an Agent Framework agent backed by the supplied IChatClient. + /// + public AIAgent Create(IChatClient chatClient) + { + ArgumentNullException.ThrowIfNull(chatClient); + + ServiceHealthTool healthTool = + new(_evidenceStore); + + DeploymentTool deploymentTool = + new(_evidenceStore); + + RunbookTool runbookTool = + new(_runbookStore); + + List tools = + [ + healthTool.CreateFunction(), + deploymentTool.CreateFunction(), + runbookTool.CreateFunction() + ]; + + return chatClient.AsAIAgent( + name: "IncidentTriageAgent", + instructions: AgentInstructions, + tools: tools); + } + + + private const string AgentInstructions = + """ + You are an evidence-first incident-triage agent. + + Your job is to help investigate operational incidents + without inventing facts or overstating conclusions. + + INFORMATION TYPES + + OBSERVED EVIDENCE + Service-health records and deployment records contain + observed operational facts. + + REFERENCE GUIDANCE + Runbooks contain investigation guidance. + A runbook instruction is not evidence that a condition + actually occurred. + + EVIDENCE-FIRST RULES + + 1. Retrieve operational evidence before making factual + claims about the current incident. + + 2. Operational facts may come only from retrieved evidence. + + 3. Never invent or fill in a missing field. + + If evidence identifies a previous deployment but does + not contain its timestamp, state that the timestamp + is UNKNOWN. + + 4. Clearly distinguish: + - observed evidence, + - reasonable inference, + - runbook guidance, + - unknown information, + - unproven conclusions. + + 5. Temporal correlation is not proof of causation. + + 6. A high metric value may justify investigation, + but does not by itself establish root cause. + + 7. Cite evidence IDs when stating operational facts. + + 8. Cite runbook IDs when presenting operational guidance. + + 9. If evidence is insufficient for a conclusion, + explicitly say so. + + 10. You have read-only tools. + + Never claim that you restarted, rolled back, + deployed, deleted, modified, repaired, + or otherwise changed a system. + + RESPONSE STYLE + + When performing an incident assessment, prefer these sections: + + OBSERVED EVIDENCE + Facts directly supported by evidence IDs. + + REASONABLE INFERENCES + Interpretations that follow from the evidence but + are not themselves observed facts. + + RUNBOOK GUIDANCE + Recommended investigation steps from reference material. + + UNKNOWNS + Information that is missing from available evidence. + + UNPROVEN CLAIMS + Conclusions that available evidence does not establish. + """; +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/deployments.json b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/deployments.json new file mode 100644 index 0000000..dec0b79 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/deployments.json @@ -0,0 +1,26 @@ +[ + { + "Id": "E-201", + "Type": "deployment", + "Service": "checkout-api", + "ObservedAt": "2026-09-24T14:02:00Z", + "Facts": { + "deployment": "deploy-1842", + "deployment_status": "succeeded", + "previous_deployment": "deploy-1839", + "change_summary": "pricing-rule refresh and structured logging update", + "rollback_performed": "false" + } + }, + { + "Id": "E-202", + "Type": "deployment", + "Service": "catalog-api", + "ObservedAt": "2026-09-23T19:40:00Z", + "Facts": { + "deployment": "deploy-991", + "deployment_status": "succeeded", + "rollback_performed": "false" + } + } +] \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/runbooks.json b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/runbooks.json new file mode 100644 index 0000000..3708b91 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/runbooks.json @@ -0,0 +1,26 @@ +[ + { + "Id": "RB-CHECKOUT-03", + "Service": "checkout-api", + "Title": "Checkout API elevated 5xx response", + "RecommendedChecks": [ + "Compare error-rate onset with recent deployments.", + "Inspect database connection-pool utilization.", + "Check database connection counts before and after deployment.", + "Review traces for timeout or connection-acquisition failures.", + "Compare configuration with the previous known-good release." + ], + "Caution": "These checks are diagnostic guidance. Their presence in this runbook does not establish root cause." + }, + { + "Id": "RB-CATALOG-02", + "Service": "catalog-api", + "Title": "Catalog API degradation", + "RecommendedChecks": [ + "Inspect downstream dependency latency.", + "Review cache hit rate.", + "Check database query duration." + ], + "Caution": "These checks are diagnostic guidance. Their presence in this runbook does not establish root cause." + } +] \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/services.json b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/services.json new file mode 100644 index 0000000..dc4f3a2 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Data/services.json @@ -0,0 +1,25 @@ +[ + { + "Id": "E-101", + "Type": "service_health", + "Service": "checkout-api", + "ObservedAt": "2026-09-24T14:12:00Z", + "Facts": { + "status": "degraded", + "error_rate": "18.7%", + "error_spike_started": "2026-09-24T14:09:00Z", + "database_pool_utilization": "96%" + } + }, + { + "Id": "E-102", + "Type": "service_health", + "Service": "catalog-api", + "ObservedAt": "2026-09-24T14:12:00Z", + "Facts": { + "status": "healthy", + "error_rate": "0.2%", + "database_pool_utilization": "34%" + } + } +] \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceItem.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceItem.cs new file mode 100644 index 0000000..087733a --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceItem.cs @@ -0,0 +1,26 @@ +namespace IncidentTriageAgent.Evidence; + +/// +/// Represents one deterministic piece of operational evidence. +/// +/// Evidence items contain observed facts only. +/// They must not contain model-generated conclusions or inferred facts. +/// +public sealed record EvidenceItem( + string Id, + string Type, + string Service, + DateTimeOffset ObservedAt, + IReadOnlyDictionary Facts) +{ + /// + /// Returns a fact value when the field exists. + /// Missing fields remain missing rather than being inferred. + /// + public string? GetFact(string key) + { + return Facts.TryGetValue(key, out string? value) + ? value + : null; + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceStore.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceStore.cs new file mode 100644 index 0000000..5d16580 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Evidence/EvidenceStore.cs @@ -0,0 +1,152 @@ +using System.Reflection; +using System.Text.Json; + +namespace IncidentTriageAgent.Evidence; + +/// +/// Loads deterministic operational evidence from the sample's +/// embedded JSON data files. +/// +/// The evidence is intentionally fixed so that tool behavior and +/// automated tests remain repeatable without a live monitoring system. +/// +public sealed class EvidenceStore +{ + private static readonly JsonSerializerOptions JsonOptions = + new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly IReadOnlyList _items; + + + public EvidenceStore() + { + List items = []; + + items.AddRange( + LoadEvidence("services.json")); + + items.AddRange( + LoadEvidence("deployments.json")); + + _items = items; + } + + + /// + /// Returns the newest evidence item of the requested type + /// for the requested service. + /// + public EvidenceItem? GetLatest( + string serviceName, + string evidenceType) + { + if (string.IsNullOrWhiteSpace(serviceName) || + string.IsNullOrWhiteSpace(evidenceType)) + { + return null; + } + + string service = + Normalize(serviceName); + + string type = + evidenceType + .Trim() + .ToLowerInvariant(); + + return _items + .Where(item => + item.Service.Equals( + service, + StringComparison.OrdinalIgnoreCase) && + item.Type.Equals( + type, + StringComparison.OrdinalIgnoreCase)) + .OrderByDescending( + item => item.ObservedAt) + .FirstOrDefault(); + } + + + /// + /// Returns all evidence currently available for a service. + /// + public IReadOnlyList GetForService( + string serviceName) + { + if (string.IsNullOrWhiteSpace(serviceName)) + { + return []; + } + + string service = + Normalize(serviceName); + + return _items + .Where(item => + item.Service.Equals( + service, + StringComparison.OrdinalIgnoreCase)) + .OrderBy( + item => item.ObservedAt) + .ToList(); + } + + + /// + /// Loads one embedded JSON evidence file. + /// + /// Resource discovery uses the filename suffix rather than + /// assuming a particular root namespace. + /// + private static IReadOnlyList LoadEvidence( + string fileName) + { + Assembly assembly = + typeof(EvidenceStore).Assembly; + + string expectedSuffix = + $".Data.{fileName}"; + + string? resourceName = + assembly + .GetManifestResourceNames() + .SingleOrDefault(name => + name.EndsWith( + expectedSuffix, + StringComparison.OrdinalIgnoreCase)); + + if (resourceName is null) + { + throw new InvalidOperationException( + $"Embedded evidence file '{fileName}' was not found."); + } + + using Stream stream = + assembly.GetManifestResourceStream( + resourceName) + ?? throw new InvalidOperationException( + $"Embedded evidence file '{fileName}' could not be opened."); + + List? evidence = + JsonSerializer.Deserialize>( + stream, + JsonOptions); + + return evidence + ?? throw new InvalidOperationException( + $"Embedded evidence file '{fileName}' contained no valid data."); + } + + + private static string Normalize( + string serviceName) + { + return serviceName + .Trim() + .ToLowerInvariant(); + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookItem.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookItem.cs new file mode 100644 index 0000000..af59ffd --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookItem.cs @@ -0,0 +1,14 @@ +namespace IncidentTriageAgent.Guidance; + +/// +/// Represents operational reference guidance. +/// +/// A runbook is not observed incident evidence. +/// It describes investigation steps that engineers may follow. +/// +public sealed record RunbookItem( + string Id, + string Service, + string Title, + string[] RecommendedChecks, + string Caution); \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookStore.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookStore.cs new file mode 100644 index 0000000..af9f106 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Guidance/RunbookStore.cs @@ -0,0 +1,90 @@ +using System.Reflection; +using System.Text.Json; + +namespace IncidentTriageAgent.Guidance; + +/// +/// Loads deterministic runbook guidance from the +/// sample's embedded JSON data file. +/// +public sealed class RunbookStore +{ + private static readonly JsonSerializerOptions JsonOptions = + new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly IReadOnlyList _runbooks; + + + public RunbookStore() + { + _runbooks = + LoadRunbooks("runbooks.json"); + } + + + /// + /// Returns the runbook for the requested service. + /// + public RunbookItem? GetForService( + string serviceName) + { + if (string.IsNullOrWhiteSpace(serviceName)) + { + return null; + } + + string service = + serviceName + .Trim() + .ToLowerInvariant(); + + return _runbooks + .FirstOrDefault(runbook => + runbook.Service.Equals( + service, + StringComparison.OrdinalIgnoreCase)); + } + + + private static IReadOnlyList LoadRunbooks( + string fileName) + { + Assembly assembly = + typeof(RunbookStore).Assembly; + + string expectedSuffix = + $".Data.{fileName}"; + + string? resourceName = + assembly + .GetManifestResourceNames() + .SingleOrDefault(name => + name.EndsWith( + expectedSuffix, + StringComparison.OrdinalIgnoreCase)); + + if (resourceName is null) + { + throw new InvalidOperationException( + $"Embedded runbook file '{fileName}' was not found."); + } + + using Stream stream = + assembly.GetManifestResourceStream( + resourceName) + ?? throw new InvalidOperationException( + $"Embedded runbook file '{fileName}' could not be opened."); + + List? runbooks = + JsonSerializer.Deserialize>( + stream, + JsonOptions); + + return runbooks + ?? throw new InvalidOperationException( + $"Embedded runbook file '{fileName}' contained no valid data."); + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/IncidentTriageAgent.csproj b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/IncidentTriageAgent.csproj new file mode 100644 index 0000000..807ccfe --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/IncidentTriageAgent.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Program.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Program.cs new file mode 100644 index 0000000..7af98a5 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Program.cs @@ -0,0 +1,209 @@ +using System.ClientModel; +using IncidentTriageAgent.Agents; +using IncidentTriageAgent.Evidence; +using IncidentTriageAgent.Guidance; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using OpenAI; + +const string DefaultModel = + "openai/gpt-4o-mini"; + +const string DefaultIncident = + """ + Investigate the checkout-api incident. + + Using only the available operational evidence and reference guidance: + + 1. state what the observed evidence establishes, + 2. identify reasonable inferences, + 3. list the relevant runbook guidance, + 4. identify unknown information, + 5. and state which root-cause claims remain unproven. + + Do not invent missing facts. + Do not treat correlation as proof of causation. + """; + +const string DefaultFollowUp = + """ + Can I truthfully write in the incident report: + + "deploy-1842 caused the checkout-api outage"? + + Give a concise evidence-based answer. + """; + + +// ------------------------------------------------------------------- +// Configuration +// ------------------------------------------------------------------- + +string? apiKey = + Environment.GetEnvironmentVariable( + "OPENROUTER_API_KEY"); + +if (string.IsNullOrWhiteSpace(apiKey)) +{ + Console.WriteLine( + "OPENROUTER_API_KEY is not configured."); + + Console.WriteLine(); + + Console.WriteLine( + "The project can build and test without an API key."); + + Console.WriteLine( + "Run 'dotnet test' to verify the deterministic evidence and tool layers."); + + Console.WriteLine(); + + Console.WriteLine( + "Set OPENROUTER_API_KEY only when you want to run the live agent."); + + return; +} + + +string modelName = + Environment.GetEnvironmentVariable( + "OPENROUTER_MODEL") + ?? DefaultModel; + + +// ------------------------------------------------------------------- +// Provider +// ------------------------------------------------------------------- +// +// Provider-specific configuration stops here. +// +// The rest of the sample works with IChatClient. +// ------------------------------------------------------------------- + +OpenAIClientOptions clientOptions = + new() + { + Endpoint = + new Uri( + "https://openrouter.ai/api/v1") + }; + +OpenAIClient openAiClient = + new( + new ApiKeyCredential(apiKey), + clientOptions); + +IChatClient chatClient = + openAiClient + .GetChatClient(modelName) + .AsIChatClient(); + + +// ------------------------------------------------------------------- +// Deterministic data sources +// ------------------------------------------------------------------- + +EvidenceStore evidenceStore = + new(); + +RunbookStore runbookStore = + new(); + + +// ------------------------------------------------------------------- +// Agent +// ------------------------------------------------------------------- + +TriageAgentFactory agentFactory = + new( + evidenceStore, + runbookStore); + +AIAgent agent = + agentFactory.Create( + chatClient); + +AgentSession session = + await agent.CreateSessionAsync(); + + +// ------------------------------------------------------------------- +// Header +// ------------------------------------------------------------------- + +Console.WriteLine( + "============================================"); + +Console.WriteLine( + " Evidence-First Incident Triage Agent"); + +Console.WriteLine( + "============================================"); + +Console.WriteLine(); + +Console.WriteLine( + $"Model: {modelName}"); + +Console.WriteLine( + $"Agent: {agent.GetType().Name}"); + +Console.WriteLine( + $"Session: {session.GetType().Name}"); + +Console.WriteLine(); + + +// ------------------------------------------------------------------- +// Incident assessment +// ------------------------------------------------------------------- + +Console.WriteLine( + "=== INCIDENT ASSESSMENT ==="); + +Console.WriteLine(); + +await StreamAgentResponseAsync( + agent, + session, + DefaultIncident); + + +// ------------------------------------------------------------------- +// Same-session follow-up +// ------------------------------------------------------------------- + +Console.WriteLine(); +Console.WriteLine(); + +Console.WriteLine( + "=== SAME SESSION FOLLOW-UP ==="); + +Console.WriteLine(); + +await StreamAgentResponseAsync( + agent, + session, + DefaultFollowUp); + +Console.WriteLine(); + + +// ------------------------------------------------------------------- +// Streaming helper +// ------------------------------------------------------------------- + +static async Task StreamAgentResponseAsync( + AIAgent agent, + AgentSession session, + string prompt) +{ + await foreach ( + AgentResponseUpdate update + in agent.RunStreamingAsync( + prompt, + session)) + { + Console.Write(update); + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/DeploymentTool.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/DeploymentTool.cs new file mode 100644 index 0000000..465a236 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/DeploymentTool.cs @@ -0,0 +1,63 @@ +using System.ComponentModel; +using IncidentTriageAgent.Evidence; +using Microsoft.Extensions.AI; + +namespace IncidentTriageAgent.Tools; + +/// +/// Read-only tool that exposes observed deployment evidence. +/// +public sealed class DeploymentTool +{ + private readonly EvidenceStore _store; + + public DeploymentTool(EvidenceStore store) + { + _store = store; + } + + + /// + /// Creates the Agent Framework function exposed to the model. + /// + public AIFunction CreateFunction() + { + return AIFunctionFactory.Create( + GetRecentDeployment, + name: "get_recent_deployment", + description: + "Returns observed read-only deployment evidence " + + "for a named service."); + } + + + [Description( + "Returns observed deployment evidence for a service.")] + public string GetRecentDeployment( + [Description("Service name, for example checkout-api.")] + string serviceName) + { + EvidenceItem? evidence = + _store.GetLatest( + serviceName, + "deployment"); + + if (evidence is null) + { + return + $"NO-EVIDENCE | No deployment record exists for '{serviceName}'."; + } + + return $""" + {evidence.Id} + evidence_type={evidence.Type} + service={evidence.Service} + deployment={evidence.GetFact("deployment")} + deployment_status={evidence.GetFact("deployment_status")} + completed_at={evidence.ObservedAt:O} + previous_deployment={evidence.GetFact("previous_deployment")} + change_summary={evidence.GetFact("change_summary")} + rollback_performed={evidence.GetFact("rollback_performed")} + """; + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/RunbookTool.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/RunbookTool.cs new file mode 100644 index 0000000..4efaec8 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/RunbookTool.cs @@ -0,0 +1,77 @@ +using System.ComponentModel; +using IncidentTriageAgent.Guidance; +using Microsoft.Extensions.AI; + +namespace IncidentTriageAgent.Tools; + +/// +/// Read-only tool that exposes operational reference guidance. +/// +/// Runbook content is guidance, not observed incident evidence. +/// +public sealed class RunbookTool +{ + private readonly RunbookStore _store; + + public RunbookTool( + RunbookStore store) + { + _store = store; + } + + + public AIFunction CreateFunction() + { + return AIFunctionFactory.Create( + GetRunbook, + name: "get_runbook", + description: + "Returns read-only operational runbook guidance for a named service. " + + "Runbook guidance describes recommended investigation steps; " + + "it is not evidence that a particular failure actually occurred."); + } + + + [Description( + "Returns operational runbook guidance for a service. " + + "Runbook content is reference guidance, not observed incident evidence.")] + public string GetRunbook( + [Description("Service name, for example checkout-api.")] + string serviceName) + { + RunbookItem? runbook = + _store.GetForService(serviceName); + + if (runbook is null) + { + return + $"NO-GUIDANCE | No runbook exists for '{serviceName}'."; + } + + List lines = + [ + runbook.Id, + "information_type=runbook_guidance", + $"service={runbook.Service}", + $"title={runbook.Title}", + "", + "recommended_checks:" + ]; + + for (int i = 0; + i < runbook.RecommendedChecks.Length; + i++) + { + lines.Add( + $"{i + 1}. {runbook.RecommendedChecks[i]}"); + } + + lines.Add(""); + lines.Add("caution:"); + lines.Add(runbook.Caution); + + return string.Join( + Environment.NewLine, + lines); + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/ServiceHealthTool.cs b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/ServiceHealthTool.cs new file mode 100644 index 0000000..e68ab05 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/src/IncidentTriageAgent/Tools/ServiceHealthTool.cs @@ -0,0 +1,62 @@ +using System.ComponentModel; +using IncidentTriageAgent.Evidence; +using Microsoft.Extensions.AI; + +namespace IncidentTriageAgent.Tools; + +/// +/// Read-only tool that exposes observed service-health evidence. +/// +public sealed class ServiceHealthTool +{ + private readonly EvidenceStore _store; + + public ServiceHealthTool(EvidenceStore store) + { + _store = store; + } + + + /// + /// Creates the Agent Framework function exposed to the model. + /// + public AIFunction CreateFunction() + { + return AIFunctionFactory.Create( + GetServiceHealth, + name: "get_service_health", + description: + "Returns observed read-only operational health evidence " + + "for a named service."); + } + + + [Description( + "Returns observed operational health evidence for a service.")] + public string GetServiceHealth( + [Description("Service name, for example checkout-api.")] + string serviceName) + { + EvidenceItem? evidence = + _store.GetLatest( + serviceName, + "service_health"); + + if (evidence is null) + { + return + $"NO-EVIDENCE | No service health record exists for '{serviceName}'."; + } + + return $""" + {evidence.Id} + evidence_type={evidence.Type} + service={evidence.Service} + observed_at={evidence.ObservedAt:O} + status={evidence.GetFact("status")} + error_rate={evidence.GetFact("error_rate")} + error_spike_started={evidence.GetFact("error_spike_started")} + database_pool_utilization={evidence.GetFact("database_pool_utilization")} + """; + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/EvidenceStoreTests.cs b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/EvidenceStoreTests.cs new file mode 100644 index 0000000..29b901b --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/EvidenceStoreTests.cs @@ -0,0 +1,101 @@ +using IncidentTriageAgent.Evidence; +using Xunit; + +namespace IncidentTriageAgent.Tests; + +public sealed class EvidenceStoreTests +{ + [Fact] + public void GetLatest_ReturnsCheckoutHealthEvidence() + { + EvidenceStore store = new(); + + EvidenceItem? evidence = + store.GetLatest( + "checkout-api", + "service_health"); + + Assert.NotNull(evidence); + + Assert.Equal("E-101", evidence.Id); + Assert.Equal("degraded", evidence.GetFact("status")); + Assert.Equal("18.7%", evidence.GetFact("error_rate")); + Assert.Equal( + "96%", + evidence.GetFact("database_pool_utilization")); + } + + + [Fact] + public void GetLatest_NormalizesServiceName() + { + EvidenceStore store = new(); + + EvidenceItem? evidence = + store.GetLatest( + " CHECKOUT-API ", + "deployment"); + + Assert.NotNull(evidence); + Assert.Equal("E-201", evidence.Id); + Assert.Equal( + "deploy-1842", + evidence.GetFact("deployment")); + } + + + [Fact] + public void DeploymentEvidence_DoesNotInventMissingTimestamp() + { + EvidenceStore store = new(); + + EvidenceItem? evidence = + store.GetLatest( + "checkout-api", + "deployment"); + + Assert.NotNull(evidence); + + Assert.Equal( + "deploy-1839", + evidence.GetFact("previous_deployment")); + + Assert.Null( + evidence.GetFact( + "previous_deployment_completed_at")); + } + + + [Fact] + public void UnknownService_ReturnsNoEvidence() + { + EvidenceStore store = new(); + + EvidenceItem? evidence = + store.GetLatest( + "unknown-api", + "service_health"); + + Assert.Null(evidence); + } + + + [Fact] + public void GetForService_ReturnsBothCheckoutEvidenceItems() + { + EvidenceStore store = new(); + + IReadOnlyList evidence = + store.GetForService("checkout-api"); + + Assert.Equal(2, evidence.Count); + + Assert.Contains( + evidence, + item => item.Id == "E-101"); + + Assert.Contains( + evidence, + item => item.Id == "E-201"); + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/IncidentTriageAgent.Tests.csproj b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/IncidentTriageAgent.Tests.csproj new file mode 100644 index 0000000..446d973 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/IncidentTriageAgent.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/RunbookStoreTests.cs b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/RunbookStoreTests.cs new file mode 100644 index 0000000..ac7045b --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/RunbookStoreTests.cs @@ -0,0 +1,60 @@ +using IncidentTriageAgent.Guidance; +using Xunit; + +namespace IncidentTriageAgent.Tests; + +public sealed class RunbookStoreTests +{ + [Fact] + public void GetForService_ReturnsCheckoutRunbook() + { + RunbookStore store = new(); + + RunbookItem? runbook = + store.GetForService("checkout-api"); + + Assert.NotNull(runbook); + + Assert.Equal( + "RB-CHECKOUT-03", + runbook.Id); + + Assert.Equal( + "Checkout API elevated 5xx response", + runbook.Title); + + Assert.Equal( + 5, + runbook.RecommendedChecks.Length); + } + + + [Fact] + public void GetForService_NormalizesServiceName() + { + RunbookStore store = new(); + + RunbookItem? runbook = + store.GetForService( + " CHECKOUT-API "); + + Assert.NotNull(runbook); + + Assert.Equal( + "RB-CHECKOUT-03", + runbook.Id); + } + + + [Fact] + public void UnknownService_ReturnsNoRunbook() + { + RunbookStore store = new(); + + RunbookItem? runbook = + store.GetForService( + "payments-api"); + + Assert.Null(runbook); + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/ToolTests.cs b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/ToolTests.cs new file mode 100644 index 0000000..cfa051d --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/tests/IncidentTriageAgent.Tests/ToolTests.cs @@ -0,0 +1,127 @@ +using IncidentTriageAgent.Evidence; +using IncidentTriageAgent.Guidance; +using IncidentTriageAgent.Tools; +using Microsoft.Extensions.AI; +using Xunit; + +namespace IncidentTriageAgent.Tests; + +public sealed class ToolTests +{ + [Fact] + public void ServiceHealthTool_ReturnsEvidence101() + { + EvidenceStore store = new(); + ServiceHealthTool tool = new(store); + + string result = + tool.GetServiceHealth("checkout-api"); + + Assert.Contains("E-101", result); + Assert.Contains("status=degraded", result); + Assert.Contains("error_rate=18.7%", result); + + Assert.Contains( + "database_pool_utilization=96%", + result); + } + + + [Fact] + public void ServiceHealthTool_RejectsUnknownService() + { + EvidenceStore store = new(); + ServiceHealthTool tool = new(store); + + string result = + tool.GetServiceHealth("payments-api"); + + Assert.Contains("NO-EVIDENCE", result); + } + + + [Fact] + public void DeploymentTool_ReturnsEvidence201() + { + EvidenceStore store = new(); + DeploymentTool tool = new(store); + + string result = + tool.GetRecentDeployment("checkout-api"); + + Assert.Contains("E-201", result); + Assert.Contains("deployment=deploy-1842", result); + + Assert.Contains( + "previous_deployment=deploy-1839", + result); + } + + + [Fact] + public void DeploymentTool_DoesNotInventPreviousDeploymentTimestamp() + { + EvidenceStore store = new(); + DeploymentTool tool = new(store); + + string result = + tool.GetRecentDeployment("checkout-api"); + + Assert.DoesNotContain( + "previous_deployment_completed_at", + result); + } + + + [Fact] + public void RunbookTool_ReturnsGuidanceNotEvidence() + { + RunbookStore store = new(); + RunbookTool tool = new(store); + + string result = + tool.GetRunbook("checkout-api"); + + Assert.Contains( + "RB-CHECKOUT-03", + result); + + Assert.Contains( + "information_type=runbook_guidance", + result); + + Assert.Contains( + "does not establish root cause", + result); + } + + + [Fact] + public void AllThreeTools_CreateAgentFunctions() + { + EvidenceStore evidenceStore = new(); + RunbookStore runbookStore = new(); + + ServiceHealthTool healthTool = + new(evidenceStore); + + DeploymentTool deploymentTool = + new(evidenceStore); + + RunbookTool runbookTool = + new(runbookStore); + + AIFunction healthFunction = + healthTool.CreateFunction(); + + AIFunction deploymentFunction = + deploymentTool.CreateFunction(); + + AIFunction runbookFunction = + runbookTool.CreateFunction(); + + Assert.NotNull(healthFunction); + Assert.NotNull(deploymentFunction); + Assert.NotNull(runbookFunction); + } +} \ No newline at end of file diff --git a/dotnet-ai/agent-framework-incident-triage/verified-environment.json b/dotnet-ai/agent-framework-incident-triage/verified-environment.json new file mode 100644 index 0000000..fa74704 --- /dev/null +++ b/dotnet-ai/agent-framework-incident-triage/verified-environment.json @@ -0,0 +1,40 @@ +{ + "verifiedDate": "2026-09-26", + "targetFramework": "net10.0", + "dotnetSdk": "10.0.401", + "solutionFormat": "slnx", + "directPackages": { + "Microsoft.Agents.AI.OpenAI": "1.22.0" + }, + "resolvedPackages": { + "Microsoft.Agents.AI": "1.22.0", + "Microsoft.Agents.AI.Abstractions": "1.22.0", + "Microsoft.Extensions.AI": "10.10.0", + "Microsoft.Extensions.AI.Abstractions": "10.10.0", + "Microsoft.Extensions.AI.OpenAI": "10.10.0", + "OpenAI": "2.13.0", + "System.ClientModel": "1.15.0" + }, + "verification": { + "build": "pass", + "tests": { + "status": "pass", + "total": 14, + "failed": 0, + "skipped": 0 + }, + "liveAgent": { + "status": "pass", + "provider": "OpenRouter", + "model": "openai/gpt-4o-mini", + "streaming": true, + "agentSession": true, + "functionTools": true + } + }, + "security": { + "apiKeyStoredInRepository": false, + "toolsAreReadOnly": true, + "liveModelRequiredForTests": false + } +} \ No newline at end of file From f7a32239ad16aeab8644ca430c90db4bae99ee10 Mon Sep 17 00:00:00 2001 From: BALAJI BASHYAM Date: Sun, 27 Sep 2026 15:06:21 +0000 Subject: [PATCH 2/3] Add Agent Framework sample to repository README --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 82135fd..e224c44 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Each sample folder contains a focused implementation of one tutorial topic. The | --- | --- | --- | | [`dotnet-ai/provider-agnostic-chat-gateway`](dotnet-ai/provider-agnostic-chat-gateway/) | One HTTP chat endpoint using Ollama and optional OpenAI providers through `Microsoft.Extensions.AI.IChatClient` | [Build a switchable multi-provider AI gateway with IChatClient](https://www.dotnet-guide.com/tutorials/dotnet-ai/provider-agnostic-chat-gateway/) | | [`dotnet-ai/hybrid-search-ef-core-pgvector`](dotnet-ai/hybrid-search-ef-core-pgvector/) | Minimal Reciprocal Rank Fusion demo combining pre-ranked keyword and vector results | [Hybrid Search in .NET with EF Core 10 and pgvector](https://www.dotnet-guide.com/tutorials/dotnet-ai/hybrid-search-ef-core-pgvector/) | +| [`dotnet-ai/agent-framework-incident-triage`](dotnet-ai/agent-framework-incident-triage/) | Evidence-first Microsoft Agent Framework sample with read-only tools, deterministic incident evidence, `AgentSession` reuse, and correlation-vs-causation boundaries | [Microsoft Agent Framework Tutorial in C#: Build a Tool-Using AI Agent](https://www.dotnet-guide.com/tutorials/dotnet-ai/agent-framework-incident-triage/) | | [`dotnet-aspire/orchestrate-distributed-system`](dotnet-aspire/orchestrate-distributed-system/) | Minimal Aspire AppHost coordinating a web project and API with service discovery and startup ordering | [Aspire in .NET: Orchestrate, Run, and Deploy a Distributed System from One App Host](https://www.dotnet-guide.com/tutorials/dotnet-aspire/orchestrate-distributed-system/) | | [`software-architecture/architecture-testing-dotnet`](software-architecture/architecture-testing-dotnet/) | Minimal NetArchTest.eNhancedEdition rule that prevents Domain from depending on outer layers | [Architecture Testing in .NET: Enforce Layer and Module Boundaries with NetArchTest and ArchUnitNET](https://www.dotnet-guide.com/tutorials/software-architecture/architecture-testing-dotnet/) | | [`distributed-systems/transactional-outbox-ef-core`](distributed-systems/transactional-outbox-ef-core/) | Minimal EF Core and SQLite demonstration that saves business state and an outbox message atomically, then publishes it through a one-shot relay | [Transactional Outbox Pattern in .NET with EF Core (.NET 10): Fix the Dual-Write Problem](https://www.dotnet-guide.com/tutorials/distributed-systems/transactional-outbox-ef-core/) | @@ -68,6 +69,14 @@ dotnet restore dotnet run ``` +### Agent Framework incident triage + +```powershell +cd dotnet-ai\agent-framework-incident-triage +dotnet restore +dotnet test +``` + ## Repository structure ```text @@ -81,6 +90,14 @@ tutorials/ | |-- HybridSearchMinimal.csproj | |-- Program.cs | `-- README.md +| `-- agent-framework-incident-triage/ +| |-- AgentFrameworkIncidentTriage.slnx +| |-- README.md +| |-- global.json +| |-- verified-environment.json +| |-- docs/ +| |-- src/ +| `-- tests/ |-- dotnet-aspire/ | `-- orchestrate-distributed-system/ | |-- AspireOrchestrationMinimal.slnx From 42a555b55b1feed64f1a7499696d0d22443ffdc0 Mon Sep 17 00:00:00 2001 From: BALAJI BASHYAM Date: Sun, 27 Sep 2026 15:06:21 +0000 Subject: [PATCH 3/3] Test Agent Framework incident triage sample in GitHub Actions --- .github/workflows/build-samples.yml | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/build-samples.yml b/.github/workflows/build-samples.yml index 9f262ca..12391ca 100644 --- a/.github/workflows/build-samples.yml +++ b/.github/workflows/build-samples.yml @@ -6,6 +6,7 @@ on: paths: - "dotnet-ai/provider-agnostic-chat-gateway/**" - "dotnet-ai/hybrid-search-ef-core-pgvector/**" + - "dotnet-ai/agent-framework-incident-triage/**" - "dotnet-aspire/orchestrate-distributed-system/**" - "software-architecture/architecture-testing-dotnet/**" - "distributed-systems/transactional-outbox-ef-core/**" @@ -37,6 +38,7 @@ on: paths: - "dotnet-ai/provider-agnostic-chat-gateway/**" - "dotnet-ai/hybrid-search-ef-core-pgvector/**" + - "dotnet-ai/agent-framework-incident-triage/**" - "dotnet-aspire/orchestrate-distributed-system/**" - "software-architecture/architecture-testing-dotnet/**" - "distributed-systems/transactional-outbox-ef-core/**" @@ -2012,3 +2014,35 @@ jobs: exit 1 fi + + test-agent-framework-incident-triage: + name: Test Agent Framework incident triage sample + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v5 + + - name: Install .NET 10 SDK + uses: actions/setup-dotnet@v5 + with: + dotnet-version: "10.0.x" + + - name: Restore + run: > + dotnet restore + dotnet-ai/agent-framework-incident-triage/AgentFrameworkIncidentTriage.slnx + + - name: Build + run: > + dotnet build + dotnet-ai/agent-framework-incident-triage/AgentFrameworkIncidentTriage.slnx + --configuration Release + --no-restore + + - name: Test deterministic layers + run: > + dotnet test + dotnet-ai/agent-framework-incident-triage/AgentFrameworkIncidentTriage.slnx + --configuration Release + --no-build