Add the agent sample and regression harness

Directory.Packages.props

@@ -19,7 +19,13 @@
     <PackageVersion Include="HotChocolate.Types.Analyzers" Version="16.6.4" />
     <PackageVersion Include="Macross.Json.Extensions" Version="3.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly" Version="10.0.11" />
+    <PackageVersion Include="Azure.Identity" Version="1.21.0" />
+    <PackageVersion Include="Microsoft.Extensions.AI" Version="10.10.0" />
     <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.10.0" />
+    <PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.10.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.11" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.11" />
+    <PackageVersion Include="OpenAI" Version="2.14.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="10.0.11" />
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.11" />
     <PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.11" />

Linq2GraphQL.CI.slnf

@@ -7,6 +7,7 @@
       "src\\Linq2GraphQL.Client.AI\\Linq2GraphQL.Client.AI.csproj",
       "src\\Linq2GraphQL.Generator\\Linq2GraphQL.Generator.csproj",
       "test\\Linq2GraphQL.Console\\Linq2GraphQL.Console.csproj",
+      "test\\Linq2GraphQL.Agent.Sample\\Linq2GraphQL.Agent.Sample.csproj",
       "test\\Linq2GraphQL.TestClient\\Linq2GraphQL.TestClient.csproj",
       "test\\Linq2GraphQL.TestClientNullable\\Linq2GraphQL.TestClientNullable.csproj",
       "test\\Linq2GraphQL.TestServer\\Linq2GraphQL.TestServer.csproj",

Linq2GraphQL.Release.slnf

@@ -4,6 +4,7 @@
     "projects": [
       "src\\Linq2GraphQL.Client\\Linq2GraphQL.Client.csproj",
       "src\\Linq2GraphQL.Client.Subscriptions\\Linq2GraphQL.Client.Subscriptions.csproj",
+      "src\\Linq2GraphQL.Client.AI\\Linq2GraphQL.Client.AI.csproj",
       "src\\Linq2GraphQL.Generator\\Linq2GraphQL.Generator.csproj"
     ]
   }

Linq2GraphQL.sln

@@ -30,6 +30,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Linq2GraphQL.TestClientNull
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Linq2GraphQL.Console", "test\Linq2GraphQL.Console\Linq2GraphQL.Console.csproj", "{058B468F-CE72-4CFF-8847-87CEED642C4E}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Linq2GraphQL.Agent.Sample", "test\Linq2GraphQL.Agent.Sample\Linq2GraphQL.Agent.Sample.csproj", "{6E3C2B71-4D8A-4F1E-9B57-2A9C0D4E8F13}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Linq2GraphQL.Generator", "src\Linq2GraphQL.Generator\Linq2GraphQL.Generator.csproj", "{C0E1A444-A834-4FC0-8436-38066EA8D37D}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Linq2GraphQL.Client", "src\Linq2GraphQL.Client\Linq2GraphQL.Client.csproj", "{B2EAACC4-0B04-458B-B425-DE83BED807EA}"
@@ -96,6 +98,10 @@ Global
 		{058B468F-CE72-4CFF-8847-87CEED642C4E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{058B468F-CE72-4CFF-8847-87CEED642C4E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{058B468F-CE72-4CFF-8847-87CEED642C4E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6E3C2B71-4D8A-4F1E-9B57-2A9C0D4E8F13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6E3C2B71-4D8A-4F1E-9B57-2A9C0D4E8F13}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6E3C2B71-4D8A-4F1E-9B57-2A9C0D4E8F13}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6E3C2B71-4D8A-4F1E-9B57-2A9C0D4E8F13}.Release|Any CPU.Build.0 = Release|Any CPU
 		{C0E1A444-A834-4FC0-8436-38066EA8D37D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C0E1A444-A834-4FC0-8436-38066EA8D37D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C0E1A444-A834-4FC0-8436-38066EA8D37D}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -132,6 +138,7 @@ Global
 		{F9EAFD31-9AF1-47DF-9C78-003C7AC67C5D} = {5BE1963B-E57D-4CF5-B03E-644D8F39B4D2}
 		{C1908816-34E7-4B41-9E39-2283C59DB017} = {5BE1963B-E57D-4CF5-B03E-644D8F39B4D2}
 		{058B468F-CE72-4CFF-8847-87CEED642C4E} = {5BE1963B-E57D-4CF5-B03E-644D8F39B4D2}
+		{6E3C2B71-4D8A-4F1E-9B57-2A9C0D4E8F13} = {5BE1963B-E57D-4CF5-B03E-644D8F39B4D2}
 		{C0E1A444-A834-4FC0-8436-38066EA8D37D} = {F646B02D-53FC-4C88-8941-2255A2F4DAF6}
 		{A941BD4C-8B04-4273-AC0B-CF86D71D7E0B} = {884758A6-B667-4CA0-B186-AFB6331A24A5}
 		{3C66EF43-E66D-46DE-B2BB-434E7FC20487} = {884758A6-B667-4CA0-B186-AFB6331A24A5}

README.md

@@ -237,6 +237,44 @@ var launches = await userClient.Query.Launches().Select().ExecuteAsync();
 `WithAccessToken` sets `Authorization: Bearer <token>` and asks the delegate for the token on every request;
 `WithRequestConfiguration` takes any hook.
 
+## Agent mode (preview)
+
+`Linq2GraphQL.Client.AI` lets an AI agent query your API in plain GraphQL, limited to the operations you
+allow. Generate the client with `--agent` (`-ag`) so it carries the schema, then turn it into tools for
+Microsoft.Extensions.AI:
+
+```cs
+var tools = spaceXClient.AsAgentTools(o =>
+{
+    o.AllowQuery(c => c.Query.Launches);
+    o.AllowQuery(c => c.Query.Rockets);
+    o.Describe<Launch>(l => l.Details, "Free-text mission summary; may be empty.");
+    o.Instructions("Dates are UTC.");
+    o.AccessToken = ct => tokens.GetAccessTokenForUserAsync(user, ct);
+    o.OnQuery(log => logger.LogInformation("{Query} {Errors}", log.Query, log.ValidationErrors));
+});
+
+var response = await chatClient.GetResponseAsync(question, new ChatOptions { Tools = [..tools] });
+```
+
+The agent gets four tools: `search_schema`, `describe_operation`, `describe_type` and `graphql_query`. It
+sees only the allowed operations and the types they reach, and every query is validated against that
+scope before it is sent. The server stays responsible for authorization; give the agent a down-scoped or
+on-behalf-of token.
+
+`test/Linq2GraphQL.Agent.Sample` runs an agent against the test server. It uses Azure AI Foundry through
+your Azure sign-in (`az login`, no keys), or any OpenAI-compatible endpoint such as Ollama:
+
+```powershell
+dotnet user-secrets --project test/Linq2GraphQL.Agent.Sample set Agent:Endpoint "https://<resource>.cognitiveservices.azure.com/"
+dotnet user-secrets --project test/Linq2GraphQL.Agent.Sample set Agent:Deployment "<deployment>"
+dotnet run --project test/Linq2GraphQL.Agent.Sample            # chat
+dotnet run --project test/Linq2GraphQL.Agent.Sample -- eval    # run questions.json and report
+```
+
+For Ollama, set `Agent:Provider` to `OpenAICompatible` and `Agent:Endpoint` to `http://localhost:11434/v1/`.
+Add `--verbose` to see each tool call. See [the design doc](docs/design/agent-mode.md) for the full picture.
+
 ## Safe Mode
 
 Turning on *SafeMode* will make the client before the first request to do an introspection query to the endpoint. The

docs/design/agent-mode.md

@@ -1,6 +1,6 @@
 # Linq2GraphQL Agent Mode Design
 
-**Status:** in progress. Plan steps 1-3 (the groundwork, SDL input and SDL output) are implemented. The AI package is being built in slices: (1) package skeleton and scoped schema, (2) renderer, (3) validator, (4) the four tools, (5) the `AsAgentTools` adapter, (6) sample and regression harness. Slices 1 to 4 are merged; slice 5 (the adapter) is in review. See [Plan](#plan-and-risks).
+**Status:** in progress. Plan steps 1-3 (the groundwork, SDL input and SDL output) are implemented. The AI package is being built in slices: (1) package skeleton and scoped schema, (2) renderer, (3) validator, (4) the four tools, (5) the `AsAgentTools` adapter, (6) sample and regression harness. Slices 1 to 5 are merged; slice 6 (the sample and regression harness) is in review. See [Plan](#plan-and-risks).
 
 ## Summary
 
@@ -420,7 +420,7 @@ Subscriptions stay out entirely; they don't fit the request/response shape of to
 1. **Core prerequisites** (done): the per-request hook and the schema cache-key fix.
 2. **SDL input in the generator** (done): it introduced GraphQL-Parser.
 3. **SDL output** (done): the `--agent` flag, `IGraphSchemaSource`, and derived clients for per-user tokens. `[Description]` attributes and the two extra introspection fields shipped with step 2.
-4. **Preview and pilots:** the first version as a prerelease package (`-preview`), complete enough to run end to end, with a runnable sample against the test server. Usefulness depends on the use case, so it is evaluated in one or two pilot apps with a real schema and real users, ideally different ones (interactive and background, small and large schema). The evaluation is part of the product: `OnQuery` logs (questions, queries, validation errors, tool calls per answer), simple user feedback per answer in the pilot app, and which questions the team moves to dedicated tools. The pilots can start before step 3 by using an exported `schema.graphql`.
+4. **Preview and pilots:** the first version as a prerelease package (`-preview`), complete enough to run end to end, with a runnable sample against the test server. Usefulness depends on the use case, so it is evaluated in one or two pilot apps with a real schema and real users, ideally different ones (interactive and background, small and large schema). The evaluation is part of the product: `OnQuery` logs (questions, queries, validation errors, tool calls per answer), simple user feedback per answer in the pilot app, and which questions the team moves to dedicated tools. The pilots can start before step 3 by using an exported `schema.graphql`. The sample (`test/Linq2GraphQL.Agent.Sample`) runs a chat loop against the in-process test server, with Azure AI Foundry (Entra ID, no keys) or any OpenAI-compatible endpoint such as Ollama, and an `eval` mode that runs a question set and reports pass rate, tool calls, rejected queries and tokens; it is the regression harness for prompt and tool changes. CI runs the loop without a model: a scripted chat client replays tool calls through Microsoft.Extensions.AI's function invocation (`AgentLoopTests`).
 5. **Decide from the pilots:** did the teams get a working agent quickly, were users satisfied with the answers, where did it fail (filters, paging, large schemas), and what did they specialise? If it holds up, release `Linq2GraphQL.Client.AI`. If not, the core and generator improvements still stand.
 
 **Risks**

test/Linq2GraphQL.Agent.Sample/AgentSampleApp.cs

@@ -0,0 +1,205 @@
+using System.Diagnostics;
+using System.Text.Json;
+using Linq2GraphQL.Client;
+using Linq2GraphQL.Client.AI;
+using Linq2GraphQL.TestClient;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+
+namespace Linq2GraphQL.Agent.Sample;
+
+/// <summary>
+/// A small agent over the test GraphQL server, built with AsAgentTools.
+///   chat                 talk to the agent (the default)
+///   eval [file]          run the questions in questions.json and report answer rate, tool calls and tokens
+/// The test server runs in-process unless Agent:ServerUrl points at a running one.
+/// </summary>
+public static class AgentSampleApp
+{
+    private const string SystemPrompt =
+        "You answer questions about the data behind a GraphQL API, using the tools. If you are unsure which " +
+        "operation or field to use, call describe_operation, describe_type or search_schema before querying. " +
+        "If a query is rejected, fix every error listed and try again. Answer briefly, with the values you found.";
+
+    public static async Task<int> Main(string[] args)
+    {
+        var configuration = new ConfigurationBuilder()
+            .SetBasePath(AppContext.BaseDirectory)
+            .AddJsonFile("appsettings.json", optional: true)
+            .AddUserSecrets(typeof(AgentSampleApp).Assembly, optional: true)
+            .AddEnvironmentVariables("LINQ2GRAPHQL_")
+            .Build();
+
+        var settings = AgentSettings.From(configuration, out var problem);
+        if (settings == null)
+        {
+            Console.Error.WriteLine(problem);
+            Console.Error.WriteLine("""
+
+                Set the model to use, for example with user secrets (stored outside the repository):
+
+                  dotnet user-secrets --project test/Linq2GraphQL.Agent.Sample set Agent:Endpoint "https://<resource>.cognitiveservices.azure.com/"
+                  dotnet user-secrets --project test/Linq2GraphQL.Agent.Sample set Agent:Deployment "<deployment>"
+
+                Azure AI Foundry signs in with your own account (az login or Visual Studio); no key is needed.
+                For another OpenAI-compatible endpoint, such as Ollama, also set Agent:Provider to
+                OpenAICompatible and, if the endpoint needs one, Agent:ApiKey.
+                Environment variables work too: LINQ2GRAPHQL_Agent__Endpoint and so on.
+                """);
+            return 1;
+        }
+
+        using var server = StartServer(configuration["Agent:ServerUrl"], out var httpClient);
+        var client = new SampleClient(httpClient, Options.Create(new GraphClientOptions()),
+            new ServiceCollection().AddMemoryCache().BuildServiceProvider());
+        var chat = ChatClientFactory.Create(settings);
+        var verbose = args.Contains("--verbose");
+
+        var mode = args.FirstOrDefault(e => !e.StartsWith("--")) ?? "chat";
+        return mode switch
+        {
+            "chat" => await ChatAsync(chat, client, verbose),
+            "eval" => await EvalAsync(chat, client, args.Skip(1).FirstOrDefault(e => !e.StartsWith("--")) ?? "questions.json", verbose),
+            _ => Usage()
+        };
+    }
+
+    /// <summary>
+    /// The agent's scope over the test schema. Everything the agent sees comes from here.
+    /// </summary>
+    internal static IList<AIFunction> Tools(SampleClient client, Action<AgentQueryLog>? log) => client.AsAgentTools(o =>
+    {
+        o.Instructions("Dates are UTC.");
+        o.AllowQuery(c => c.Query.Hello).Describe("Greets someone by name; a quick way to test the API.");
+        o.AllowQuery(c => c.Query.Customers).Describe("All customers with their orders.");
+        o.AllowQuery(c => c.Query.Orders).Describe("Orders, paged and filterable.");
+        o.AllowQuery(c => c.Query.Animals).Describe("The animals: pigs and spiders.");
+        o.Describe<Customer>("A customer account.");
+        o.Example("Customer statuses", "Every customer's name and status",
+            c => c.Query.Customers().Select(e => e.Select(x => new { x.CustomerName, x.Status })));
+        if (log != null)
+        {
+            o.OnQuery(log);
+        }
+    });
+
+    private static async Task<int> ChatAsync(IChatClient chat, SampleClient client, bool verbose)
+    {
+        var tools = Tools(client, verbose ? PrintQuery : null);
+        var options = new ChatOptions { Tools = [..tools] };
+        var history = new List<ChatMessage> { new(ChatRole.System, SystemPrompt) };
+
+        Console.WriteLine("Ask about customers, orders or animals. An empty line ends the chat.");
+        while (true)
+        {
+            Console.Write("\n> ");
+            var question = Console.ReadLine();
+            if (string.IsNullOrWhiteSpace(question))
+            {
+                return 0;
+            }
+
+            history.Add(new ChatMessage(ChatRole.User, question));
+            var response = await chat.GetResponseAsync(history, options);
+            history.AddMessages(response);
+            Console.WriteLine(response.Text);
+        }
+    }
+
+    private static async Task<int> EvalAsync(IChatClient chat, SampleClient client, string file, bool verbose)
+    {
+        var path = Path.IsPathRooted(file) ? file : Path.Combine(AppContext.BaseDirectory, file);
+        var questions = JsonSerializer.Deserialize<List<EvalQuestion>>(await File.ReadAllTextAsync(path),
+            new JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? [];
+
+        var results = new List<EvalResult>();
+        foreach (var question in questions)
+        {
+            var queries = new List<AgentQueryLog>();
+            var tools = Tools(client, log =>
+            {
+                queries.Add(log);
+                if (verbose)
+                {
+                    PrintQuery(log);
+                }
+            });
+
+            var watch = Stopwatch.StartNew();
+            var response = await chat.GetResponseAsync(
+                [new ChatMessage(ChatRole.System, SystemPrompt), new ChatMessage(ChatRole.User, question.Question)],
+                new ChatOptions { Tools = [..tools] });
+
+            var answer = response.Text;
+            var result = new EvalResult(
+                question.Question,
+                question.Expected.All(e => answer.Contains(e, StringComparison.OrdinalIgnoreCase)),
+                response.Messages.SelectMany(e => e.Contents).OfType<FunctionCallContent>().Count(),
+                queries.Count(e => !e.WasSent),
+                response.Usage?.TotalTokenCount ?? 0,
+                watch.Elapsed,
+                answer);
+            results.Add(result);
+
+            Console.WriteLine($"{(result.Correct ? "PASS" : "FAIL")}  tools {result.ToolCalls,2}  rejected {result.RejectedQueries}  " +
+                              $"tokens {result.Tokens,6}  {result.Duration.TotalSeconds,5:0.0}s  {result.Question}");
+            if (!result.Correct)
+            {
+                Console.WriteLine($"      expected: {string.Join(", ", question.Expected)}");
+                Console.WriteLine($"      answer:   {answer.ReplaceLineEndings(" ")}");
+            }
+        }
+
+        Console.WriteLine();
+        Console.WriteLine($"Answered {results.Count(e => e.Correct)} of {results.Count} correctly; " +
+                          $"{results.Average(e => e.ToolCalls):0.0} tool calls, " +
+                          $"{results.Average(e => e.Tokens):0} tokens and " +
+                          $"{results.Average(e => e.Duration.TotalSeconds):0.0}s per question on average.");
+
+        var report = Path.Combine(AppContext.BaseDirectory, $"eval-{DateTime.UtcNow:yyyyMMdd-HHmmss}.json");
+        await File.WriteAllTextAsync(report, JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true }));
+        Console.WriteLine($"Details: {report}");
+
+        return results.All(e => e.Correct) ? 0 : 2;
+    }
+
+    private static IDisposable StartServer(string? serverUrl, out HttpClient httpClient)
+    {
+        if (!string.IsNullOrWhiteSpace(serverUrl))
+        {
+            httpClient = new HttpClient { BaseAddress = new Uri(serverUrl) };
+            return httpClient;
+        }
+
+        var application = new WebApplicationFactory<Program>();
+        httpClient = application.CreateClient(new() { BaseAddress = new Uri("https://localhost/graphql/") });
+        return application;
+    }
+
+    private static void PrintQuery(AgentQueryLog log)
+    {
+        var color = Console.ForegroundColor;
+        Console.ForegroundColor = ConsoleColor.DarkGray;
+        Console.WriteLine($"  [{(log.WasSent ? "sent" : "rejected")}] {log.Query.ReplaceLineEndings(" ")}");
+        foreach (var error in log.ValidationErrors.Concat(log.Errors))
+        {
+            Console.WriteLine($"    {error}");
+        }
+
+        Console.ForegroundColor = color;
+    }
+
+    private static int Usage()
+    {
+        Console.Error.WriteLine("Usage: Linq2GraphQL.Agent.Sample [chat | eval [questions.json]] [--verbose]");
+        return 1;
+    }
+
+    private sealed record EvalQuestion(string Question, string[] Expected);
+
+    private sealed record EvalResult(string Question, bool Correct, int ToolCalls, int RejectedQueries, long Tokens,
+        TimeSpan Duration, string Answer);
+}