From 291673b87875c09033ac51d6de934629f9a703d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20Dang=C3=A5rden?= Date: Thu, 15 Oct 2026 17:09:47 +0200 Subject: [PATCH] Improve agent recovery from the first eval run - Suggest a field whose name ends with the guessed one at a camelCase boundary (name -> customerName), the most common rejected query. - Add a hint with the server's limit to HotChocolate page-size errors (HC0051), which the SDL cannot express. - Sample: replace the 'fastest animal' question, which the schema cannot answer, and state the orders page limit in its description. --- docs/design/agent-mode.md | 4 +-- .../Schema/NameSuggestions.cs | 31 ++++++++++++++----- .../Tools/AgentToolset.cs | 20 +++++++++++- .../AgentSampleApp.cs | 2 +- test/Linq2GraphQL.Agent.Sample/questions.json | 2 +- test/Linq2GraphQL.Tests/AgentToolsetTests.cs | 20 ++++++++++++ .../Linq2GraphQL.Tests/QueryValidatorTests.cs | 2 ++ 7 files changed, 69 insertions(+), 12 deletions(-) diff --git a/docs/design/agent-mode.md b/docs/design/agent-mode.md index 2026ab74..1d095153 100644 --- a/docs/design/agent-mode.md +++ b/docs/design/agent-mode.md @@ -337,9 +337,9 @@ The design should already beat generic approaches; these measures address the we | Weak spot | What we do | | --- | --- | | **Regressions** | An **evaluation harness** as regression protection, not as the go/no-go decision (that comes from the pilots). Its questions come from real logged pilot questions, anonymised where needed, plus the developer examples. It runs against the test server whenever the renderer or validator changes, recording answer rate, tool calls and tokens, so a "better" format can't silently make things worse. It lives in the repo as a separate project, outside CI since it needs an API key. | -| **Error recovery** | All errors are returned at once, not just the first. Every error states what is wrong, where (the path in the query) and what to do instead: valid alternatives and "did you mean" suggestions for near-miss names (`custmer` → `customer`). | +| **Error recovery** | All errors are returned at once, not just the first. Every error states what is wrong, where (the path in the query) and what to do instead: valid alternatives and "did you mean" suggestions for near-miss names (`custmer` → `customer`) and for a guessed short name that ends a real one (`name` → `customerName`, the most common miss in the first eval run). | | **Filter syntax** | The exact syntax is shown once as a rule with a worked example, and `describe_operation` and `describe_type` list the operators per field type (strings: `eq`, `contains`, `startsWith`; numbers: `eq`, `gt`, `lt`). Arguments and input objects are validated in the first version, so a wrong filter gets a precise error. | -| **Paging** | Paged fields are rendered with the exact query pattern next to them. A targeted error catches the classic mistake: `customers { id }` returns "`customers` is paged. Use `customers { nodes { id } pageInfo { hasNextPage endCursor } }`". | +| **Paging** | Paged fields are rendered with the exact query pattern next to them. A targeted error catches the classic mistake: `customers { id }` returns "`customers` is paged. Use `customers { nodes { id } pageInfo { hasNextPage endCursor } }`". The page-size limit is not in the SDL, so when the server rejects a page size (HotChocolate `HC0051`) the error gets a hint with the limit from the error's extensions; operation descriptions can also state it. | | **Exploration cost on large schemas** | `describe_operation` returns a generated starter query, so the agent often needs no further exploration. `search_schema` ranks name matches above description matches and returns field paths (`Customer.invoices`), not only type names. `describe_type` also returns the types of a type's fields at Outline level, saving a round-trip per hop. Examples for common tasks (see [Public API](#public-api)) are the biggest lever, and the docs should push them. | ## Security model diff --git a/src/Linq2GraphQL.Client.AI/Schema/NameSuggestions.cs b/src/Linq2GraphQL.Client.AI/Schema/NameSuggestions.cs index c7e3d48b..1355f0a0 100644 --- a/src/Linq2GraphQL.Client.AI/Schema/NameSuggestions.cs +++ b/src/Linq2GraphQL.Client.AI/Schema/NameSuggestions.cs @@ -18,13 +18,30 @@ public static string Suggest(string name, IEnumerable candidates) return list.Count == 0 ? suggestion : $"{suggestion} Available: {List(list)}."; } - public static string Closest(string name, IEnumerable candidates) => - candidates - .Select(e => (Name: e, Distance: Distance(name.ToLowerInvariant(), e.ToLowerInvariant()))) - .Where(e => e.Distance <= Math.Max(2, name.Length / 3)) - .OrderBy(e => e.Distance) - .Select(e => e.Name) - .FirstOrDefault(); + /// + /// A near miss by spelling, or else a name that ends with the guessed one as a camelCase word: agents + /// often ask for name or id where the schema has customerName or customerId. + /// + public static string Closest(string name, IEnumerable candidates) + { + var list = candidates as IList ?? candidates.ToList(); + return list + .Select(e => (Name: e, Distance: Distance(name.ToLowerInvariant(), e.ToLowerInvariant()))) + .Where(e => e.Distance <= Math.Max(2, name.Length / 3)) + .OrderBy(e => e.Distance) + .Select(e => e.Name) + .FirstOrDefault() + ?? list + .Where(e => EndsWithWord(e, name)) + .OrderBy(e => e.Length) + .FirstOrDefault(); + } + + private static bool EndsWithWord(string candidate, string word) => + word.Length > 0 + && candidate.Length > word.Length + && candidate.EndsWith(word, StringComparison.OrdinalIgnoreCase) + && (char.IsUpper(candidate[^word.Length]) || candidate[^(word.Length + 1)] == '_'); public static string List(IEnumerable names) { diff --git a/src/Linq2GraphQL.Client.AI/Tools/AgentToolset.cs b/src/Linq2GraphQL.Client.AI/Tools/AgentToolset.cs index d0fdec41..b1fde13b 100644 --- a/src/Linq2GraphQL.Client.AI/Tools/AgentToolset.cs +++ b/src/Linq2GraphQL.Client.AI/Tools/AgentToolset.cs @@ -70,7 +70,7 @@ public async Task QueryAsync(string query, JsonElement? variables = null return JsonSerializer.Serialize(new { data = result.Data.ValueKind == JsonValueKind.Undefined ? (JsonElement?)null : result.Data, - errors = result.Errors?.Select(e => new { message = e.Message, path = e.Path }) + errors = result.Errors?.Select(e => new { message = e.Message, path = e.Path, hint = Hint(e) }) }, ResultJson); } catch (GraphQueryRequestException ex) @@ -80,6 +80,24 @@ public async Task QueryAsync(string query, JsonElement? variables = null } } + /// + /// What to do about a server error the schema cannot predict. The page-size limit is the common one: it is + /// not in the SDL, so agents guess a page size and retry with smaller guesses unless told the limit. + /// + private static string Hint(GraphQueryError error) + { + if (error.Extensions?.TryGetValue("code", out var code) != true || code?.ToString() != "HC0051") + { + return null; + } + + var max = error.Extensions.TryGetValue("maxAllowedItems", out var value) ? value?.ToString() : null; + var field = error.Extensions.TryGetValue("coordinate", out var coordinate) ? $" for {coordinate}" : ""; + return max == null + ? $"The page size{field} is limited. Leave out first/last to get the server's default page, then page with after: endCursor." + : $"At most {max} items per page{field}. Use first: {max} or leave it out, then page with after: endCursor while hasNextPage is true."; + } + /// /// Reports a query to the OnQuery loggers. A failing logger must not break the agent's call. /// diff --git a/test/Linq2GraphQL.Agent.Sample/AgentSampleApp.cs b/test/Linq2GraphQL.Agent.Sample/AgentSampleApp.cs index 30478d5b..20be6081 100644 --- a/test/Linq2GraphQL.Agent.Sample/AgentSampleApp.cs +++ b/test/Linq2GraphQL.Agent.Sample/AgentSampleApp.cs @@ -76,7 +76,7 @@ internal static IList Tools(SampleClient client, Action c.Query.Hello).Describe("Greets someone by name; a quick way to test the API."); o.AllowQuery(c => c.Query.Customers).Describe("All customers with their orders."); - o.AllowQuery(c => c.Query.Orders).Describe("Orders, paged and filterable."); + o.AllowQuery(c => c.Query.Orders).Describe("Orders, paged and filterable. At most 2 orders per page."); o.AllowQuery(c => c.Query.Animals).Describe("The animals: pigs and spiders."); o.Describe("A customer account."); o.Example("Customer statuses", "Every customer's name and status", diff --git a/test/Linq2GraphQL.Agent.Sample/questions.json b/test/Linq2GraphQL.Agent.Sample/questions.json index 518e504c..f18997cb 100644 --- a/test/Linq2GraphQL.Agent.Sample/questions.json +++ b/test/Linq2GraphQL.Agent.Sample/questions.json @@ -6,5 +6,5 @@ { "question": "Which customer has an order delivered to Garden Inc?", "expected": ["Universal Ltd"] }, { "question": "What is the price of item 12345 on Universal Ltd's orders?", "expected": ["56.4"] }, { "question": "What is the name of the spider, and how many legs does it have?", "expected": ["Imse", "8"] }, - { "question": "Which animal is the fastest?", "expected": ["Imse"] } + { "question": "Which animal has the most legs?", "expected": ["Imse"] } ] diff --git a/test/Linq2GraphQL.Tests/AgentToolsetTests.cs b/test/Linq2GraphQL.Tests/AgentToolsetTests.cs index 78ac28ca..d333ed44 100644 --- a/test/Linq2GraphQL.Tests/AgentToolsetTests.cs +++ b/test/Linq2GraphQL.Tests/AgentToolsetTests.cs @@ -85,6 +85,26 @@ public async Task Query_ServerError_IsReturnedAsErrors() result.GetProperty("errors")[0].GetProperty("message").GetString()!.ShouldContain("boom"); } + [Fact] + public async Task Query_PageSizeTooLarge_HintsTheServerLimit() + { + var result = Json(await tools.QueryAsync("{ orders(first: 100) { nodes { orderId } } }")); + + result.GetProperty("errors")[0].GetProperty("hint").GetString() + .ShouldBe("At most 2 items per page for Query.orders. Use first: 2 or leave it out, then page with after: endCursor while hasNextPage is true."); + } + + [Fact] + public async Task Query_OtherServerError_HasNoHint() + { + var tools = new AgentToolset(ScopedSchema.Create(SampleClient.Sdl, o => o.AllowQuery(c => c.Query.RaiseError)), + client.GraphClient); + + var result = Json(await tools.QueryAsync("{ raiseError(message: \"boom\") }")); + + result.GetProperty("errors")[0].TryGetProperty("hint", out _).ShouldBeFalse(); + } + [Fact] public void DescribeOperations_Snapshot() { diff --git a/test/Linq2GraphQL.Tests/QueryValidatorTests.cs b/test/Linq2GraphQL.Tests/QueryValidatorTests.cs index 6ea3f781..849f6fd6 100644 --- a/test/Linq2GraphQL.Tests/QueryValidatorTests.cs +++ b/test/Linq2GraphQL.Tests/QueryValidatorTests.cs @@ -83,6 +83,8 @@ public void Typo_SuggestsTheClosestName() { ShouldFail("{ custmers { customerName } }").ShouldContain("Did you mean `customers`?"); ShouldFail("{ customers { custmerName } }").ShouldContain("Customer has no field custmerName. Did you mean `customerName`?"); + ShouldFail("{ customers { name } }").ShouldContain("Customer has no field name. Did you mean `customerName`?"); + ShouldFail("{ customers { id } }").ShouldContain("Customer has no field id. Did you mean `customerId`?"); } [Fact]