diff --git a/src/core/analytics.test.ts b/src/core/analytics.test.ts index 4b9b87a..21c6a4b 100644 --- a/src/core/analytics.test.ts +++ b/src/core/analytics.test.ts @@ -1,6 +1,24 @@ import fs from "fs"; import path from "path"; -import { TOOL_ARG_ALLOWLIST } from "./analytics"; +import { TOOL_ARG_ALLOWLIST, withToolAnalytics } from "./analytics"; +import { ResponseBuilder } from "./response-builder"; + +describe("tool error flag", () => { + const text = (t: string) => ({ content: [{ type: "text" as const, text: t }] }); + beforeAll(() => { + process.env.MCP_ANALYTICS_DISABLED = "1"; + }); + + it.each([ + ["ResponseBuilder.error", () => ResponseBuilder.error("Boom", "details"), true], + ["a hand-built ❌ answer", () => text("❌ **Error:** boom"), true], + ["a success answer", () => ResponseBuilder.success("done"), undefined], + ["markdown that mentions ❌ later", () => text("# Report\n\n❌ 1 failed call"), undefined], + ])("%s → isError %s", async (_label, build, expected) => { + const result = await withToolAnalytics("localstack-docs", {}, async () => build()); + expect((result as { isError?: boolean }).isError).toBe(expected); + }); +}); describe("analytics allowlist coverage", () => { it("has allowlist entries for all tool metadata names", () => { diff --git a/src/core/analytics.ts b/src/core/analytics.ts index e956ed2..85706c4 100644 --- a/src/core/analytics.ts +++ b/src/core/analytics.ts @@ -276,6 +276,9 @@ export async function withToolAnalytics( const isErrorResponse = isErrorLikeToolResponse(result); success = !isErrorResponse; if (isErrorResponse) { + // Some tools build ❌ answers by hand instead of with ResponseBuilder.error; every tool + // passes through here, so flag those too. + (result as { isError?: boolean }).isError = true; errorName = "ToolResponseError"; errorMessage = extractErrorMessageFromResult(result) || "Tool returned an error response"; } diff --git a/src/core/response-builder.ts b/src/core/response-builder.ts index 1d3f193..eac76bd 100644 --- a/src/core/response-builder.ts +++ b/src/core/response-builder.ts @@ -1,5 +1,7 @@ interface ToolResponse { content: Array<{ type: "text"; text: string }>; + /** MCP's flag for a failed tool call; clients and test harnesses read it, not the ❌ prefix. */ + isError?: boolean; } export class ResponseBuilder { @@ -16,6 +18,7 @@ export class ResponseBuilder { } return { content: [{ type: "text", text }], + isError: true, }; } diff --git a/tests/mcp/azure-offline.spec.mjs b/tests/mcp/azure-offline.spec.mjs index 9653614..e62be05 100644 --- a/tests/mcp/azure-offline.spec.mjs +++ b/tests/mcp/azure-offline.spec.mjs @@ -21,6 +21,7 @@ test("shell syntax is refused before anything runs", async ({ mcp }) => { const result = await mcp.callTool("localstack-azure-client", { command: "group list; echo pwned", }); + expect(result).toBeToolError(); expect(text(result)).toMatch(/^❌ /); expect(text(result)).toMatch(/forbidden shell syntax/); }); @@ -36,7 +37,9 @@ test("no emulator on the configured port: the answer names the gateway and the s }) => { // A missing az is reported first; CI runners have it. test.skip(!azRuns, "needs the Azure CLI on PATH"); - const answer = text(await mcp.callTool("localstack-azure-client", { command: "group list" })); + const result = await mcp.callTool("localstack-azure-client", { command: "group list" }); + expect(result).toBeToolError(); + const answer = text(result); expect(answer).toMatch(/^❌ \*\*LocalStack Azure Emulator Not Ready\*\*/); expect(answer).toMatch(/not running at http:\/\/[^\s:]+:\d+/); expect(answer).toContain("service: azure");