Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion src/core/analytics.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,24 @@
import fs from "fs";
import path from "path";
import { TOOL_ARG_ALLOWLIST } from "./analytics";
import { TOOL_ARG_ALLOWLIST, withToolAnalytics } from "./analytics";
import { ResponseBuilder } from "./response-builder";

describe("tool error flag", () => {
const text = (t: string) => ({ content: [{ type: "text" as const, text: t }] });
beforeAll(() => {
process.env.MCP_ANALYTICS_DISABLED = "1";
});

it.each([
["ResponseBuilder.error", () => ResponseBuilder.error("Boom", "details"), true],
["a hand-built ❌ answer", () => text("❌ **Error:** boom"), true],
["a success answer", () => ResponseBuilder.success("done"), undefined],
["markdown that mentions ❌ later", () => text("# Report\n\n❌ 1 failed call"), undefined],
])("%s → isError %s", async (_label, build, expected) => {
const result = await withToolAnalytics("localstack-docs", {}, async () => build());
expect((result as { isError?: boolean }).isError).toBe(expected);
});
});

describe("analytics allowlist coverage", () => {
it("has allowlist entries for all tool metadata names", () => {
Expand Down
3 changes: 3 additions & 0 deletions src/core/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,9 @@ export async function withToolAnalytics<T>(
const isErrorResponse = isErrorLikeToolResponse(result);
success = !isErrorResponse;
if (isErrorResponse) {
// Some tools build ❌ answers by hand instead of with ResponseBuilder.error; every tool
// passes through here, so flag those too.
(result as { isError?: boolean }).isError = true;
errorName = "ToolResponseError";
errorMessage = extractErrorMessageFromResult(result) || "Tool returned an error response";
}
Expand Down
3 changes: 3 additions & 0 deletions src/core/response-builder.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
interface ToolResponse {
content: Array<{ type: "text"; text: string }>;
/** MCP's flag for a failed tool call; clients and test harnesses read it, not the ❌ prefix. */
isError?: boolean;
}

export class ResponseBuilder {
Expand All @@ -16,6 +18,7 @@ export class ResponseBuilder {
}
return {
content: [{ type: "text", text }],
isError: true,
};
}

Expand Down
5 changes: 4 additions & 1 deletion tests/mcp/azure-offline.spec.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ test("shell syntax is refused before anything runs", async ({ mcp }) => {
const result = await mcp.callTool("localstack-azure-client", {
command: "group list; echo pwned",
});
expect(result).toBeToolError();
expect(text(result)).toMatch(/^❌ /);
expect(text(result)).toMatch(/forbidden shell syntax/);
});
Expand All @@ -36,7 +37,9 @@ test("no emulator on the configured port: the answer names the gateway and the s
}) => {
// A missing az is reported first; CI runners have it.
test.skip(!azRuns, "needs the Azure CLI on PATH");
const answer = text(await mcp.callTool("localstack-azure-client", { command: "group list" }));
const result = await mcp.callTool("localstack-azure-client", { command: "group list" });
expect(result).toBeToolError();
const answer = text(result);
expect(answer).toMatch(/^❌ \*\*LocalStack Azure Emulator Not Ready\*\*/);
expect(answer).toMatch(/not running at http:\/\/[^\s:]+:\d+/);
expect(answer).toContain("service: azure");
Expand Down
Loading