From eb4bc792b67c4960f5822b040dc5c5962caf9074 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 7 Nov 2026 12:16:24 +0100 Subject: [PATCH] feat(ai): list_maple_feedback, the Maple team's cross-org reader for agent feedback Agents file feedback through send_maple_feedback and POST /v2/agent_feedback, but nothing let the Maple team read it across orgs. AgentFeedbackService.triage reads every org's reports and refuses any caller org but MAPLE_INTERNAL_ORG_ID (fails closed when unset). The MCP tool also refuses Maple's own agents and the bot, so customer feedback text only reaches a person or API key on the /mcp transport. --- apps/ai/src/mcp/tool-descriptors.generated.ts | 1 + .../__tests__/list-maple-feedback.test.ts | 124 ++++++++++++++ apps/ai/src/mcp/tools/list-maple-feedback.ts | 152 ++++++++++++++++++ apps/ai/src/mcp/tools/registry.ts | 2 + apps/ai/src/worker.ts | 2 + packages/backend/src/platform/Env.ts | 3 + .../feedback/AgentFeedbackService.test.ts | 127 ++++++++++++++- .../services/feedback/AgentFeedbackService.ts | 65 +++++++- packages/domain/src/http/agent-feedback.ts | 9 ++ packages/domain/src/mcp-outputs/catalog.ts | 3 +- packages/domain/src/mcp-outputs/feedback.ts | 32 ++++ 11 files changed, 512 insertions(+), 8 deletions(-) create mode 100644 apps/ai/src/mcp/tools/__tests__/list-maple-feedback.test.ts create mode 100644 apps/ai/src/mcp/tools/list-maple-feedback.ts diff --git a/apps/ai/src/mcp/tool-descriptors.generated.ts b/apps/ai/src/mcp/tool-descriptors.generated.ts index 90be69ca43..c526ed06e7 100644 --- a/apps/ai/src/mcp/tool-descriptors.generated.ts +++ b/apps/ai/src/mcp/tool-descriptors.generated.ts @@ -125,6 +125,7 @@ export const MCP_TOOL_DESCRIPTORS: ReadonlyArray = [ {"name":"list_error_issue_events","title":"List Error Issue Events","description":"List the audit-log events for an issue (state transitions, claims, comments, agent notes, fix proposals) in reverse-chronological order.","inputSchema":{"type":"object","properties":{"issue_id":{"type":"string","description":"The error issue ID from list_error_issues (a fingerprint from find_errors is not an issue id): its UUID or its REST id (iss_\u2026)"},"limit":{"type":"number","description":"Max events to return (default 100, max 500)"}},"required":["issue_id"],"additionalProperties":false},"outputSchema":{"type":"object","properties":{"issueId":{"type":"string"},"events":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"type":{"type":"string"},"fromState":{"anyOf":[{"type":"string"},{"type":"null"}]},"toState":{"anyOf":[{"type":"string"},{"type":"null"}]},"actorId":{"anyOf":[{"type":"string"},{"type":"null"}]},"createdAt":{"type":"string"},"payload":{"type":"object"},"actor":{"type":"string"}},"required":["id","type","fromState","toState","actorId","createdAt","payload","actor"],"additionalProperties":true}},"total":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"limit":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]}},"required":["issueId","events","total","limit"],"additionalProperties":true},"annotations":{"title":"List Error Issue Events","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}}, {"name":"register_agent","title":"Register Agent","description":"Register an LLM agent with the error-issue system so it can claim and transition issues. Must be called from a human session (not an agent API key). Returns an actor ID to pin via API-key metadata.","inputSchema":{"type":"object","properties":{"name":{"type":"string","description":"Unique agent name within the org (1..100 chars)"},"model":{"type":"string","description":"Model identifier, e.g. 'claude-opus-4.7'"},"capabilities_json":{"type":"string","contentMediaType":"application/json","contentSchema":{"type":"array","items":{"type":"string"}},"description":"JSON array of capability tags, e.g. [\"auto-triage\",\"patch-proposer\"]"}},"required":["name"],"additionalProperties":false},"outputSchema":{"type":"object","properties":{"id":{"type":"string"},"agentName":{"anyOf":[{"type":"string"},{"type":"null"}]},"model":{"anyOf":[{"type":"string"},{"type":"null"}]},"capabilities":{"type":"array","items":{"type":"string"}}},"required":["id","agentName","model","capabilities"],"additionalProperties":true},"annotations":{"title":"Register Agent","readOnlyHint":false,"destructiveHint":false,"idempotentHint":false,"openWorldHint":false}}, {"name":"send_maple_feedback","title":"Send Maple Feedback","description":"Send feedback about Maple itself (this MCP server, its tools, API, or docs) to the Maple team: a bug you hit, a tool or capability you were missing, a misleading description, or something that worked well. Use it when Maple got in the way of your task, not for problems in the user's own services. The report goes to the Maple team and is visible to the user's organization, so ask the user first with a one-line summary, unless their instructions already allow sending Maple feedback. Describe Maple's behaviour, not the user's data: leave out secrets, customer data, and raw log or span contents.","inputSchema":{"type":"object","properties":{"kind":{"type":"string","enum":["bug","feature_request","improvement","documentation","praise","other"],"description":"bug (broken or wrong), feature_request (missing capability), improvement (works but could be better), documentation (a description or doc misled you), praise, or other"},"summary":{"type":"string","description":"One line: what happened or what you want (max 200 chars)"},"reason":{"type":"string","description":"Why you are sending this: what you were trying to do, what got in the way, and why it matters (max 4000 chars)"},"details":{"type":"string","description":"Reproduction steps, the call you made, expected versus actual (max 10000 chars)"},"impact":{"type":"string","enum":["blocking","degraded","minor"],"description":"blocking (could not finish), degraded (finished with a workaround), or minor"},"related_to":{"type":"string","description":"The tool, API endpoint, or page this is about, e.g. 'search_traces'"},"agent_type":{"type":"string","enum":["coding_agent","chat_assistant","autonomous_agent","ci","other"],"description":"coding_agent (Claude Code, Cursor, Codex), chat_assistant, autonomous_agent, ci, or other"},"agent_name":{"type":"string","description":"Your client or agent name, e.g. 'claude-code'"},"model":{"type":"string","description":"The model you run on, e.g. 'claude-opus-5-5'"},"agent_version":{"type":"string","description":"Your client or agent version"}},"required":["kind","summary","reason","agent_type"],"additionalProperties":false},"outputSchema":{"type":"object","properties":{"id":{"type":"string"},"kind":{"type":"string"},"impact":{"anyOf":[{"type":"string"},{"type":"null"}]},"summary":{"type":"string"},"agentType":{"type":"string"},"agentName":{"anyOf":[{"type":"string"},{"type":"null"}]},"agentModel":{"anyOf":[{"type":"string"},{"type":"null"}]},"createdAt":{"type":"string"}},"required":["id","kind","impact","summary","agentType","agentName","agentModel","createdAt"],"additionalProperties":true},"annotations":{"title":"Send Maple Feedback","readOnlyHint":false,"destructiveHint":false,"idempotentHint":false,"openWorldHint":false}}, + {"name":"list_maple_feedback","title":"List Maple Feedback","description":"Maple team only: read the feedback agents sent about Maple (send_maple_feedback, POST /v2/agent_feedback) across every organization, newest first, with the full reason and details. Refused for any other organization.","inputSchema":{"type":"object","properties":{"kind":{"type":"string","enum":["bug","feature_request","improvement","documentation","praise","other"],"description":"Only this kind of feedback"},"impact":{"type":"string","enum":["blocking","degraded","minor"],"description":"Only this impact"},"start_time":{"type":"string","description":"Only feedback sent at or after this time, UTC (YYYY-MM-DD HH:mm:ss or ISO 8601)"},"related_to":{"type":"string","description":"Only feedback whose related tool, endpoint or page contains this text (case-insensitive)"},"org_id":{"type":"string","description":"Only feedback from this organization (`org_\u2026`)"},"limit":{"type":"number","description":"Max reports to return (default 50, max 200)"}},"additionalProperties":false},"outputSchema":{"type":"object","properties":{"reports":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"orgId":{"type":"string"},"userId":{"type":"string"},"kind":{"type":"string"},"impact":{"anyOf":[{"type":"string"},{"type":"null"}]},"summary":{"type":"string"},"reason":{"type":"string"},"details":{"anyOf":[{"type":"string"},{"type":"null"}]},"relatedTo":{"anyOf":[{"type":"string"},{"type":"null"}]},"agentType":{"type":"string"},"agentName":{"anyOf":[{"type":"string"},{"type":"null"}]},"agentModel":{"anyOf":[{"type":"string"},{"type":"null"}]},"agentVersion":{"anyOf":[{"type":"string"},{"type":"null"}]},"source":{"type":"string"},"createdAt":{"type":"string"}},"required":["id","orgId","userId","kind","impact","summary","reason","details","relatedTo","agentType","agentName","agentModel","agentVersion","source","createdAt"],"additionalProperties":true}},"limit":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"filters":{"type":"object","properties":{"kind":{"type":"string"},"impact":{"type":"string"},"startTime":{"type":"string"},"relatedTo":{"type":"string"},"orgId":{"type":"string"}},"additionalProperties":true}},"required":["reports","limit","filters"],"additionalProperties":true},"annotations":{"title":"List Maple Feedback","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}}, {"name":"list_error_incidents","title":"List Error Incidents","description":"Error incidents: flare-ups of an error issue (alert-rule breaches are list_alert_incidents). An issue gets a 'first_seen' incident when it opens and a 'regression' incident when it fires after being resolved; incidents auto-resolve after ~30 min of silence. No time window: without issue_id, every currently open incident in the org (newest 500); with it, that issue's incidents, open and resolved (newest 200).","inputSchema":{"type":"object","properties":{"issue_id":{"type":"string","description":"Only this issue's incidents, resolved ones included. Omit for open incidents org-wide.: its UUID or its REST id (iss_\u2026)"}},"additionalProperties":false},"outputSchema":{"type":"object","properties":{"incidents":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"issueId":{"type":"string"},"status":{"type":"string"},"reason":{"type":"string"},"firstTriggeredAt":{"type":"string"},"lastTriggeredAt":{"type":"string"},"resolvedAt":{"anyOf":[{"type":"string"},{"type":"null"}]},"occurrenceCount":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]}},"required":["id","issueId","status","reason","firstTriggeredAt","lastTriggeredAt","resolvedAt","occurrenceCount"],"additionalProperties":true}},"total":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"openCount":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"issueId":{"type":"string"}},"required":["incidents","total","openCount"],"additionalProperties":true},"annotations":{"title":"List Error Incidents","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}}, {"name":"get_error_notification_policy","title":"Get Error Notification Policy","description":"Read the org-wide error notification policy: whether it is on, which alert destinations it delivers to (name, UUID and REST id), which events notify, min_occurrence_count and the occurrence count a new error's first notification actually waits for. update_error_notification_policy changes it.","inputSchema":{"type":"object","properties":{},"additionalProperties":false},"outputSchema":{"type":"object","properties":{"enabled":{"type":"boolean"},"destinationIds":{"type":"array","items":{"type":"string"}},"destinationNamesUnavailable":{"type":"boolean"},"destinations":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"publicId":{"type":"string"},"name":{"anyOf":[{"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"type":"string"},{"type":"null"}]},"enabled":{"type":"boolean"}},"required":["id","publicId","name","type","enabled"],"additionalProperties":true}},"notifyOnFirstSeen":{"type":"boolean"},"notifyOnRegression":{"type":"boolean"},"notifyOnResolve":{"type":"boolean"},"minOccurrenceCount":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"firstNotificationThreshold":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"severity":{"type":"string"},"updatedAt":{"type":"string"},"updatedBy":{"type":"string"}},"required":["enabled","destinationIds","destinations","notifyOnFirstSeen","notifyOnRegression","notifyOnResolve","minOccurrenceCount","firstNotificationThreshold","severity","updatedAt","updatedBy"],"additionalProperties":true},"annotations":{"title":"Get Error Notification Policy","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}}, {"name":"update_error_notification_policy","title":"Update Error Notification Policy","description":"Configure the org-wide error notification policy: whether error incidents (first seen, regression, auto-resolve) are delivered to alert destinations, and which ones. Org admins and engineers only. Omit a field to leave it unchanged; get_error_notification_policy reads the current policy. A new error becomes an issue (and can notify) only after 3 occurrences; min_occurrence_count cannot lower that.","inputSchema":{"type":"object","properties":{"enabled":{"type":"boolean","description":"Enable notifications overall"},"destination_ids":{"type":"array","items":{"type":"string"},"description":"Alert destinations (from list_alert_destinations) to deliver to, replacing the current ones. Pass [] to clear them. Each is its UUID or its REST id (dest_\u2026)"},"notify_on_first_seen":{"type":"boolean","description":"Notify when a new error becomes an issue (after its 3rd occurrence)"},"notify_on_regression":{"type":"boolean","description":"Notify when a resolved issue re-occurs"},"notify_on_resolve":{"type":"boolean","description":"Notify when an incident is auto-resolved after the silence window"},"min_occurrence_count":{"type":"number","description":"Only notify when an incident opens with at least this many occurrences, checked once when it opens. New errors already need 3 to become an issue, so values of 3 or less only affect regressions"},"severity":{"type":"string","enum":["warning","critical"],"description":"Severity label carried on the notifications"}},"additionalProperties":false},"outputSchema":{"type":"object","properties":{"enabled":{"type":"boolean"},"destinationIds":{"type":"array","items":{"type":"string"}},"destinationNamesUnavailable":{"type":"boolean"},"destinations":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"publicId":{"type":"string"},"name":{"anyOf":[{"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"type":"string"},{"type":"null"}]},"enabled":{"type":"boolean"}},"required":["id","publicId","name","type","enabled"],"additionalProperties":true}},"notifyOnFirstSeen":{"type":"boolean"},"notifyOnRegression":{"type":"boolean"},"notifyOnResolve":{"type":"boolean"},"minOccurrenceCount":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"firstNotificationThreshold":{"anyOf":[{"type":"number"},{"type":"string","enum":["Infinity","-Infinity","NaN"]}]},"severity":{"type":"string"},"updatedAt":{"type":"string"},"updatedBy":{"type":"string"}},"required":["enabled","destinationIds","destinations","notifyOnFirstSeen","notifyOnRegression","notifyOnResolve","minOccurrenceCount","firstNotificationThreshold","severity","updatedAt","updatedBy"],"additionalProperties":true},"annotations":{"title":"Update Error Notification Policy","readOnlyHint":false,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false}}, diff --git a/apps/ai/src/mcp/tools/__tests__/list-maple-feedback.test.ts b/apps/ai/src/mcp/tools/__tests__/list-maple-feedback.test.ts new file mode 100644 index 0000000000..d3aa2c040a --- /dev/null +++ b/apps/ai/src/mcp/tools/__tests__/list-maple-feedback.test.ts @@ -0,0 +1,124 @@ +import { afterEach, describe, expect, it } from "@effect/vitest" +import { Array as Arr, Context, Effect, Layer, Result, Schema } from "effect" +import { OrgId, RoleName, UserId } from "@maple/domain/http" +import { ListMapleFeedbackOutput } from "@maple/domain/mcp-outputs" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import type { TenantContext } from "@maple/backend/services/auth/tenant-context" +import { AgentFeedbackService } from "@maple/backend/services/feedback/AgentFeedbackService" +import { type AuditActorInfo, CurrentAuditActor } from "@maple/backend/services/auth/audit-actor" +import { testEnv } from "@maple/backend/services/integrations/vcs/__tests__/harness" +import { CurrentMcpTenant } from "../../lib/query-warehouse" +import { executeRegisteredMcpToolUnscoped } from "../registry" + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const INTERNAL = Schema.decodeSync(OrgId)("org_maple_internal") +const CUSTOMER = Schema.decodeSync(OrgId)("org_customer") +const USER = Schema.decodeSync(UserId)("user_feedback") + +const tenant = (orgId: OrgId): TenantContext => ({ + orgId, + userId: USER, + roles: [Schema.decodeSync(RoleName)("root")], + authMode: "self_hosted", +}) + +const OAUTH_USER: AuditActorInfo = { type: "user", source: "mcp" } + +const layer = ( + orgId: OrgId, + options: { readonly actor?: AuditActorInfo; readonly env?: Record } = {}, +) => + Layer.mergeAll( + AgentFeedbackService.layer.pipe( + Layer.provide( + Layer.mergeAll( + createTestDb(createdDbs).layer, + testEnv(options.env ?? { MAPLE_INTERNAL_ORG_ID: INTERNAL }), + ), + ), + ), + Layer.succeed(CurrentMcpTenant, tenant(orgId)), + Layer.succeed(CurrentAuditActor, "actor" in options ? options.actor : OAUTH_USER), + ) + +// The registry's requirements name every tool's services; this tool reaches only the ones above. +const withoutUnreachedServices = (effect: Effect.Effect): Effect.Effect => + Effect.provide(effect, Context.makeUnsafe(new Map())) + +const call = (params: unknown) => + executeRegisteredMcpToolUnscoped("list_maple_feedback", params, "mcp").pipe( + Effect.result, + withoutUnreachedServices, + ) + +const expectRefused = (result: Result.Result) => { + expect(Result.isFailure(result)).toBe(true) + if (Result.isFailure(result)) { + expect(result.failure).toMatchObject({ + _tag: "@maple/mcp/errors/McpAuthInvalidError", + reason: "insufficient_scope", + }) + } +} + +const seed = Effect.gen(function* () { + const service = yield* AgentFeedbackService + yield* Effect.forEach([INTERNAL, CUSTOMER], (orgId) => + service.submit(orgId, USER, { + kind: "bug", + summary: `from ${orgId}`, + reason: "Secret customer context", + agent: { type: "coding_agent" }, + source: "mcp", + }), + ) +}) + +describe("list_maple_feedback", () => { + it.effect("gives Maple's internal org every org's feedback", () => + Effect.gen(function* () { + yield* seed + const result = yield* call({}) + expect(Result.isSuccess(result)).toBe(true) + if (Result.isSuccess(result)) { + const output = Schema.decodeUnknownSync(ListMapleFeedbackOutput)( + result.success.structuredContent, + ) + expect(Arr.map(output.reports, (report) => report.orgId).sort()).toEqual( + [CUSTOMER, INTERNAL].sort(), + ) + expect(result.success.content[0]?.text).toContain("Secret customer context") + } + }).pipe(Effect.provide(layer(INTERNAL))), + ) + + it.effect("refuses a customer org, even for its own feedback", () => + Effect.gen(function* () { + yield* seed + expectRefused(yield* call({ org_id: CUSTOMER })) + }).pipe(Effect.provide(layer(CUSTOMER))), + ) + + it.effect("refuses Maple's own agents and the bot (internal service token) in the internal org", () => + Effect.gen(function* () { + yield* seed + expectRefused(yield* call({})) + }).pipe(Effect.provide(layer(INTERNAL, { actor: { type: "system", source: "system" } }))), + ) + + it.effect("refuses an in-process call that carries no transport actor", () => + Effect.gen(function* () { + yield* seed + expectRefused(yield* call({})) + }).pipe(Effect.provide(layer(INTERNAL, { actor: undefined }))), + ) + + it.effect("refuses every org while MAPLE_INTERNAL_ORG_ID is unset", () => + Effect.gen(function* () { + yield* seed + expectRefused(yield* call({})) + }).pipe(Effect.provide(layer(INTERNAL, { env: {} }))), + ) +}) diff --git a/apps/ai/src/mcp/tools/list-maple-feedback.ts b/apps/ai/src/mcp/tools/list-maple-feedback.ts new file mode 100644 index 0000000000..8ea246c128 --- /dev/null +++ b/apps/ai/src/mcp/tools/list-maple-feedback.ts @@ -0,0 +1,152 @@ +import { DateTime, Effect, Schema } from "effect" +import { AgentFeedbackImpact, AgentFeedbackKind, OrgId } from "@maple/domain/http" +import { AgentFeedbackPublicId } from "@maple/domain/http/v2" +import { ListMapleFeedbackOutput } from "@maple/domain/mcp-outputs" +import { parseWarehouseDateTime } from "@maple/query-engine" +import { AgentFeedbackService } from "@maple/backend/services/feedback/AgentFeedbackService" +import { CurrentAuditActor } from "@maple/backend/services/auth/audit-actor" +import { McpAuthInvalidError, McpInvalidInputError, McpNotReadyError, type McpToolRegistrar } from "./types" +import { CurrentMcpTenant } from "../lib/query-warehouse" +import * as P from "../lib/params" +import { doc } from "../lib/tool-doc" + +const TOOL = "list_maple_feedback" +const encodePublicId = Schema.encodeSync(AgentFeedbackPublicId) +const decodeOrgId = Schema.decodeUnknownOption(OrgId) +const iso = (ms: number) => DateTime.formatIso(DateTime.makeUnsafe(ms)) + +/** + * The Maple team's triage view of `send_maple_feedback` and `POST /v2/agent_feedback`. Listed on + * every surface like any public tool, but `AgentFeedbackService.triage` refuses every org except + * `MAPLE_INTERNAL_ORG_ID`, and this handler refuses any caller but a person or API key on the + * public `/mcp` transport: Maple's own agents and the bot never carry customer feedback text. + */ +export function registerListMapleFeedbackTool(server: McpToolRegistrar) { + server.define({ + name: TOOL, + title: "List Maple Feedback", + description: + "Maple team only: read the feedback agents sent about Maple (send_maple_feedback, POST /v2/agent_feedback) across every organization, newest first, with the full reason and details. Refused for any other organization.", + parameters: Schema.Struct({ + kind: P.optionalOneOf(AgentFeedbackKind.literals, "Only this kind of feedback"), + impact: P.optionalOneOf(AgentFeedbackImpact.literals, "Only this impact"), + start_time: P.optionalTimestamp("Only feedback sent at or after this time"), + related_to: P.optionalText( + "Only feedback whose related tool, endpoint or page contains this text (case-insensitive)", + ), + org_id: P.optionalText("Only feedback from this organization (`org_…`)"), + limit: P.limit({ default: 50, max: 200, noun: "reports" }), + }), + output: ListMapleFeedbackOutput, + aliases: { since: "start_time" }, + hints: { readOnly: true }, + phrases: ["Reading agent feedback"], + handler: Effect.fn("McpTool.listMapleFeedback")(function* (params) { + const tenant = yield* CurrentMcpTenant + const actor = yield* CurrentAuditActor + if (actor === undefined || actor.type === "system") { + return yield* new McpAuthInvalidError({ + message: `\`${TOOL}\` is for the Maple team over the MCP transport, not for Maple's own agents.`, + reason: "insufficient_scope", + }) + } + const orgId = + params.org_id === undefined + ? undefined + : yield* Effect.fromOption(decodeOrgId(params.org_id)).pipe( + Effect.mapError( + () => + new McpInvalidInputError({ + message: "org_id must be an organization id such as `org_…`.", + parameter: "org_id", + }), + ), + ) + const sinceMs = + params.start_time === undefined ? undefined : parseWarehouseDateTime(params.start_time) + + const service = yield* AgentFeedbackService + const reports = yield* service + .triage(tenant.orgId, { + kind: params.kind, + impact: params.impact, + sinceMs, + relatedTo: params.related_to, + orgId, + limit: params.limit, + }) + .pipe( + Effect.catchTags({ + "@maple/http/errors/AgentFeedbackTriageForbiddenError": (error) => + new McpAuthInvalidError({ message: error.message, reason: "insufficient_scope" }), + "@maple/http/errors/AgentFeedbackPersistenceError": () => + new McpNotReadyError({ + message: "Feedback could not be read right now.", + retryAfterSeconds: 5, + }), + }), + ) + + return { + reports: reports.map((report) => ({ + id: encodePublicId(report.id), + orgId: report.orgId, + userId: report.userId, + kind: report.kind, + impact: report.impact, + summary: report.summary, + reason: report.reason, + details: report.details, + relatedTo: report.relatedTo, + agentType: report.agent.type, + agentName: report.agent.name, + agentModel: report.agent.model, + agentVersion: report.agent.version, + source: report.source, + createdAt: iso(report.createdAtMs), + })), + limit: params.limit, + filters: { + ...(params.kind !== undefined && { kind: params.kind }), + ...(params.impact !== undefined && { impact: params.impact }), + ...(sinceMs !== undefined && { startTime: iso(sinceMs) }), + ...(params.related_to !== undefined && { relatedTo: params.related_to }), + ...(orgId !== undefined && { orgId }), + }, + } + }), + render: (output) => ({ + title: `Maple feedback: ${output.reports.length} report${output.reports.length === 1 ? "" : "s"}`, + blocks: + output.reports.length === 0 + ? [doc.text("No feedback matches these filters.")] + : [ + doc.table( + ["Sent", "Kind", "Impact", "Related to", "Org", "Agent", "Summary"], + output.reports.map((report) => [ + report.createdAt, + report.kind, + report.impact ?? "-", + report.relatedTo ?? "-", + report.orgId, + [report.agentName, report.agentModel].filter(Boolean).join(" · ") || + report.agentType, + report.summary, + ]), + ), + ...output.reports.flatMap((report) => [ + doc.heading(`${report.id}: ${report.summary}`), + doc.text(report.reason), + ...(report.details === null ? [] : [doc.code("text", report.details)]), + ]), + ...(output.reports.length === output.limit + ? [ + doc.text( + `Showing the newest ${output.limit}. Narrow with start_time, kind, impact or related_to to see older reports.`, + ), + ] + : []), + ], + }), + }) +} diff --git a/apps/ai/src/mcp/tools/registry.ts b/apps/ai/src/mcp/tools/registry.ts index 8de7928f14..c3aed4396c 100644 --- a/apps/ai/src/mcp/tools/registry.ts +++ b/apps/ai/src/mcp/tools/registry.ts @@ -77,6 +77,7 @@ import { registerLinkPullRequestTool } from "./link-pull-request" import { registerProposeFixTool } from "./propose-fix" import { registerRegisterAgentTool } from "./register-agent" import { registerSendMapleFeedbackTool } from "./send-maple-feedback" +import { registerListMapleFeedbackTool } from "./list-maple-feedback" import { registerReleaseErrorIssueTool } from "./release-error-issue" import { registerSetIssueSeverityTool } from "./set-issue-severity" import { registerMergeErrorIssuesTool } from "./merge-error-issues" @@ -520,6 +521,7 @@ const collectMapleToolDefinitions = (): ReadonlyArray => { registerListErrorIssueEventsTool(registrar) registerRegisterAgentTool(registrar) registerSendMapleFeedbackTool(registrar) + registerListMapleFeedbackTool(registrar) registerListErrorIncidentsTool(registrar) registerGetErrorNotificationPolicyTool(registrar) registerUpdateErrorNotificationPolicyTool(registrar) diff --git a/apps/ai/src/worker.ts b/apps/ai/src/worker.ts index ee9d003bc1..01220b82b5 100644 --- a/apps/ai/src/worker.ts +++ b/apps/ai/src/worker.ts @@ -60,6 +60,8 @@ const configuredEnv = ({ stage, region, domains, profile }: MapleStackContext) = optionalPlain("MAPLE_DECISION_MODEL"), // The chat agent authenticates to `/mcp` as an internal caller. optionalSecret("INTERNAL_SERVICE_TOKEN"), + // Maple's own org: the only one `list_maple_feedback` reads across orgs for. + optionalPlain("MAPLE_INTERNAL_ORG_ID"), // The source and sandbox tools resolve repositories through the GitHub App, so // they need its reader credentials here too, not only on api. githubAppSourceEnv, diff --git a/packages/backend/src/platform/Env.ts b/packages/backend/src/platform/Env.ts index 91ce8ecbfd..05300f80e9 100644 --- a/packages/backend/src/platform/Env.ts +++ b/packages/backend/src/platform/Env.ts @@ -109,6 +109,8 @@ export interface EnvConfig { readonly MAPLE_PRODUCT_EVENTS_INGEST_KEY: Option.Option readonly SD_INTERNAL_TOKEN: Option.Option readonly INTERNAL_SERVICE_TOKEN: Option.Option + /** The org Maple's own team and telemetry live in; the only one that reads agent feedback across orgs. */ + readonly MAPLE_INTERNAL_ORG_ID: Option.Option readonly EMAIL_FROM: string readonly LINEAR_OAUTH_CLIENT_ID: Option.Option readonly LINEAR_OAUTH_CLIENT_SECRET: Option.Option @@ -267,6 +269,7 @@ const envConfig = Config.all({ MAPLE_PRODUCT_EVENTS_INGEST_KEY: optionalRedacted("MAPLE_PRODUCT_EVENTS_INGEST_KEY"), SD_INTERNAL_TOKEN: optionalRedacted("SD_INTERNAL_TOKEN"), INTERNAL_SERVICE_TOKEN: optionalRedacted("INTERNAL_SERVICE_TOKEN"), + MAPLE_INTERNAL_ORG_ID: optionalString("MAPLE_INTERNAL_ORG_ID"), EMAIL_FROM: stringWithDefault("EMAIL_FROM", "Maple "), LINEAR_OAUTH_CLIENT_ID: optionalString("LINEAR_OAUTH_CLIENT_ID"), LINEAR_OAUTH_CLIENT_SECRET: optionalRedacted("LINEAR_OAUTH_CLIENT_SECRET"), diff --git a/packages/backend/src/services/feedback/AgentFeedbackService.test.ts b/packages/backend/src/services/feedback/AgentFeedbackService.test.ts index a02a66573e..ac19d7e325 100644 --- a/packages/backend/src/services/feedback/AgentFeedbackService.test.ts +++ b/packages/backend/src/services/feedback/AgentFeedbackService.test.ts @@ -1,8 +1,9 @@ import { afterEach, assert, describe, it } from "@effect/vitest" -import { Effect, Layer, Schema, Array as Arr } from "effect" +import { Effect, Layer, Order, Result, Schema, Array as Arr } from "effect" import { TestClock } from "effect/testing" import { OrgId, UserId } from "@maple/domain/http" import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { testEnv } from "@maple/backend/services/integrations/vcs/__tests__/harness" import { AgentFeedbackService } from "./AgentFeedbackService" const trackedDbs: TestDb[] = [] @@ -12,10 +13,20 @@ const ORG = Schema.decodeSync(OrgId)("org_feedback") const OTHER_ORG = Schema.decodeSync(OrgId)("org_other") const USER = Schema.decodeSync(UserId)("user_feedback") -const provide = (effect: Effect.Effect) => - effect.pipe( - Effect.provide(AgentFeedbackService.layer.pipe(Layer.provide(createTestDb(trackedDbs).layer))), - ) +const INTERNAL_ORG = Schema.decodeSync(OrgId)("org_maple_internal") + +const provideWith = + (env: Record) => + (effect: Effect.Effect) => + effect.pipe( + Effect.provide( + AgentFeedbackService.layer.pipe( + Layer.provide(Layer.mergeAll(createTestDb(trackedDbs).layer, testEnv(env))), + ), + ), + ) + +const provide = provideWith({ MAPLE_INTERNAL_ORG_ID: INTERNAL_ORG }) describe("AgentFeedbackService", () => { it.effect("stores the kind, agent and reason, and normalizes blank optionals to null", () => @@ -78,4 +89,110 @@ describe("AgentFeedbackService", () => { }), ), ) + + describe("triage", () => { + const seed = Effect.gen(function* () { + const service = yield* AgentFeedbackService + yield* service.submit(ORG, USER, { + kind: "bug", + impact: "blocking", + summary: "run_sql rejects a valid query", + reason: "Needed a count", + relatedTo: "run_sql", + agent: { type: "coding_agent" }, + source: "mcp", + }) + yield* service.submit(ORG, USER, { + kind: "improvement", + summary: "runXsql decoy", + reason: "Matches run_sql only if `_` is a wildcard", + relatedTo: "runXsql", + agent: { type: "coding_agent" }, + source: "mcp", + }) + yield* TestClock.adjust("1 hour") + yield* service.submit(OTHER_ORG, USER, { + kind: "documentation", + impact: "minor", + summary: "search_traces description is vague", + reason: "Guessed the filter", + relatedTo: "Search_Traces", + agent: { type: "chat_assistant" }, + source: "api", + }) + return service + }) + + it.effect("reads every org's feedback, newest first, for Maple's internal org", () => + provide( + Effect.gen(function* () { + const service = yield* seed + const all = yield* service.triage(INTERNAL_ORG, { limit: 10 }) + assert.deepStrictEqual( + Arr.map(all, (row) => row.orgId), + [OTHER_ORG, ORG, ORG], + ) + }), + ), + ) + + it.effect("narrows by kind, impact, since, org and a case-insensitive related_to", () => + provide( + Effect.gen(function* () { + const service = yield* seed + const summaries = (filter: Parameters[1]) => + service + .triage(INTERNAL_ORG, filter) + .pipe(Effect.map((rows) => Arr.map(rows, (row) => row.summary))) + assert.deepStrictEqual(yield* summaries({ limit: 10, kind: "bug" }), [ + "run_sql rejects a valid query", + ]) + assert.deepStrictEqual(yield* summaries({ limit: 10, impact: "minor" }), [ + "search_traces description is vague", + ]) + assert.deepStrictEqual(yield* summaries({ limit: 10, relatedTo: "search_" }), [ + "search_traces description is vague", + ]) + // `_` is a literal here, not a LIKE wildcard: the `runXsql` decoy stays out. + assert.deepStrictEqual(yield* summaries({ limit: 10, relatedTo: "run_sql" }), [ + "run_sql rejects a valid query", + ]) + assert.deepStrictEqual( + Arr.sort(yield* summaries({ limit: 10, orgId: ORG }), Order.String), + ["runXsql decoy", "run_sql rejects a valid query"], + ) + const newest = yield* service.triage(INTERNAL_ORG, { limit: 1 }) + assert.deepStrictEqual(yield* summaries({ limit: 10, sinceMs: newest[0]?.createdAtMs }), [ + "search_traces description is vague", + ]) + }), + ), + ) + + it.effect("refuses any other org, including one that sent feedback", () => + provide( + Effect.gen(function* () { + const service = yield* seed + const result = yield* Effect.result(service.triage(ORG, { limit: 10 })) + assert.isTrue(Result.isFailure(result)) + if (Result.isFailure(result)) { + assert.strictEqual( + result.failure._tag, + "@maple/http/errors/AgentFeedbackTriageForbiddenError", + ) + } + }), + ), + ) + + it.effect("refuses every org while MAPLE_INTERNAL_ORG_ID is unset", () => + provideWith({})( + Effect.gen(function* () { + const service = yield* seed + const result = yield* Effect.result(service.triage(INTERNAL_ORG, { limit: 10 })) + assert.isTrue(Result.isFailure(result)) + }), + ), + ) + }) }) diff --git a/packages/backend/src/services/feedback/AgentFeedbackService.ts b/packages/backend/src/services/feedback/AgentFeedbackService.ts index b8a7d07260..fbd371935e 100644 --- a/packages/backend/src/services/feedback/AgentFeedbackService.ts +++ b/packages/backend/src/services/feedback/AgentFeedbackService.ts @@ -6,11 +6,13 @@ import { AgentFeedbackKind, AgentFeedbackPersistenceError, AgentFeedbackSource, + AgentFeedbackTriageForbiddenError, AgentType, } from "@maple/domain/http" import { AgentFeedbackId, type OrgId, type UserId } from "@maple/domain/primitives" -import { Clock, Context, Effect, Layer, Schema, String as Str } from "effect" +import { Clock, Context, Effect, Layer, Option, Schema, String as Str } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" +import { Env } from "@maple/backend/platform/Env" import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" export interface AgentFeedback { @@ -49,6 +51,17 @@ export interface SubmitAgentFeedbackInput { readonly source: AgentFeedbackSource } +/** The Maple team's cross-org read. Every filter narrows; none set reads the newest across all orgs. */ +export interface AgentFeedbackTriageFilter { + readonly kind?: AgentFeedbackKind | undefined + readonly impact?: AgentFeedbackImpact | undefined + readonly sinceMs?: number | undefined + /** Case-insensitive substring of `related_to`. */ + readonly relatedTo?: string | undefined + readonly orgId?: OrgId | undefined + readonly limit: number +} + export interface AgentFeedbackServiceApi { readonly submit: ( orgId: OrgId, @@ -60,6 +73,17 @@ export interface AgentFeedbackServiceApi { orgId: OrgId, page: { readonly limit: number; readonly offset: number }, ) => Effect.Effect, AgentFeedbackPersistenceError> + /** + * Feedback from every org, newest first. Only `MAPLE_INTERNAL_ORG_ID` may call it; any other + * caller, or any caller while it is unset, is refused before the database is touched. + */ + readonly triage: ( + callerOrgId: OrgId, + filter: AgentFeedbackTriageFilter, + ) => Effect.Effect< + ReadonlyArray, + AgentFeedbackPersistenceError | AgentFeedbackTriageForbiddenError + > } const makePersistenceError = makePersistenceErrorMapper( @@ -104,6 +128,8 @@ export class AgentFeedbackService extends Context.Service + db.run( + PG.from(AgentFeedbackTable) + .select() + .where(($) => [ + filter.orgId === undefined ? undefined : $.orgId.eq(filter.orgId), + filter.kind === undefined ? undefined : $.kind.eq(filter.kind), + filter.impact === undefined ? undefined : $.impact.eq(filter.impact), + filter.sinceMs === undefined ? undefined : $.createdAt.gte(filter.sinceMs), + pattern === undefined ? undefined : $.relatedTo.ilike(pattern), + ]) + .orderBy(["createdAt", "desc"], ["id", "desc"]) + .limit(filter.limit) + .crossTenant(), + ), + ) + yield* Effect.annotateCurrentSpan("result.rowCount", rows.length) + return rows.map(toFeedback) + }) + + return { submit, list, triage } satisfies AgentFeedbackServiceApi }), }, ) { diff --git a/packages/domain/src/http/agent-feedback.ts b/packages/domain/src/http/agent-feedback.ts index 53bb4c8282..ab0187f605 100644 --- a/packages/domain/src/http/agent-feedback.ts +++ b/packages/domain/src/http/agent-feedback.ts @@ -62,3 +62,12 @@ export class AgentFeedbackPersistenceError extends HttpTaggedError()( + "@maple/http/errors/AgentFeedbackTriageForbiddenError", + { message: Schema.String }, +) {} diff --git a/packages/domain/src/mcp-outputs/catalog.ts b/packages/domain/src/mcp-outputs/catalog.ts index cce084d2b7..fafdb75a38 100644 --- a/packages/domain/src/mcp-outputs/catalog.ts +++ b/packages/domain/src/mcp-outputs/catalog.ts @@ -2,7 +2,7 @@ * Every MCP tool's output schema, by tool name: the one list both the registry (which checks * each tool declares exactly this schema) and the chat UI's `StructuredToolOutput` derive from. */ -import { SendMapleFeedbackOutput } from "./feedback" +import { ListMapleFeedbackOutput, SendMapleFeedbackOutput } from "./feedback" import { InspectInfraOutput, ListInfraOutput } from "./infra" import { ListSourcesOutput } from "./sources" import { InspectUserOutput, ListUsersOutput } from "./users" @@ -271,6 +271,7 @@ export const McpToolOutputs = { search_source_code: SearchSourceCodeOutput, search_traces: SearchTracesOutput, send_maple_feedback: SendMapleFeedbackOutput, + list_maple_feedback: ListMapleFeedbackOutput, service_deployments: ServiceDeploymentsOutput, service_map: ServiceMapOutput, sync_railway: SyncRailwayOutput, diff --git a/packages/domain/src/mcp-outputs/feedback.ts b/packages/domain/src/mcp-outputs/feedback.ts index 9d10a6338c..c4e441a66e 100644 --- a/packages/domain/src/mcp-outputs/feedback.ts +++ b/packages/domain/src/mcp-outputs/feedback.ts @@ -12,3 +12,35 @@ export const SendMapleFeedbackOutput = Schema.Struct({ agentModel: Schema.NullOr(Schema.String), createdAt: Schema.String, }) + +/** One report as the Maple team triages it: the full text, plus who sent it from where. */ +export const MapleFeedbackReport = Schema.Struct({ + id: Schema.String, + orgId: Schema.String, + userId: Schema.String, + kind: Schema.String, + impact: Schema.NullOr(Schema.String), + summary: Schema.String, + reason: Schema.String, + details: Schema.NullOr(Schema.String), + relatedTo: Schema.NullOr(Schema.String), + agentType: Schema.String, + agentName: Schema.NullOr(Schema.String), + agentModel: Schema.NullOr(Schema.String), + agentVersion: Schema.NullOr(Schema.String), + source: Schema.String, + createdAt: Schema.String, +}) + +/** Output schema for `list_maple_feedback`. */ +export const ListMapleFeedbackOutput = Schema.Struct({ + reports: Schema.Array(MapleFeedbackReport), + limit: Schema.Number, + filters: Schema.Struct({ + kind: Schema.optionalKey(Schema.String), + impact: Schema.optionalKey(Schema.String), + startTime: Schema.optionalKey(Schema.String), + relatedTo: Schema.optionalKey(Schema.String), + orgId: Schema.optionalKey(Schema.String), + }), +})