diff --git a/apps/api/src/routes/v2/alert-deliveries.http.ts b/apps/api/src/routes/v2/alert-deliveries.http.ts index 2c57ece5da..0676cd6118 100644 --- a/apps/api/src/routes/v2/alert-deliveries.http.ts +++ b/apps/api/src/routes/v2/alert-deliveries.http.ts @@ -1,12 +1,14 @@ import { HttpApiBuilder } from "effect/http-api" -import type { AlertDeliveryEventDocument } from "@maple/domain/http" -import { CurrentTenant } from "@maple/domain/http" +import type { AlertDeliveryEventDocument, AlertNotificationEventType } from "@maple/domain/http" +import { CurrentTenant, isAlertNotificationEventType } from "@maple/domain/http" import type { V2AlertDelivery } from "@maple/domain/http/v2" import { MapleApiV2, paginateOffsetQuery, timestamp, timestampOrNull } from "@maple/domain/http/v2" -import { Effect, Array as Arr } from "effect" +import { Effect, Result, Array as Arr } from "effect" import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService" -const toV2Delivery = (doc: AlertDeliveryEventDocument): V2AlertDelivery => ({ +const toV2Delivery = ( + doc: AlertDeliveryEventDocument & { readonly eventType: AlertNotificationEventType }, +): V2AlertDelivery => ({ id: doc.id, object: "alert_delivery", incident_id: doc.incidentId, @@ -39,10 +41,19 @@ export const HttpV2AlertDeliveriesLive = HttpApiBuilder.group(MapleApiV2, "alert ? { incidentId: query.incident_id } : undefined), ...(query.rule_id !== undefined ? { ruleId: query.rule_id } : undefined), + notificationsOnly: true, limit, offset, }) - .pipe(Effect.map((response) => Arr.map(response.events, toV2Delivery))), + .pipe( + Effect.map((response) => + Arr.filterMap(response.events, (doc) => + isAlertNotificationEventType(doc.eventType) + ? Result.succeed(toV2Delivery({ ...doc, eventType: doc.eventType })) + : Result.failVoid, + ), + ), + ), ) return { object: "list" as const, ...page } }), diff --git a/apps/api/src/routes/v2/alert-incidents.http.ts b/apps/api/src/routes/v2/alert-incidents.http.ts index 9235babcda..cf3b09f7be 100644 --- a/apps/api/src/routes/v2/alert-incidents.http.ts +++ b/apps/api/src/routes/v2/alert-incidents.http.ts @@ -1,6 +1,6 @@ import { HttpApiBuilder } from "effect/http-api" import type { AlertIncidentDocument, AlertIncidentEventDocument } from "@maple/domain/http" -import { CurrentTenant } from "@maple/domain/http" +import { CurrentTenant, isAlertNotificationEventType } from "@maple/domain/http" import type { V2AlertIncident, V2AlertIncidentEvent } from "@maple/domain/http/v2" import { MapleApiV2, paginateOffsetQuery } from "@maple/domain/http/v2" import { Duration, Effect, Option, Array as Arr } from "effect" @@ -31,7 +31,11 @@ const toV2Incident = (doc: AlertIncidentDocument): V2AlertIncident => ({ last_observed_value: doc.lastObservedValue, last_sample_count: doc.lastSampleCount, dedupe_key: doc.dedupeKey, - last_delivered_event_type: doc.lastDeliveredEventType, + // An acknowledgement never becomes the last delivered event; narrow it for the public enum. + last_delivered_event_type: + doc.lastDeliveredEventType !== null && isAlertNotificationEventType(doc.lastDeliveredEventType) + ? doc.lastDeliveredEventType + : null, last_notified_at: doc.lastNotifiedAt, hold_reason: doc.holdReason, held_since: doc.heldSince, diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index ea5e5f79ae..921d0b759d 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -434,6 +434,19 @@ describe("v2 alerts over HTTP", () => { )`, ) + // An internal PagerDuty acknowledgement sync is not a notification: the public list leaves it out. + await executeSql( + harness.testDb, + `insert into alert_delivery_events ( + id, org_id, rule_id, destination_id, delivery_key, event_type, + attempt_number, status, scheduled_at, payload_json, created_at, updated_at + ) + select '00000000-0000-4000-9000-0000000000ac', rules.org_id, rules.id, destinations.id, + 'ack-sync', 'acknowledge', 1, 'success', timestamp '2026-08-02 12:00:00+00', '{}'::jsonb, + timestamp '2026-08-02 12:00:00+00', timestamp '2026-08-02 12:00:00+00' + from (select id, org_id from alert_rules where org_id = 'org_alerts_e2e' limit 1) as rules + cross join (select id from alert_destinations where org_id = 'org_alerts_e2e' limit 1) as destinations`, + ) const first = await harness.request("GET", "/v2/alerts/deliveries?limit=60", key.secret) expect(first.status).toBe(200) expect(first.body.data).toHaveLength(60) @@ -455,7 +468,7 @@ describe("v2 alerts over HTTP", () => { expect( Arr.some( [...first.body.data, ...second.body.data], - (item) => item.delivery_key === "foreign-delivery", + (item) => item.delivery_key === "foreign-delivery" || item.delivery_key === "ack-sync", ), ).toBe(false) diff --git a/apps/chat-bot/src/relay/run.ts b/apps/chat-bot/src/relay/run.ts index 8c07741cc4..f7f2dddb98 100644 --- a/apps/chat-bot/src/relay/run.ts +++ b/apps/chat-bot/src/relay/run.ts @@ -10,7 +10,13 @@ import type { ChatConnector, ConnectorCredentials, InboundEvent } from "@maple/chat-platform" import { connectors } from "@maple/chat-platform/connectors" import type { ChatConnectorId, OrgId } from "@maple/domain/primitives" -import type { IntegrationsPersistenceError } from "@maple/domain/http" +import { + AlertIncidentAckSource, + AlertIncidentId, + type IntegrationsPersistenceError, +} from "@maple/domain/http" +import { AlertIncidentAckService } from "@maple/backend/services/alerts/AlertIncidentAckService" +import { AlertChatCardService } from "@maple/backend/services/alerts/AlertChatCardService" import { runtimeTelemetry } from "@maple/infra/worker-telemetry" import { ChatSessions, type ChatSessionsApi } from "@maple/backend/platform/bindings" import { type ChatSessionNamespace, chatSessionsLayerIfBound } from "@maple/backend/platform/chat-sessions" @@ -35,6 +41,7 @@ import { relayWithWorkspace, withWorkspace, WorkspaceLookupFailed, + type RelayAlertAckOutcome, type RelayPorts, type ResolvedWorkspace, } from "./turn.ts" @@ -201,6 +208,68 @@ const lookupWorkspace = ( ).pipe(lookupFailed(connectorId, "The chat workspace could not be read")), ) +const decodeAckSource = Schema.decodeUnknownOption(AlertIncidentAckSource) +const decodeIncidentId = Schema.decodeUnknownOption(AlertIncidentId) + +/** + * An alert card's ack click, in one connection: the ack (which also queues the PagerDuty + * acknowledgement), then the card redrawn with its new state. Every failure is an outcome. + */ +const alertAck = + (host: Pick): RelayPorts["alertAck"] => + (request) => { + const source = decodeAckSource(request.connector) + const incidentId = decodeIncidentId(request.incidentId) + if (Option.isNone(source)) return Effect.succeed({ kind: "unavailable" }) + if (Option.isNone(incidentId)) return Effect.succeed({ kind: "not_found" }) + return withDatabase( + host.env, + Effect.gen(function* () { + const acks = yield* AlertIncidentAckService + const cards = yield* AlertChatCardService + const actor = { userId: request.userId, name: request.displayName, source: source.value } + const change = + request.action === "acknowledge" + ? acks.acknowledge(request.orgId, incidentId.value, actor) + : acks.unacknowledge(request.orgId, incidentId.value, actor) + const applied = yield* change.pipe( + Effect.as<"applied" | "resolved" | "not_found">("applied"), + Effect.catchTags({ + "@maple/http/errors/AlertIncidentNotFoundError": () => + Effect.succeed("not_found" as const), + "@maple/http/errors/AlertIncidentResolvedError": () => + Effect.succeed("resolved" as const), + }), + ) + if (applied === "not_found") return { kind: "not_found" } satisfies RelayAlertAckOutcome + // The ack stands even when the card cannot be redrawn; the clicker is told instead. + const card = yield* cards + .cardForMessage(request.orgId, incidentId.value, request.messageId) + .pipe( + Effect.tapError((error) => + Effect.logWarning("An alert card could not be rebuilt").pipe( + Effect.annotateLogs({ "error.type": error._tag }), + ), + ), + Effect.orElseSucceed(() => Option.none()), + ) + return { kind: applied, card } satisfies RelayAlertAckOutcome + }).pipe( + // Built per event: the services hold the event's own connection. + Effect.provide([AlertIncidentAckService.layer, AlertChatCardService.layer]), + ), + ).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.logError("An alert acknowledgement from chat failed").pipe( + Effect.annotateLogs({ "error.type": summarizeCause(cause) }), + Effect.as({ kind: "unavailable" }), + ), + ), + ) + } + /** What a relayed event reads from the graph rather than the object: settings and the chat-session port. */ interface RelayEnvironment { readonly settings: RelaySettings @@ -258,6 +327,7 @@ const ports = ( ownsConversation: host.ownsConversation, rememberConversation: host.rememberConversation, recordTurn: host.recordTurn, + alertAck: alertAck(host), }) /** diff --git a/apps/chat-bot/src/relay/settle.test.ts b/apps/chat-bot/src/relay/settle.test.ts index bedf92a90a..ddb24fb214 100644 --- a/apps/chat-bot/src/relay/settle.test.ts +++ b/apps/chat-bot/src/relay/settle.test.ts @@ -179,6 +179,7 @@ const ports = ( announceUnlinked: Effect.succeed(false), ownsConversation: () => Effect.succeed(false), rememberConversation: () => Effect.void, + alertAck: () => Effect.succeed({ kind: "unavailable" }), // Cloned, as Durable Object storage does: nothing the fiber holds survives it. recordTurn: (checkpoint) => Effect.suspend(() => { diff --git a/apps/chat-bot/src/relay/turn.test.ts b/apps/chat-bot/src/relay/turn.test.ts index 65811e75a6..9f501383ce 100644 --- a/apps/chat-bot/src/relay/turn.test.ts +++ b/apps/chat-bot/src/relay/turn.test.ts @@ -22,6 +22,7 @@ import { type ChatSessionStub, chatSessionClientFromStub } from "@maple/domain/c import { ChatConnectorId, OrgId, UserId } from "@maple/domain/primitives" import { chatActionControlId, + chatAlertAckControlId, chatConnectorId, ChatOutboundError, ConnectorCredentials, @@ -52,7 +53,14 @@ import { } from "effect" import { TestClock } from "effect/testing" import { ConversationNotRecorded, FOLLOW_UP_WINDOW_MS } from "./conversation.ts" -import { relayInboundEvent, relayWithWorkspace, WorkspaceLookupFailed, type RelayPorts } from "./turn.ts" +import { + relayInboundEvent, + relayWithWorkspace, + WorkspaceLookupFailed, + type RelayAlertAckOutcome, + type RelayAlertAckRequest, + type RelayPorts, +} from "./turn.ts" const TESTCHAT = chatConnectorId("testchat") const ORG = Schema.decodeSync(OrgId)("org_1") @@ -271,6 +279,8 @@ interface Host { readonly announced: { count: number } /** How many workspace lookups the events caused — one database connection each. */ readonly lookups: { count: number } + /** Every alert ack the relay asked the host for. */ + readonly alertAcks: Array } const host = ( @@ -288,8 +298,11 @@ const host = ( readonly opened?: ReadonlyArray /** The relay object could not record the conversation the bot just opened. */ readonly rememberFails?: boolean + /** What the alert ack port answers. */ + readonly alertAck?: (request: RelayAlertAckRequest) => RelayAlertAckOutcome }, ): Host => { + const alertAcks: Array = [] const forgotten: Array = [] const charts: Array<{ orgId: OrgId; ref: ChatChartRef }> = [] const opened = new Set(options?.opened ?? []) @@ -301,6 +314,7 @@ const host = ( opened, announced, lookups, + alertAcks, ports: { outbound, supportsIdentity: options?.supportsIdentity ?? false, @@ -355,6 +369,11 @@ const host = ( ) : Effect.sync(() => void opened.add(conversation.conversationKey)), recordTurn: () => Effect.void, + alertAck: (request) => + Effect.sync(() => { + alertAcks.push(request) + return options?.alertAck?.(request) ?? { kind: "unavailable" } + }), }, } } @@ -1278,6 +1297,152 @@ describe("settling an approval somebody clicked", () => { ) }) +describe("acknowledging an alert from its card", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + const ackClick = (action: "acknowledge" | "unacknowledge") => + click({ actionToken: chatAlertAckControlId(action, INCIDENT), messageId: "alert-msg" }) + const redrawn: ReadonlyArray = [ + { + kind: "alert", + color: "#e01e5a", + title: "Checkout errors", + summary: "", + fields: [], + imageUrl: null, + imageAlt: "", + links: [], + footer: [], + sentAtMs: null, + acknowledge: { incidentId: INCIDENT, acknowledgedBy: "Ada" }, + }, + ] + + it.effect("acks in the workspace's org and redraws the clicked card", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "applied", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(deployment.alertAcks).toEqual([ + { + orgId: ORG, + connector: TESTCHAT, + incidentId: INCIDENT, + action: "acknowledge", + messageId: "alert-msg", + userId: null, + displayName: "Ada", + }, + ]) + expect(platform.calls).toHaveLength(1) + expect(platform.calls[0]?.verb).toBe("edit") + expect(platform.calls[0]?.ref.messageId).toBe("alert-msg") + expect(platform.calls[0]?.blocks).toEqual(redrawn) + }), + ) + + it.effect("records the linked Maple user when the connector can name the clicker", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + supportsIdentity: true, + linkedUserId: ADA, + alertAck: () => ({ kind: "applied", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("unacknowledge"), deployment.ports) + + expect(deployment.alertAcks[0]).toMatchObject({ action: "unacknowledge", userId: ADA }) + }), + ) + + it.effect("acks an unlinked clicker under their platform name rather than refusing", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + supportsIdentity: true, + alertAck: () => ({ kind: "applied", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(deployment.alertAcks[0]).toMatchObject({ userId: null, displayName: "Ada" }) + expect(platform.calls[0]?.verb).toBe("edit") + }), + ) + + it.effect("redraws a resolved incident's card and says so privately", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "resolved", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["edit", "whisper"]) + expect(notices(blocksOf(platform.calls))).toEqual([ + "That incident is already resolved, so there is nothing to acknowledge.", + ]) + }), + ) + + it.effect("tells the clicker when the incident is gone, and edits nothing", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "not_found" }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["whisper"]) + expect(notices(blocksOf(platform.calls))).toEqual(["That incident no longer exists in Maple."]) + }), + ) + + it.effect("confirms privately when there is no card to redraw", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "applied", card: Option.none() }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["whisper"]) + expect(notices(blocksOf(platform.calls))).toEqual(["Acknowledged."]) + }), + ) + + it.effect("asks nothing of the host for a workspace nobody linked", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { linked: false }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(deployment.alertAcks).toEqual([]) + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["whisper"]) + }), + ) + + it.effect("treats a malformed ack control as not Maple's", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined) + + yield* relayInboundEvent(click({ actionToken: "alert_ack:not-a-uuid" }), deployment.ports) + + expect(deployment.alertAcks).toEqual([]) + expect(platform.calls).toEqual([]) + }), + ) +}) + describe("relaying everything else a connector reports", () => { it.effect("unlinks a workspace the bot was removed from", () => Effect.gen(function* () { diff --git a/apps/chat-bot/src/relay/turn.ts b/apps/chat-bot/src/relay/turn.ts index de2e31651d..46667a4709 100644 --- a/apps/chat-bot/src/relay/turn.ts +++ b/apps/chat-bot/src/relay/turn.ts @@ -14,9 +14,11 @@ import { ConnectorCredentials, decodeChatActionControlId, + decodeChatAlertAckControlId, driveChatTurn, WORKSPACE_CREDENTIALS, type ChatActionRequest, + type ChatAlertAckAction, type ChatBlock, type ChatChartRef, type ChatConversation, @@ -76,6 +78,27 @@ export class WorkspaceLookupFailed extends Schema.TaggedError> } + | { readonly kind: "resolved"; readonly card: Option.Option> } + | { readonly kind: "not_found" } + | { readonly kind: "unavailable" } + export interface RelayPorts { /** The connector's outbound half — how a turn is shown on the platform it was asked on. */ readonly outbound: ChatOutbound @@ -127,6 +150,8 @@ export interface RelayPorts { readonly announceUnlinked: Effect.Effect /** Checkpoint the turn's posted messages, for settling it after an eviction (`./settle.ts`). */ readonly recordTurn: (checkpoint: RelayTurnCheckpoint) => Effect.Effect + /** Acknowledge or unacknowledge an alert incident from its card. Never fails; see the outcome. */ + readonly alertAck: (request: RelayAlertAckRequest) => Effect.Effect } /** @@ -145,6 +170,10 @@ const BUSY_NOTICE = "Still working on the previous message here — ask again on const UNAVAILABLE_NOTICE = "Maple's agent can't be reached from here right now." +const ALERT_RESOLVED_NOTICE = "That incident is already resolved, so there is nothing to acknowledge." +const ALERT_GONE_NOTICE = "That incident no longer exists in Maple." +const ALERT_UNAVAILABLE_NOTICE = "Maple couldn't record that just now. Try again in a moment." + /** The control outlived what it pointed at: a wiped conversation, or a build that changed the log. */ const PROPOSAL_GONE_NOTICE = "That change isn't waiting for a decision any more." @@ -467,6 +496,10 @@ const settleAction = Effect.fn("ChatBotRelay.settleApproval")(function* ( ports: RelayPorts, ) { yield* Effect.annotateCurrentSpan({ "maple.chat.connector": action.connector }) + const alertAck = decodeChatAlertAckControlId(action.actionToken) + if (Option.isSome(alertAck)) { + return yield* settleAlertAck(action, alertAck.value.action, alertAck.value.incidentId, ports) + } const control = decodeChatActionControlId(action.actionToken) // A platform hands back whatever was on the control that was clicked, which includes controls // Maple never rendered. Not ours is not an error. @@ -571,6 +604,78 @@ const settleAction = Effect.fn("ChatBotRelay.settleApproval")(function* ( } }) +/** + * A click on an alert card's ack button. Anyone in the channel may acknowledge: where the connector + * can link accounts the ack is recorded as that Maple user, otherwise under the platform name. + * The org is the workspace's; the incident id off the control is only looked up inside it. + */ +const settleAlertAck = Effect.fn("ChatBotRelay.settleAlertAck")(function* ( + action: InboundAction, + ackAction: ChatAlertAckAction, + incidentId: string, + ports: RelayPorts, +) { + yield* Effect.annotateCurrentSpan({ + "maple.chat.alert_ack.action": ackAction, + "maple.alert.incident_id": incidentId, + }) + const transport = yield* ports.outbound.transport + const clicker = decodeExternalUserId(action.actor.id) + const workspace = yield* Effect.exit( + ports.resolveWorkspace(action.connector, action.workspaceId, Option.getOrUndefined(clicker)), + ) + if (Exit.isFailure(workspace) || Option.isNone(workspace.value)) { + yield* Effect.annotateCurrentSpan({ "maple.chat.alert_ack": "unavailable" }) + return yield* tellClicker(transport, action, notice(ALERT_UNAVAILABLE_NOTICE)) + } + const { orgId, linkedUserId } = workspace.value.value + yield* Effect.annotateCurrentSpan({ orgId }) + + const outcome = yield* ports.alertAck({ + orgId, + connector: action.connector, + incidentId, + action: ackAction, + messageId: action.messageId, + userId: linkedUserId ?? null, + displayName: action.actor.displayName, + }) + yield* Effect.annotateCurrentSpan({ "maple.chat.alert_ack": outcome.kind }) + const redraw = (card: Option.Option>) => + Option.match(card, { + onNone: () => Effect.void, + onSome: (blocks) => + transport.edit({ target: replyTarget(action), messageId: action.messageId }, blocks).pipe( + Effect.tapError((error) => + Effect.logWarning("An acknowledged alert card could not be redrawn").pipe( + Effect.annotateLogs({ "error.type": error.operation }), + ), + ), + Effect.ignore, + ), + }) + switch (outcome.kind) { + case "applied": + // No card to redraw (a digest, an old delivery): say it privately instead. + return yield* Option.isSome(outcome.card) + ? redraw(outcome.card) + : tellClicker( + transport, + action, + notice(ackAction === "acknowledge" ? "Acknowledged." : "Unacknowledged."), + ) + case "resolved": + yield* redraw(outcome.card) + return yield* tellClicker(transport, action, notice(ALERT_RESOLVED_NOTICE)) + case "not_found": + return yield* tellClicker(transport, action, notice(ALERT_GONE_NOTICE)) + case "unavailable": + return yield* tellClicker(transport, action, notice(ALERT_UNAVAILABLE_NOTICE)) + default: + return outcome satisfies never + } +}) + /** * Put the decision on the message that carried the controls. * diff --git a/packages/backend/src/services/alerts/AlertChatCardService.ts b/packages/backend/src/services/alerts/AlertChatCardService.ts new file mode 100644 index 0000000000..878e514646 --- /dev/null +++ b/packages/backend/src/services/alerts/AlertChatCardService.ts @@ -0,0 +1,191 @@ +/** + * Redraws an alert card Maple already posted to a chat channel, with the incident's current + * acknowledgement. The card is rebuilt from the delivery that posted it (its snapshotted payload), + * so an edit keeps the chart, values and template the original showed. + */ +import type { ChatBlock } from "@maple/chat-platform" +import { + AlertComparator, + AlertSeverity, + AlertSignalType, + type AlertEventType, + type AlertIncidentId, + type AlertPersistenceError, + type OrgId, +} from "@maple/domain/http" +import * as PG from "@maple-dev/effect-orm/postgres" +import { AlertDeliveryEvents, AlertIncidents } from "@maple/db/tables" +import { Context, Effect, Layer, Option, Schema, Array as Arr } from "effect" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { makePersistenceError } from "./alert-persistence" +import { renderTitleBody } from "./AlertDeliveryDispatch" +import { resolveSignalDisplay } from "./alert-signal-display" +import { decodeStoredAlertRuleMetadata, selectStoredAlertRules } from "./AlertRuleModel" +import { StoredDeliveryPayloadSchema } from "./delivery/stored-payload" +import { buildChatAlertBlocks, incidentAcknowledgedBy } from "./delivery/transports/chat" + +export interface AlertChatCardServiceApi { + /** + * The card a chat message carries, redrawn now. `None` when the message is not a firing card + * Maple posted for this incident (a digest, another org's message, a delivery too old to redraw). + */ + readonly cardForMessage: ( + orgId: OrgId, + incidentId: AlertIncidentId, + messageId: string, + ) => Effect.Effect>, AlertPersistenceError> +} + +const decodePayload = Schema.decodeUnknownOption(StoredDeliveryPayloadSchema) +const decodeSeverity = Schema.decodeUnknownOption(AlertSeverity) +const decodeSignalType = Schema.decodeUnknownOption(AlertSignalType) +const decodeComparator = Schema.decodeUnknownOption(AlertComparator) + +export class AlertChatCardService extends Context.Service()( + "@maple/backend/services/alerts/AlertChatCardService", + { + make: Effect.gen(function* () { + const database = yield* Database + const dbExecute = makeDbExecute(database, "AlertChatCardService", makePersistenceError) + + const cardForMessage = Effect.fn("AlertChatCardService.cardForMessage")(function* ( + orgId: OrgId, + incidentId: AlertIncidentId, + messageId: string, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.alert.incident_id": incidentId }) + const [deliveries, incidents] = yield* Effect.all( + [ + dbExecute((db) => + db.run( + PG.from(AlertDeliveryEvents) + .select() + .where(($) => [ + $.orgId.eq(orgId), + $.incidentId.eq(incidentId), + $.providerReference.eq(messageId), + $.status.eq("success"), + $.eventType.in_("trigger", "renotify"), + ]) + .orderBy(["scheduledAt", "desc"]) + .limit(1), + ), + ), + dbExecute((db) => + db.run( + PG.from(AlertIncidents) + .select() + .where(($) => [$.orgId.eq(orgId), $.id.eq(incidentId)]) + .limit(1), + ), + ), + ], + { concurrency: "unbounded" }, + ) + const delivery = Arr.head(deliveries) + const incident = Arr.head(incidents) + if (Option.isNone(delivery) || Option.isNone(incident)) { + yield* Effect.annotateCurrentSpan("maple.alert.card_found", false) + return Option.none>() + } + const row = delivery.value + const current = incident.value + const payload = decodePayload(row.payloadJson) + const linkUrl = Option.flatMap(payload, (value) => Option.fromUndefinedOr(value.linkUrl)) + const signalType = decodeSignalType(current.signalType) + const comparator = decodeComparator(current.comparator) + const eventType: AlertEventType = row.eventType === "renotify" ? "renotify" : "trigger" + if ( + Option.isNone(payload) || + Option.isNone(linkUrl) || + Option.isNone(signalType) || + Option.isNone(comparator) + ) { + yield* Effect.annotateCurrentSpan("maple.alert.card_found", false) + return Option.none>() + } + const stored = payload.value + const ruleRows = yield* dbExecute((db) => + db.run( + selectStoredAlertRules() + .where(($) => [$.orgId.eq(orgId), $.id.eq(row.ruleId)]) + .limit(1), + ), + ) + const ruleRow = Arr.head(ruleRows) + // A rule whose stored config no longer decodes still redraws, from the snapshot alone. + const rule = Option.isNone(ruleRow) + ? Option.none() + : yield* Effect.option(decodeStoredAlertRuleMetadata(ruleRow.value)) + const context = { + ruleId: row.ruleId, + ruleName: Option.match(ruleRow, { + onNone: () => stored.rule?.name ?? current.ruleName, + onSome: (value) => value.name, + }), + eventType, + severity: Option.getOrElse( + Option.orElse(decodeSeverity(stored.rule?.severity), () => + decodeSeverity(current.severity), + ), + (): AlertSeverity => "warning", + ), + signalType: signalType.value, + signalDisplay: resolveSignalDisplay({ + signalType: signalType.value, + queryBuilderDraft: Option.match(rule, { + onNone: () => null, + onSome: (value) => value.queryBuilderDraft, + }), + rawQueryReducer: Option.match(ruleRow, { + onNone: () => null, + onSome: (value) => value.reducer, + }), + }), + comparator: comparator.value, + threshold: current.threshold, + thresholdUpper: current.thresholdUpper, + value: stored.observed?.value ?? null, + sampleCount: stored.observed?.sampleCount ?? null, + detail: stored.observed?.detail ?? null, + groupKey: current.groupKey, + windowMinutes: Option.match(ruleRow, { + onNone: () => stored.rule?.windowMinutes ?? 5, + onSome: (value) => value.windowMinutes, + }), + incidentId: current.id, + incidentStatus: current.status, + incidentAcknowledgedBy: incidentAcknowledgedBy(current), + sparkline: stored.chart?.sparkline ?? null, + chartUrl: stored.chart?.url ?? null, + dedupeKey: current.dedupeKey, + template: + stored.template ?? + Option.match(rule, { + onNone: () => null, + onSome: (value) => value.notificationTemplate, + }), + sentAtMs: row.attemptedAt ?? row.scheduledAt, + } + yield* Effect.annotateCurrentSpan("maple.alert.card_found", true) + return Option.some( + buildChatAlertBlocks({ + context, + templated: renderTitleBody( + context, + "chat", + linkUrl.value, + stored.chatUrl ?? linkUrl.value, + ), + linkUrl: linkUrl.value, + }), + ) + }) + + return { cardForMessage } satisfies AlertChatCardServiceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts b/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts new file mode 100644 index 0000000000..0d22a88a74 --- /dev/null +++ b/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts @@ -0,0 +1,191 @@ +/** + * What an acknowledgement reaches beyond the incident row: the PagerDuty acknowledgements it + * queues, and the chat card it redraws. + */ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { AlertIncidentId, AlertRuleId, OrgId } from "@maple/domain/http" +import { Effect, Layer, Option, Schema, Array as Arr } from "effect" +import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" +import { AlertChatCardService } from "./AlertChatCardService" +import { AlertIncidentAckService, type AlertIncidentActor } from "./AlertIncidentAckService" + +const ORG = Schema.decodeSync(OrgId)("org_ack_chat") +const RULE = Schema.decodeSync(AlertRuleId)("00000000-0000-4000-8000-000000000030") +const INCIDENT = Schema.decodeSync(AlertIncidentId)("00000000-0000-4000-8000-000000000031") +const RESOLVED = Schema.decodeSync(AlertIncidentId)("00000000-0000-4000-8000-000000000032") + +const PD_RULE = "00000000-0000-4000-8000-0000000000d1" +const PD_DISABLED = "00000000-0000-4000-8000-0000000000d2" +const CHAT_DEST = "00000000-0000-4000-8000-0000000000d3" +const PD_PRIOR = "00000000-0000-4000-8000-0000000000d4" +const PD_UNRELATED = "00000000-0000-4000-8000-0000000000d5" + +const MESSAGE_TS = "1762400000.000100" + +const sam: AlertIncidentActor = { userId: null, name: "sam", source: "slack" } + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const makeLayer = (testDb: TestDb) => + Layer.mergeAll(AlertIncidentAckService.layer, AlertChatCardService.layer).pipe( + Layer.provide(testDb.layer), + ) + +const destination = (id: string, name: string, type: string, enabled: boolean) => + `('${id}', '${ORG}', '${name}', '${type}', ${enabled}, '{}', '', '', '', now(), now(), 'u', 'u')` + +const seed = (testDb: TestDb) => + Effect.promise(async () => { + await executeSql( + testDb, + `INSERT INTO alert_destinations + (id, org_id, name, type, enabled, config_json, secret_ciphertext, secret_iv, secret_tag, + created_at, updated_at, created_by, updated_by) + VALUES ${Arr.join( + [ + destination(PD_RULE, "On-call", "pagerduty", true), + destination(PD_DISABLED, "Old on-call", "pagerduty", false), + destination(CHAT_DEST, "#alerts", "chat", true), + destination(PD_PRIOR, "Former on-call", "pagerduty", true), + destination(PD_UNRELATED, "Other team", "pagerduty", true), + ], + ", ", + )}`, + ) + await executeSql( + testDb, + `INSERT INTO alert_rules + (id, org_id, name, severity, signal_type, comparator, threshold, window_minutes, + destination_ids_json, reducer, no_data_behavior, created_at, updated_at, created_by, updated_by) + VALUES ($1, $2, 'Checkout errors', 'warning', 'error_rate', 'gt', 5, 5, $3::jsonb, + 'avg', 'skip', now(), now(), 'u', 'u')`, + [RULE, ORG, JSON.stringify([PD_RULE, PD_DISABLED, CHAT_DEST])], + ) + await executeSql( + testDb, + `INSERT INTO alert_incidents + (id, org_id, rule_id, incident_key, rule_name, group_key, signal_type, severity, status, + comparator, threshold, first_triggered_at, last_triggered_at, resolved_at, + dedupe_key, created_at, updated_at) + VALUES + ($1, $3, $4, 'k-open', 'Checkout errors', 'checkout', 'error_rate', 'warning', 'open', + 'gt', 5, '2026-11-06T09:00:00Z', '2026-11-06T09:10:00Z', null, 'd-open', now(), now()), + ($2, $3, $4, 'k-res', 'Checkout errors', 'checkout', 'error_rate', 'warning', 'resolved', + 'gt', 5, '2026-11-06T08:00:00Z', '2026-11-06T08:10:00Z', '2026-11-06T08:20:00Z', 'd-res', now(), now())`, + [INCIDENT, RESOLVED, ORG, RULE], + ) + const payload = JSON.stringify({ + eventType: "trigger", + incidentId: INCIDENT, + dedupeKey: "d-open", + rule: { + name: "Checkout errors", + severity: "warning", + signalType: "error_rate", + windowMinutes: 5, + }, + observed: { value: 7.5, sampleCount: 1200 }, + template: null, + chart: null, + linkUrl: `https://app.maple.dev/alerts/incidents/${INCIDENT}`, + chatUrl: `https://app.maple.dev/alerts/rules/${RULE}`, + }) + // The card the channel got, a page the former on-call destination got, and a page to the + // rule's on-call that is still waiting to go out. + await executeSql( + testDb, + `INSERT INTO alert_delivery_events + (id, org_id, incident_id, rule_id, destination_id, delivery_key, event_type, attempt_number, + status, scheduled_at, attempted_at, provider_reference, payload_json, created_at, updated_at) + VALUES + ('00000000-0000-4000-8000-0000000000e1', $1, $2, $3, $4, 'k1', 'trigger', 1, 'success', + '2026-11-06T09:00:00Z', '2026-11-06T09:00:01Z', $5, $6::jsonb, now(), now()), + ('00000000-0000-4000-8000-0000000000e2', $1, $2, $3, $7, 'k2', 'trigger', 1, 'success', + '2026-11-06T09:00:00Z', '2026-11-06T09:00:01Z', 'd-open', $6::jsonb, now(), now()), + ('00000000-0000-4000-8000-0000000000e3', $1, $2, $3, $8, 'k3', 'trigger', 2, 'queued', + '2026-11-06T09:05:00Z', null, null, $6::jsonb, now(), now())`, + [ORG, INCIDENT, RULE, CHAT_DEST, MESSAGE_TS, payload, PD_PRIOR, PD_RULE], + ) + }) + +const queuedAcks = (testDb: TestDb) => + Effect.promise(() => + testDb.pglite.query<{ destination_id: string; status: string; payload_json: { dedupeKey: string } }>( + `SELECT destination_id, status, payload_json FROM alert_delivery_events + WHERE event_type = 'acknowledge' ORDER BY destination_id`, + ), + ).pipe(Effect.map((result) => result.rows)) + +describe("acknowledging an incident queues PagerDuty acknowledgements", () => { + it.effect("only to the PagerDuty destinations this incident already paged", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const acks = yield* AlertIncidentAckService + + yield* acks.acknowledge(ORG, INCIDENT, sam) + + const queued = yield* queuedAcks(testDb) + assert.deepStrictEqual( + Arr.map(queued, (row) => [row.destination_id, row.status, row.payload_json.dedupeKey]), + // The rule's on-call has not been paged yet: an ack there would land before the page. + [[PD_PRIOR, "queued", "d-open"]], + ) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("queues nothing for an unacknowledge or for an ack that changed nothing", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const acks = yield* AlertIncidentAckService + + yield* acks.acknowledge(ORG, INCIDENT, sam) + yield* acks.acknowledge(ORG, INCIDENT, sam) + yield* acks.unacknowledge(ORG, INCIDENT, sam) + + assert.lengthOf(yield* queuedAcks(testDb), 1) + }).pipe(Effect.provide(makeLayer(testDb))) + }) +}) + +describe("redrawing the chat card an ack was clicked on", () => { + it.effect("shows who acknowledged, and the Acknowledge control again once it is undone", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const acks = yield* AlertIncidentAckService + const cards = yield* AlertChatCardService + + yield* acks.acknowledge(ORG, INCIDENT, sam) + const acked = Option.getOrUndefined(yield* cards.cardForMessage(ORG, INCIDENT, MESSAGE_TS)) + const card = acked?.[0] + assert.strictEqual(card?.kind, "alert") + if (card?.kind !== "alert") return + assert.include(card.title, "Checkout errors") + assert.deepStrictEqual(card.acknowledge, { incidentId: INCIDENT, acknowledgedBy: "sam" }) + + yield* acks.unacknowledge(ORG, INCIDENT, sam) + const unacked = Option.getOrUndefined(yield* cards.cardForMessage(ORG, INCIDENT, MESSAGE_TS)) + const again = unacked?.[0] + assert.deepStrictEqual(again?.kind === "alert" ? again.acknowledge : undefined, { + incidentId: INCIDENT, + acknowledgedBy: null, + }) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("finds no card for a message Maple did not post for that incident or org", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const cards = yield* AlertChatCardService + const otherOrg = Schema.decodeSync(OrgId)("org_someone_else") + + assert.isTrue(Option.isNone(yield* cards.cardForMessage(ORG, INCIDENT, "1762400000.999999"))) + assert.isTrue(Option.isNone(yield* cards.cardForMessage(ORG, RESOLVED, MESSAGE_TS))) + assert.isTrue(Option.isNone(yield* cards.cardForMessage(otherOrg, INCIDENT, MESSAGE_TS))) + }).pipe(Effect.provide(makeLayer(testDb))) + }) +}) diff --git a/packages/backend/src/services/alerts/AlertIncidentAckService.ts b/packages/backend/src/services/alerts/AlertIncidentAckService.ts index 4dc8e751ba..bb82fec81e 100644 --- a/packages/backend/src/services/alerts/AlertIncidentAckService.ts +++ b/packages/backend/src/services/alerts/AlertIncidentAckService.ts @@ -4,6 +4,7 @@ * carries the history, which the timeline shows and on-call escalation reads. */ import { + AlertDeliveryEventId, AlertIncidentAckSource, AlertIncidentEventDocument, AlertIncidentEventId, @@ -17,7 +18,15 @@ import { type UserId, } from "@maple/domain/http" import * as PG from "@maple-dev/effect-orm/postgres" -import { AlertIncidentEvents, AlertIncidents } from "@maple/db/tables" +import { + AlertDeliveryEvents, + AlertDestinations, + AlertIncidentEvents, + AlertIncidents, + type AlertIncidentRow, +} from "@maple/db/tables" +import { makeAlertDeliveryKey } from "@maple/alerting-core" +import type { MapleDb } from "@maple/db/client" import { Context, Effect, Layer, Option, Schema, Array as Arr } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" @@ -62,6 +71,7 @@ export interface AlertIncidentAckServiceApi { } const decodeEventId = Schema.decodeUnknownSync(AlertIncidentEventId) +const decodeDeliveryId = Schema.decodeUnknownSync(AlertDeliveryEventId) const decodeEventType = Schema.decodeUnknownOption(AlertIncidentEventType) const decodeSource = Schema.decodeUnknownOption(AlertIncidentAckSource) const decodeIso = Schema.decodeUnknownSync(AlertIncidentEventDocument.fields.createdAt) @@ -96,6 +106,93 @@ export class AlertIncidentAckService extends Context.Service< ), ) + /** + * Queue a PagerDuty `acknowledge` for each enabled PagerDuty destination this incident already + * paged, so it never reaches a pager that was not notified or lands before the page itself. + * Written in the ack's transaction; the alerting dispatcher sends it. + */ + const queuePagerDutyAcknowledge = ( + db: MapleDb, + incident: AlertIncidentRow, + now: number, + makeUuid: () => string, + ) => + Effect.gen(function* () { + const paged = yield* db.run( + PG.from(AlertDeliveryEvents) + .select("destinationId") + .where(($) => [ + $.orgId.eq(incident.orgId), + $.incidentId.eq(incident.id), + $.status.eq("success"), + $.eventType.in_("trigger", "renotify"), + ]), + ) + const candidateIds = Arr.dedupe(Arr.map(paged, (row) => row.destinationId)) + if (candidateIds.length === 0) return 0 + const destinations = yield* db.run( + PG.from(AlertDestinations) + .select("id") + .where(($) => [ + $.orgId.eq(incident.orgId), + $.id.in_(...candidateIds), + $.type.eq("pagerduty"), + $.enabled.eq(true), + ]), + ) + yield* Effect.forEach( + destinations, + (destination) => + db.run( + PG.insertInto(AlertDeliveryEvents) + .values({ + id: decodeDeliveryId(makeUuid()), + orgId: incident.orgId, + incidentId: incident.id, + ruleId: incident.ruleId, + destinationId: destination.id, + deliveryKey: makeAlertDeliveryKey( + incident.id, + destination.id, + "acknowledge", + now, + ), + eventType: "acknowledge", + attemptNumber: 1, + status: "queued", + scheduledAt: now, + claimedAt: null, + claimExpiresAt: null, + claimedBy: null, + attemptedAt: null, + providerMessage: null, + providerReference: null, + responseCode: null, + errorMessage: null, + payloadJson: { + eventType: "acknowledge", + incidentId: incident.id, + incidentStatus: incident.status, + dedupeKey: incident.dedupeKey, + peakSeverity: incident.peakSeverity, + rule: { + id: incident.ruleId, + name: incident.ruleName, + severity: incident.severity, + groupKey: incident.groupKey, + signalType: incident.signalType, + }, + }, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing(), + ), + { discard: true }, + ) + return destinations.length + }) + /** * One guarded UPDATE per change, so two people clicking at once write one event: the loser * matches no row and reads back the state the winner left. @@ -146,6 +243,16 @@ export class AlertIncidentAckService extends Context.Service< ]) .returning(), ) + const changedRow = Arr.head(rows) + if (acknowledged && Option.isSome(changedRow)) { + const queued = yield* queuePagerDutyAcknowledge( + db, + changedRow.value, + now, + () => runtime.makeUuid(), + ) + yield* Effect.annotateCurrentSpan("maple.alert.pagerduty_acks_queued", queued) + } if (rows.length > 0) { yield* db.run( PG.insertInto(AlertIncidentEvents).values({ diff --git a/packages/backend/src/services/alerts/AlertReadModelsService.ts b/packages/backend/src/services/alerts/AlertReadModelsService.ts index bfcf088c73..283e4cac37 100644 --- a/packages/backend/src/services/alerts/AlertReadModelsService.ts +++ b/packages/backend/src/services/alerts/AlertReadModelsService.ts @@ -86,6 +86,8 @@ export interface ListAlertDeliveryEventsOptions { readonly offset?: number readonly incidentId?: AlertIncidentId readonly ruleId?: AlertRuleId + /** Leave out the internal PagerDuty `acknowledge` syncs, which the public API does not report. */ + readonly notificationsOnly?: boolean } export interface AlertReadModelsServiceApi { @@ -483,7 +485,7 @@ export class AlertReadModelsService extends Context.Service< options: ListAlertDeliveryEventsOptions = {}, ) { yield* Effect.annotateCurrentSpan("orgId", orgId) - const { incidentId, ruleId } = options + const { incidentId, ruleId, notificationsOnly } = options const rows = yield* dbExecute((db) => db.run( PG.from(AlertDeliveryEvents) @@ -492,6 +494,7 @@ export class AlertReadModelsService extends Context.Service< $.orgId.eq(orgId), incidentId === undefined ? undefined : $.incidentId.eq(incidentId), ruleId === undefined ? undefined : $.ruleId.eq(ruleId), + notificationsOnly === true ? $.eventType.neq("acknowledge") : undefined, ]) .orderBy(["createdAt", "desc"], ["id", "desc"]) .limit(options.limit ?? 100) diff --git a/packages/backend/src/services/alerts/AlertsService.delivery.test.ts b/packages/backend/src/services/alerts/AlertsService.delivery.test.ts index dd70c20e1d..a8a9203736 100644 --- a/packages/backend/src/services/alerts/AlertsService.delivery.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.delivery.test.ts @@ -4,6 +4,7 @@ import { AlertDeliveryError, AlertDeliveryRejectedError, ChatAlertDestinationConfig, + PagerDutyAlertDestinationConfig, WebhookAlertDestinationConfig, } from "@maple/domain/http" import { ChatAlertPoster } from "./ChatAlertPoster" @@ -1129,3 +1130,120 @@ describe("AlertsService held incidents", () => { }).pipe(Effect.provide(makeLayer(testDb, makeLivenessStub(state), { fetch: okFetch }))) }) }) + +describe("AlertsService PagerDuty acknowledgements", () => { + const fixedTime = 1_710_000_400_000 + + const run = (acknowledged: boolean) => { + const testDb = createTestDb(trackedDbs) + const bodies: Array = [] + const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => { + bodies.push(requestBodyText(init?.body)) + return new Response("{}", { status: 202 }) + }) as typeof fetch + return Effect.gen(function* () { + yield* TestClock.setTime(fixedTime) + const alerts = yield* AlertsService + const orgId = asOrgId("org_pd_ack") + const userId = asUserId("user_pd_ack") + const destination = yield* alerts.createDestination( + orgId, + userId, + adminRoles, + new PagerDutyAlertDestinationConfig({ + type: "pagerduty", + name: "On-call", + enabled: true, + integrationKey: "abcdefghijklmnopqrstuvwxyz012345", + }), + ) + const rule = yield* createErrorRateRule(alerts, orgId, userId, destination.id) + // Break the rule's stored query so the tick queues nothing of its own. + yield* Effect.promise(() => + executeSql(testDb, "update alert_rules set query_spec_json = $1::jsonb where id = $2", [ + "{}", + rule.id, + ]), + ) + const incidentId = "00000000-0000-4000-8000-000000000401" + yield* Effect.promise(() => + executeSql( + testDb, + `INSERT INTO alert_incidents + (id, org_id, rule_id, incident_key, rule_name, group_key, signal_type, severity, status, + comparator, threshold, first_triggered_at, last_triggered_at, dedupe_key, + last_notified_at, acknowledged_at, acknowledged_by_name, acknowledged_via, + created_at, updated_at) + VALUES ($1, $2, $3, 'k', 'Checkout error rate', '__total__', 'error_rate', 'warning', 'open', + 'gt', 5, now(), now(), 'dedupe-ack', to_timestamp(1000), $4, 'sam', 'slack', now(), now())`, + [incidentId, orgId, rule.id, acknowledged ? new Date(fixedTime - 1000) : null], + ), + ) + bodies.length = 0 + yield* Effect.promise(() => + insertDeliveryEventRow(testDb, { + id: "00000000-0000-4000-8000-000000000402", + orgId, + incidentId, + ruleId: rule.id, + destinationId: destination.id, + deliveryKey: "ack-1", + eventType: "acknowledge", + attemptNumber: 1, + status: "queued", + scheduledAt: fixedTime - 500, + payloadJson: JSON.stringify({ + eventType: "acknowledge", + incidentId, + dedupeKey: "dedupe-ack", + rule: { id: rule.id, name: "Checkout error rate", severity: "warning" }, + }), + }), + ) + yield* alerts.runSchedulerTick() + const row = yield* Effect.promise(() => + queryFirstRow<{ status: string; message: string | null; notified: Date | null }>( + testDb, + `select d.status, d.provider_message as message, i.last_notified_at as notified + from alert_delivery_events d join alert_incidents i on i.id = d.incident_id + where d.delivery_key = 'ack-1'`, + [], + ), + ) + return { bodies, row } + }).pipe( + Effect.provide( + makeLayer(testDb, makeWarehouseStub({ tracesAggregateRows: emptyWarehouseRows }), { + fetch: fetchImpl, + }), + ), + ) + } + + it.effect("sends the queued acknowledge without restarting the reminder clock", () => + Effect.gen(function* () { + const { bodies, row } = yield* run(true) + assert.deepStrictEqual( + Arr.map(bodies, (body) => JSON.parse(body)), + [ + { + routing_key: "abcdefghijklmnopqrstuvwxyz012345", + event_action: "acknowledge", + dedup_key: "dedupe-ack", + }, + ], + ) + assert.strictEqual(row?.status, "success") + assert.strictEqual(row?.notified?.getTime(), 1_000_000) + }), + ) + + it.effect("skips an acknowledge the incident no longer holds, so a fresh page stays loud", () => + Effect.gen(function* () { + const { bodies, row } = yield* run(false) + assert.deepStrictEqual(bodies, []) + assert.strictEqual(row?.status, "success") + assert.strictEqual(row?.message, "Skipped: the incident is no longer acknowledged") + }), + ) +}) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index e50a56a811..2fe8b312e0 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -58,7 +58,6 @@ import { type AlertSeverity, AlertSignalType as AlertSignalTypeSchema, AlertValidationError, - AlertNotificationTemplate, type AlertEventType as AlertEventTypeValue, type AlertRuleUpsertRequest, type AlertGroupBy, @@ -188,6 +187,8 @@ import { managedRuleMessage, } from "./AlertRuleModel" import { mapSignalUnit, resolveSignalDisplay } from "./alert-signal-display" +import { StoredDeliveryPayloadSchema, type StoredDeliveryPayload } from "./delivery/stored-payload" +import { incidentAcknowledgedBy } from "./delivery/transports/chat" import { summarizeCause } from "@maple/backend/platform/describe-cause" const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)) @@ -394,48 +395,6 @@ const DELIVERY_LEASE_TTL_MS = 30_000 /* Schemas for stored JSON formats */ /* -------------------------------------------------------------------------- */ -const StoredDeliveryPayloadSchema = Schema.StructWithRest( - Schema.Struct({ - event: Schema.optionalKey(Schema.Unknown), - eventType: Schema.optionalKey(Schema.String), - incidentId: Schema.optionalKey(Schema.NullOr(Schema.String)), - incidentStatus: Schema.optionalKey(Schema.String), - dedupeKey: Schema.optionalKey(Schema.String), - rule: Schema.optionalKey( - Schema.Struct({ - id: Schema.optionalKey(Schema.String), - name: Schema.optionalKey(Schema.String), - signalType: Schema.optionalKey(Schema.String), - severity: Schema.optionalKey(Schema.String), - groupKey: Schema.optionalKey(Schema.NullOr(Schema.String)), - comparator: Schema.optionalKey(Schema.String), - threshold: Schema.optionalKey(Schema.Number), - thresholdUpper: Schema.optionalKey(Schema.NullOr(Schema.Number)), - windowMinutes: Schema.optionalKey(Schema.Number), - severities: Schema.optionalKey(Schema.Array(Schema.String)), - }), - ), - peakSeverity: Schema.optionalKey(Schema.NullOr(Schema.String)), - observed: Schema.optionalKey( - Schema.Struct({ - value: Schema.optionalKey(Schema.NullOr(Schema.Number)), - sampleCount: Schema.optionalKey(Schema.NullOr(Schema.Number)), - detail: Schema.optionalKey(Schema.NullOr(Schema.String)), - }), - ), - template: Schema.optionalKey(Schema.NullOr(AlertNotificationTemplate)), - chart: Schema.optionalKey( - Schema.NullOr( - Schema.Struct({ - sparkline: Schema.optionalKey(Schema.String), - url: Schema.optionalKey(Schema.String), - }), - ), - ), - }), - [Schema.Record(Schema.String, Schema.Unknown)], -) - const decodeAlertRuleIdSync = Schema.decodeUnknownSync(AlertRuleDocument.fields.id) const decodeAlertIncidentIdSync = Schema.decodeUnknownSync(AlertIncidentDocument.fields.id) const decodeAlertDeliveryEventIdSync = Schema.decodeUnknownSync(AlertDeliveryEventDocument.fields.id) @@ -534,7 +493,7 @@ const isolate = const toIngestDateTime64 = warehouseDateTime64 -type StoredDeliveryPayloadType = Schema.Schema.Type +type StoredDeliveryPayloadType = StoredDeliveryPayload const parseDeliveryPayload = ( value: unknown, @@ -2422,6 +2381,36 @@ export class AlertsService extends Context.Service + db.run( + PG.from(AlertIncidents) + .select("id") + .where(($) => [ + $.orgId.eq(row.orgId), + $.id.eq(ackIncidentId), + $.status.eq("open"), + $.acknowledgedAt.isNotNull(), + ]) + .limit(1), + ), + )).length === 0) + if (isStaleAcknowledgement) { + yield* finalizeClaimedDelivery(row.id, claimTime, { + status: "success", + attemptedAt: claimTime, + providerMessage: "Skipped: the incident is no longer acknowledged", + providerReference: null, + responseCode: null, + errorMessage: null, + }) + return + } const ruleRow = ruleMap.get(row.ruleId) ?? null const payloadRule = payload.rule const storedRule = ruleRow ? yield* decodeStoredAlertRuleMetadata(ruleRow) : null @@ -2481,6 +2470,7 @@ export class AlertsService extends Context.Service db.run( diff --git a/packages/backend/src/services/alerts/delivery/context.ts b/packages/backend/src/services/alerts/delivery/context.ts index a56f10468c..b113115136 100644 --- a/packages/backend/src/services/alerts/delivery/context.ts +++ b/packages/backend/src/services/alerts/delivery/context.ts @@ -57,6 +57,8 @@ export interface DispatchContext { readonly eventType: AlertEventType readonly incidentId: string | null readonly incidentStatus: string + /** Who acknowledged the open incident, as a name to show; absent or null while unacknowledged. */ + readonly incidentAcknowledgedBy?: string | null readonly dedupeKey: string readonly windowMinutes: number readonly value: number | null diff --git a/packages/backend/src/services/alerts/delivery/dispatch.ts b/packages/backend/src/services/alerts/delivery/dispatch.ts index 2e514f9b75..d615e1ab2a 100644 --- a/packages/backend/src/services/alerts/delivery/dispatch.ts +++ b/packages/backend/src/services/alerts/delivery/dispatch.ts @@ -35,6 +35,14 @@ export const dispatchDelivery = ( /** `sendEmail` (the platform email channel) and `postChatAlert` (a chat connector). */ deps: EffectTransportDeps, ): Effect.Effect => { + // Only PagerDuty has an acknowledge; the queue never sends one elsewhere, this keeps it so. + if (context.eventType === "acknowledge" && context.secretConfig.type !== "pagerduty") { + return Effect.succeed({ + providerMessage: "Skipped: acknowledgements are only sent to PagerDuty", + providerReference: null, + responseCode: null, + }) + } const runtime: TransportRuntime = { timeoutMs } /** * Resolved once here rather than by each provider: the template is a diff --git a/packages/backend/src/services/alerts/delivery/paging-dedup.ts b/packages/backend/src/services/alerts/delivery/paging-dedup.ts index 122dbc955a..067fb02675 100644 --- a/packages/backend/src/services/alerts/delivery/paging-dedup.ts +++ b/packages/backend/src/services/alerts/delivery/paging-dedup.ts @@ -12,7 +12,7 @@ import type { DispatchContext } from "./context" * unheard. Every other rule keeps the incident's single key, which is what its * already-open alerts were raised under. */ -const criticalHasOwnKey = (context: Pick): boolean => +export const criticalHasOwnKey = (context: Pick): boolean => context.ruleSeverities !== undefined && context.ruleSeverities.includes("warning") && context.ruleSeverities.includes("critical") diff --git a/packages/backend/src/services/alerts/delivery/stored-payload.ts b/packages/backend/src/services/alerts/delivery/stored-payload.ts new file mode 100644 index 0000000000..753a46578f --- /dev/null +++ b/packages/backend/src/services/alerts/delivery/stored-payload.ts @@ -0,0 +1,52 @@ +/** + * The JSON an `alert_delivery_events` row carries: what the event saw, snapshotted when it was + * queued. Read by the dispatcher, and by the chat card refresh to redraw a card it posted. + */ +import { AlertNotificationTemplate } from "@maple/domain/http" +import { Schema } from "effect" + +export const StoredDeliveryPayloadSchema = Schema.StructWithRest( + Schema.Struct({ + event: Schema.optionalKey(Schema.Unknown), + eventType: Schema.optionalKey(Schema.String), + incidentId: Schema.optionalKey(Schema.NullOr(Schema.String)), + incidentStatus: Schema.optionalKey(Schema.String), + dedupeKey: Schema.optionalKey(Schema.String), + rule: Schema.optionalKey( + Schema.Struct({ + id: Schema.optionalKey(Schema.String), + name: Schema.optionalKey(Schema.String), + signalType: Schema.optionalKey(Schema.String), + severity: Schema.optionalKey(Schema.String), + groupKey: Schema.optionalKey(Schema.NullOr(Schema.String)), + comparator: Schema.optionalKey(Schema.String), + threshold: Schema.optionalKey(Schema.Number), + thresholdUpper: Schema.optionalKey(Schema.NullOr(Schema.Number)), + windowMinutes: Schema.optionalKey(Schema.Number), + severities: Schema.optionalKey(Schema.Array(Schema.String)), + }), + ), + peakSeverity: Schema.optionalKey(Schema.NullOr(Schema.String)), + observed: Schema.optionalKey( + Schema.Struct({ + value: Schema.optionalKey(Schema.NullOr(Schema.Number)), + sampleCount: Schema.optionalKey(Schema.NullOr(Schema.Number)), + detail: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), + ), + template: Schema.optionalKey(Schema.NullOr(AlertNotificationTemplate)), + linkUrl: Schema.optionalKey(Schema.String), + chatUrl: Schema.optionalKey(Schema.String), + chart: Schema.optionalKey( + Schema.NullOr( + Schema.Struct({ + sparkline: Schema.optionalKey(Schema.String), + url: Schema.optionalKey(Schema.String), + }), + ), + ), + }), + [Schema.Record(Schema.String, Schema.Unknown)], +) + +export type StoredDeliveryPayload = Schema.Schema.Type diff --git a/packages/backend/src/services/alerts/delivery/transports/chat.test.ts b/packages/backend/src/services/alerts/delivery/transports/chat.test.ts index 2fef43fb6b..487356c2af 100644 --- a/packages/backend/src/services/alerts/delivery/transports/chat.test.ts +++ b/packages/backend/src/services/alerts/delivery/transports/chat.test.ts @@ -117,6 +117,28 @@ describe("chat alert blocks", () => { assert.strictEqual(card.sentAtMs, 1_700_000_000_000) }) + it("offers an open incident's firing card the ack control, with who acknowledged it", () => { + assert.deepStrictEqual(cardOf(buildChatAlertBlocks(inputFor())).acknowledge, { + incidentId: "inc_1", + acknowledgedBy: null, + }) + const acked = cardOf( + buildChatAlertBlocks(inputFor({ context: { ...context, incidentAcknowledgedBy: "Ada" } })), + ) + assert.deepStrictEqual(acked.acknowledge, { incidentId: "inc_1", acknowledgedBy: "Ada" }) + }) + + it("gives no ack control to a resolve, a test or a resolved incident", () => { + for (const overrides of [ + { eventType: "resolve" as const }, + { eventType: "test" as const }, + { incidentStatus: "resolved" }, + ]) { + const card = cardOf(buildChatAlertBlocks(inputFor({ context: { ...context, ...overrides } }))) + assert.isUndefined(card.acknowledge) + } + }) + it("colours a resolve green", () => { const card = cardOf(buildChatAlertBlocks(inputFor({ context: { ...context, eventType: "resolve" } }))) assert.strictEqual(card.color, "#2eb67d") diff --git a/packages/backend/src/services/alerts/delivery/transports/chat.ts b/packages/backend/src/services/alerts/delivery/transports/chat.ts index a03a0a508f..d149df76a2 100644 --- a/packages/backend/src/services/alerts/delivery/transports/chat.ts +++ b/packages/backend/src/services/alerts/delivery/transports/chat.ts @@ -10,7 +10,8 @@ import { alertAccentColor, truncate, } from "../../alert-formatting" -import type { EffectTransport, RenderInput, SecretConfigOf } from "../Transport" +import type { TemplateRenderContext } from "../../alert-formatting" +import type { EffectTransport, ResolvedTitleBody, SecretConfigOf } from "../Transport" type Config = SecretConfigOf<"chat"> @@ -29,10 +30,42 @@ const MAX_TEMPLATED_BODY_CHARS = 1200 * and a footer carrying the sparkline, the incident and the time. A rule's own template, when it * has one, replaces the title and the summary and drops the fields, as it does there. */ -export const buildChatAlertBlocks = (input: RenderInput): ReadonlyArray => { +export interface ChatAlertCardInput { + readonly context: TemplateRenderContext & { readonly incidentAcknowledgedBy?: string | null } + readonly templated: ResolvedTitleBody | null + readonly linkUrl: string +} + +/** The name an open, acknowledged incident shows on its alert cards; null otherwise. */ +export const incidentAcknowledgedBy = ( + incident: { + readonly status: string + readonly acknowledgedAt: number | null + readonly acknowledgedByName: string | null + } | null, +): string | null => + incident === null || incident.status !== "open" || incident.acknowledgedAt === null + ? null + : (incident.acknowledgedByName ?? "someone") + +/** Only a firing card of an open incident can be acknowledged from chat. */ +const ackControl = (context: ChatAlertCardInput["context"]): Pick => + context.incidentId !== null && + context.incidentStatus === "open" && + (context.eventType === "trigger" || context.eventType === "renotify") + ? { + acknowledge: { + incidentId: context.incidentId, + acknowledgedBy: context.incidentAcknowledgedBy ?? null, + }, + } + : {} + +export const buildChatAlertBlocks = (input: ChatAlertCardInput): ReadonlyArray => { const { context, templated, linkUrl } = input const group = displayGroupKey(context.groupKey) const card: Omit = { + ...ackControl(context), kind: "alert", color: alertAccentColor(context.eventType, context.severity), imageUrl: context.chartUrl ?? null, diff --git a/packages/backend/src/services/alerts/delivery/transports/linear.ts b/packages/backend/src/services/alerts/delivery/transports/linear.ts index 4d3c584269..b59b893e57 100644 --- a/packages/backend/src/services/alerts/delivery/transports/linear.ts +++ b/packages/backend/src/services/alerts/delivery/transports/linear.ts @@ -456,6 +456,12 @@ export const linearTransport: EffectTransport = { providerReference: null, responseCode: null, } satisfies DispatchResult + case "acknowledge": + return { + providerMessage: "Skipped: acknowledgements are only sent to PagerDuty", + providerReference: null, + responseCode: null, + } satisfies DispatchResult case "test": { const team = yield* fetchTeam(deps.linear, input) return { diff --git a/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts b/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts index 9f52161ddf..a2cac0ac52 100644 --- a/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts +++ b/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts @@ -1,9 +1,11 @@ -import { Duration, Effect, Schema, String as Str } from "effect" +import { Duration, Effect, Schema, Array as Arr, String as Str } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" import { displayGroupKey, formatAlertHeadline, truncate, uptimeSummaryLine } from "../../alert-formatting" +import type { DispatchContext } from "../context" import { closesCriticalKey, criticalDedupKey, + criticalHasOwnKey, keySeverity, pagingDedupKey, primaryDedupKey, @@ -16,14 +18,31 @@ type Config = SecretConfigOf<"pagerduty"> const EVENTS_URL = "https://events.pagerduty.com/v2/enqueue" -const resolveRequest = (integrationKey: string, dedupKey: string): HttpRequestSpec => ({ +const actionRequest = ( + integrationKey: string, + action: "resolve" | "acknowledge", + dedupKey: string, +): HttpRequestSpec => ({ url: EVENTS_URL, headers: { "content-type": "application/json" }, guarded: false, sensitivePath: false, - body: encodeJson({ routing_key: integrationKey, event_action: "resolve", dedup_key: dedupKey }), + body: encodeJson({ routing_key: integrationKey, event_action: action, dedup_key: dedupKey }), }) +/** + * An acknowledgement covers every alert the incident may hold open: its base key, and the critical + * key when the rule pages critical under its own key or the incident ever reached critical (the + * rule may have dropped that tier since). PagerDuty ignores an acknowledge for a key it has no + * open alert for. There is no un-acknowledge in the Events API. + */ +export const pagerDutyAcknowledgeKeys = ( + context: Pick, +): ReadonlyArray => + criticalHasOwnKey(context) || context.incidentPeakSeverity === "critical" + ? [context.dedupeKey, criticalDedupKey(context.dedupeKey)] + : [context.dedupeKey] + /** * PagerDuty Events API v2. The body is machine-facing, so `custom_details` * carries raw numbers rather than the formatted strings the chat providers @@ -37,6 +56,10 @@ export const pagerDutyTransport: HttpTransport = { providerLabel: "PagerDuty", render: (input: RenderInput) => { const { context, templated, linkUrl, chatUrl } = input + if (context.eventType === "acknowledge") { + const [primary = context.dedupeKey] = pagerDutyAcknowledgeKeys(context) + return actionRequest(input.config.integrationKey, "acknowledge", primary) + } return { // Fixed vendor host: nothing user-supplied to guard against, and the // path carries no credential (the routing key rides in the body). @@ -80,12 +103,22 @@ export const pagerDutyTransport: HttpTransport = { } }, // The critical alert closes when the incident resolves or falls back to warning. - followUps: (input) => - closesCriticalKey(input.context) - ? [resolveRequest(input.config.integrationKey, criticalDedupKey(input.context.dedupeKey))] - : [], + followUps: (input) => { + const { context } = input + if (context.eventType === "acknowledge") { + return Arr.map(Arr.drop(pagerDutyAcknowledgeKeys(context), 1), (key) => + actionRequest(input.config.integrationKey, "acknowledge", key), + ) + } + return closesCriticalKey(context) + ? [actionRequest(input.config.integrationKey, "resolve", criticalDedupKey(context.dedupeKey))] + : [] + }, ack: (input) => ({ - providerMessage: "Delivered to PagerDuty", + providerMessage: + input.context.eventType === "acknowledge" + ? "Acknowledged in PagerDuty" + : "Delivered to PagerDuty", providerReference: pagingDedupKey(input.context, keySeverity(input.context)), }), } diff --git a/packages/backend/src/services/alerts/delivery/transports/render.test.ts b/packages/backend/src/services/alerts/delivery/transports/render.test.ts index a440a8cf5a..f70beae3d4 100644 --- a/packages/backend/src/services/alerts/delivery/transports/render.test.ts +++ b/packages/backend/src/services/alerts/delivery/transports/render.test.ts @@ -440,3 +440,42 @@ describe("transport render: uptime monitors", () => { ) }) }) + +describe("pagerduty acknowledge", () => { + const ack = (overrides: Partial) => { + const input = inputFor( + { type: "pagerduty" as const, integrationKey: "k" }, + { context: { ...context, eventType: "acknowledge", ...overrides } }, + ) + return Arr.map( + [pagerDutyTransport.render(input), ...(pagerDutyTransport.followUps?.(input) ?? [])], + (spec) => JSON.parse(spec.body), + ) + } + const tiered = { ruleSeverities: ["warning", "critical"] as const } + + it("acknowledges the incident's key with a bare acknowledge event", () => { + assert.deepStrictEqual(ack({ ruleSeverities: ["critical"] }), [ + { routing_key: "k", event_action: "acknowledge", dedup_key: "org_1:rule_1:checkout" }, + ]) + }) + + it("acknowledges both keys an incident of a tiered rule may hold open, whatever its tier now", () => { + const keys = (overrides: Partial) => + Arr.map(ack(overrides), (body) => `${body.event_action} ${body.dedup_key}`) + const both = ["acknowledge org_1:rule_1:checkout", "acknowledge org_1:rule_1:checkout:critical"] + assert.deepStrictEqual(keys({ ...tiered, severity: "critical", incidentSeverity: "critical" }), both) + // A warning alert stays open beside a critical one; PagerDuty ignores a key with no open alert. + assert.deepStrictEqual(keys({ ...tiered, severity: "warning", incidentSeverity: "warning" }), both) + }) + + it("acknowledges the critical key of an incident that peaked there, after the rule lost its tiers", () => { + assert.deepStrictEqual( + Arr.map( + ack({ ruleSeverities: ["critical"], incidentPeakSeverity: "critical" }), + (body) => `${body.event_action} ${body.dedup_key}`, + ), + ["acknowledge org_1:rule_1:checkout", "acknowledge org_1:rule_1:checkout:critical"], + ) + }) +}) diff --git a/packages/backend/src/services/push/MobilePushService.ts b/packages/backend/src/services/push/MobilePushService.ts index 727f8d5957..48b680108f 100644 --- a/packages/backend/src/services/push/MobilePushService.ts +++ b/packages/backend/src/services/push/MobilePushService.ts @@ -179,6 +179,7 @@ const wantsEvent = (device: MobileDevice, event: IncidentPushEvent): boolean => ? device.preferences.criticalIncidents : device.preferences.warningIncidents case "test": + case "acknowledge": return false } } @@ -354,6 +355,8 @@ export const renderIncidentPush = ( } case "trigger": case "test": + // Never pushed (`wantsEvent` refuses it); rendered like a trigger to keep the switch total. + case "acknowledge": return { alert: { title: `${severity} · ${rule}`, diff --git a/packages/chat-platform/src/action-token.test.ts b/packages/chat-platform/src/action-token.test.ts index 28b739b44b..9cf422f8a0 100644 --- a/packages/chat-platform/src/action-token.test.ts +++ b/packages/chat-platform/src/action-token.test.ts @@ -1,9 +1,11 @@ -import { Option } from "effect" +import { Option, String as Str } from "effect" import { describe, expect, it } from "vitest" import { chatActionControlId, chatActionDecision, + chatAlertAckControlId, decodeChatActionControlId, + decodeChatAlertAckControlId, encodeChatActionToken, } from "./action-token" @@ -50,3 +52,40 @@ describe("chat action control id", () => { expect(Option.getOrUndefined(chatActionDecision(`some-other-app:${token}`))).toBeUndefined() }) }) + +describe("alert ack control id", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + const read = (raw: string) => Option.getOrUndefined(decodeChatAlertAckControlId(raw)) + + it("round-trips both actions with the incident they name", () => { + expect(read(chatAlertAckControlId("acknowledge", INCIDENT))).toEqual({ + action: "acknowledge", + incidentId: INCIDENT, + }) + expect(read(chatAlertAckControlId("unacknowledge", INCIDENT))).toEqual({ + action: "unacknowledge", + incidentId: INCIDENT, + }) + }) + + it("fits the tightest platform's 100-character control id", () => { + expect(chatAlertAckControlId("unacknowledge", INCIDENT).length).toBeLessThanOrEqual(100) + }) + + it.each([ + "", + "alert_ack:", + "alert_ack:not-a-uuid", + `alert_ack:${INCIDENT}x`, + `alert_ack:${Str.toUpperCase(INCIDENT)}`, + `alert_ack:org_other:${INCIDENT}`, + `alert_snooze:${INCIDENT}`, + `approve:${INCIDENT}`, + ])("rejects %j", (raw) => { + expect(read(raw)).toBeUndefined() + }) + + it("is not mistaken for an approval control", () => { + expect(readControl(chatAlertAckControlId("acknowledge", INCIDENT))).toBeUndefined() + }) +}) diff --git a/packages/chat-platform/src/action-token.ts b/packages/chat-platform/src/action-token.ts index 5e4c932926..f08fe410bb 100644 --- a/packages/chat-platform/src/action-token.ts +++ b/packages/chat-platform/src/action-token.ts @@ -79,3 +79,38 @@ export const chatActionDecision = (raw: string): Option.Option Str.startsWith(`${candidate}${DECISION_SEPARATOR}`)(raw), ) + +/** The two things an alert card's button can do to its incident. */ +export type ChatAlertAckAction = "acknowledge" | "unacknowledge" + +export interface ChatAlertAckRequest { + readonly action: ChatAlertAckAction + readonly incidentId: string +} + +const ALERT_ACK_PREFIXES = { + acknowledge: "alert_ack:", + unacknowledge: "alert_unack:", +} as const satisfies Record + +const ALERT_ACK_ACTIONS: ReadonlyArray = ["acknowledge", "unacknowledge"] + +// An incident id is a UUID. Anything else on an ack prefix is forged or corrupt. +const INCIDENT_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + +/** + * An alert card's ack button. Carries the incident only: the org always comes from the workspace + * the click landed in, so a forged id can at most name an incident of that org. + */ +export const chatAlertAckControlId = (action: ChatAlertAckAction, incidentId: string): string => + `${ALERT_ACK_PREFIXES[action]}${incidentId}` + +/** Read an ack button back. `None` for anything that is not exactly one of Maple's. */ +export const decodeChatAlertAckControlId = (raw: string): Option.Option => + Option.flatMap( + Arr.findFirst(ALERT_ACK_ACTIONS, (action) => Str.startsWith(ALERT_ACK_PREFIXES[action])(raw)), + (action) => { + const incidentId = Str.slice(ALERT_ACK_PREFIXES[action].length)(raw) + return INCIDENT_ID_PATTERN.test(incidentId) ? Option.some({ action, incidentId }) : Option.none() + }, + ) diff --git a/packages/chat-platform/src/connectors/discord/render.test.ts b/packages/chat-platform/src/connectors/discord/render.test.ts index 4105460157..74b1a73298 100644 --- a/packages/chat-platform/src/connectors/discord/render.test.ts +++ b/packages/chat-platform/src/connectors/discord/render.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import type { ChatAlertBlock, ChatBlock } from "../../render/blocks" -import type { ChatActionToken } from "../../action-token" +import { decodeChatAlertAckControlId, type ChatActionToken } from "../../action-token" import { EMPTY_CONTENT, MAX_ACTION_ROWS, MAX_CONTENT_CHARS, renderDiscordMessage } from "./render" import { Array as Arr, Option, String as Str } from "effect" @@ -228,3 +228,43 @@ describe("renderDiscordMessage", () => { ]) }) }) + +describe("an alert's acknowledge control on Discord", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + + it("puts an Acknowledge button under an unacknowledged alert", () => { + const payload = renderDiscordMessage([ + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: null } }), + ]) + expect(payload.components).toHaveLength(1) + const button = payload.components[0]?.components[0] + expect(button?.label).toBe("Acknowledge") + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.custom_id ?? ""))).toEqual({ + action: "acknowledge", + incidentId: INCIDENT, + }) + expect(Arr.some(payload.embeds[0]?.fields ?? [], (field) => field.name === "Acknowledged by")).toBe( + false, + ) + }) + + it("names who acknowledged on the embed and offers Unacknowledge", () => { + const payload = renderDiscordMessage([ + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: "Ada" } }), + ]) + const button = payload.components[0]?.components[0] + expect(button?.label).toBe("Unacknowledge") + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.custom_id ?? ""))?.action).toBe( + "unacknowledge", + ) + expect(payload.embeds[0]?.fields?.[0]).toEqual({ + name: "Acknowledged by", + value: "Ada", + inline: false, + }) + }) + + it("adds no component to an alert without the control", () => { + expect(renderDiscordMessage([alert()]).components).toEqual([]) + }) +}) diff --git a/packages/chat-platform/src/connectors/discord/render.ts b/packages/chat-platform/src/connectors/discord/render.ts index 9db810fc2b..385778d1bf 100644 --- a/packages/chat-platform/src/connectors/discord/render.ts +++ b/packages/chat-platform/src/connectors/discord/render.ts @@ -8,8 +8,8 @@ * characters, a message carries at most 10 embeds, an action row at most 5 buttons, and a button's * `custom_id` is 1–100 characters. */ -import { chatActionControlId, type ChatActionToken } from "../../action-token" -import type { ChatAlertBlock, ChatBlock, ChatToolActivity } from "../../render/blocks" +import { chatActionControlId, chatAlertAckControlId, type ChatActionToken } from "../../action-token" +import type { ChatAlertAcknowledge, ChatAlertBlock, ChatBlock, ChatToolActivity } from "../../render/blocks" import { Array as Arr, Option, String as Str } from "effect" /** https://discord.com/developers — Create/Edit Message JSON params, API v10. */ @@ -32,8 +32,8 @@ export interface DiscordEmbed { export interface DiscordButton { readonly type: 2 - /** 3 = success, 4 = danger. */ - readonly style: 3 | 4 + /** 1 = primary, 2 = secondary, 3 = success, 4 = danger. */ + readonly style: 1 | 2 | 3 | 4 readonly label: string readonly custom_id: string } @@ -141,6 +141,9 @@ export const renderDiscordMessage = (blocks: ReadonlyArray): DiscordM lines.push(block.tone === "error" ? `**${block.text}**` : `_${block.text}_`) break case "alert": { + if (block.acknowledge !== undefined && rows.length < MAX_ACTION_ROWS) { + rows.push(alertAckRow(block.acknowledge)) + } const embed = alertEmbed(block) const cost = embedCost(embed) if (embeds.length < MAX_EMBEDS && embedChars + cost <= MAX_EMBED_TOTAL_CHARS) { @@ -189,8 +192,12 @@ const alertEmbed = (block: ChatAlertBlock): DiscordEmbed => { Arr.map(block.links, (link) => `[${link.label}](${link.url})`), " · ", ) + const acknowledgedBy = block.acknowledge?.acknowledgedBy ?? null const fields = Arr.map( [ + ...(acknowledgedBy === null + ? [] + : [{ name: "Acknowledged by", value: acknowledgedBy, inline: false }]), ...Arr.map(block.fields, (field) => ({ name: field.label, value: field.value, inline: true })), ...(links === "" ? [] : [{ name: "Links", value: links, inline: false }]), ].slice(0, MAX_EMBED_FIELDS), @@ -246,6 +253,25 @@ const toolLabel = (tool: ChatToolActivity): string => { return `${tool.label}${detail}${status}` } +/** An alert's ack toggle: Acknowledge while open and unacknowledged, Unacknowledge once acked. */ +const alertAckRow = (ack: ChatAlertAcknowledge): DiscordActionRow => { + const acknowledged = ack.acknowledgedBy !== null + return { + type: 1, + components: [ + { + type: 2, + style: acknowledged ? 2 : 1, + label: acknowledged ? "Unacknowledge" : "Acknowledge", + custom_id: chatAlertAckControlId( + acknowledged ? "unacknowledge" : "acknowledge", + ack.incidentId, + ), + }, + ], + } +} + /** * The two buttons, carrying the driver's action control id through Discord's `custom_id`. * diff --git a/packages/chat-platform/src/connectors/slack/render.test.ts b/packages/chat-platform/src/connectors/slack/render.test.ts index 35fa3f5c40..d8dd4be11e 100644 --- a/packages/chat-platform/src/connectors/slack/render.test.ts +++ b/packages/chat-platform/src/connectors/slack/render.test.ts @@ -4,7 +4,12 @@ */ import { Option, Array as Arr, String as Str } from "effect" import { describe, expect, it } from "vitest" -import { decodeChatActionControlId, encodeChatActionToken, type ChatActionToken } from "../../action-token" +import { + decodeChatActionControlId, + decodeChatAlertAckControlId, + encodeChatActionToken, + type ChatActionToken, +} from "../../action-token" import type { ChatAlertBlock, ChatBlock } from "../../render/blocks" import { APPROVE_ACTION, @@ -300,4 +305,60 @@ describe("the rest of the vocabulary", () => { ?.blocks[1] expect(section?.type === "section" && section.text.text).toBe("<!channel> look") }) + + describe("an alert's acknowledge control", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + const ackButton = (block: ChatAlertBlock) => { + const blocks = renderSlackMessage([block]).attachments?.[0]?.blocks ?? [] + const actions = Option.getOrUndefined(Arr.findFirst(blocks, (b) => b.type === "actions")) + return { + blocks, + button: + actions?.type === "actions" + ? Option.getOrUndefined(Arr.findFirst(actions.elements, (b) => b.value !== undefined)) + : undefined, + } + } + + it("offers Acknowledge on an unacknowledged incident, carrying a control the host can read", () => { + const { blocks, button } = ackButton( + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: null } }), + ) + expect(button?.text.text).toBe("Acknowledge") + expect(button?.url).toBeUndefined() + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.value ?? ""))).toEqual({ + action: "acknowledge", + incidentId: INCIDENT, + }) + const contexts = Arr.filter(blocks, (block) => block.type === "context") + expect( + Arr.some( + contexts, + (block) => + block.type === "context" && + Arr.some(block.elements, (element) => Str.includes("Acknowledged")(element.text)), + ), + ).toBe(false) + }) + + it("shows who acknowledged and offers Unacknowledge", () => { + const { blocks, button } = ackButton( + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: "sam_*ops*" } }), + ) + expect(button?.text.text).toBe("Unacknowledge") + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.value ?? ""))?.action).toBe( + "unacknowledge", + ) + const texts = Arr.flatMap(blocks, (block) => + block.type === "context" ? Arr.map(block.elements, (element) => element.text) : [], + ) + expect( + Arr.some(texts, (text) => Str.includes("Acknowledged by")(text) && Str.includes("sam")(text)), + ).toBe(true) + }) + + it("renders no ack button on a card without the control", () => { + expect(ackButton(alert()).button).toBeUndefined() + }) + }) }) diff --git a/packages/chat-platform/src/connectors/slack/render.ts b/packages/chat-platform/src/connectors/slack/render.ts index c36e034251..e24d1ea5de 100644 --- a/packages/chat-platform/src/connectors/slack/render.ts +++ b/packages/chat-platform/src/connectors/slack/render.ts @@ -12,7 +12,7 @@ * `section` carries at most 10 `fields` of at most 2000 each, and a link button's `url` is at most * 3000. */ -import { chatActionControlId, type ChatActionToken } from "../../action-token" +import { chatActionControlId, chatAlertAckControlId, type ChatActionToken } from "../../action-token" import type { ChatAlertBlock, ChatBlock, ChatToolActivity } from "../../render/blocks" import { escapeMrkdwn, toMrkdwn } from "./mrkdwn" import { Array as Arr, String as Str } from "effect" @@ -142,6 +142,7 @@ export const MAX_FIELD_CHARS = 2000 export const MAX_LINK_BUTTONS = 5 export const APPROVE_ACTION = "maple_approve" +export const ALERT_ACK_ACTION = "maple_alert_ack" export const DENY_ACTION = "maple_deny" /** A message with nothing in it: Slack rejects a post carrying neither text nor blocks. */ @@ -279,23 +280,32 @@ const alertAttachment = (block: ChatAlertBlock): SlackAttachment => { alt_text: clamp(block.imageAlt === "" ? "Chart" : block.imageAlt, MAX_ALT_TEXT_CHARS), }) } + const ack = block.acknowledge + if (ack !== undefined && ack.acknowledgedBy !== null) { + blocks.push(context(`:white_check_mark: Acknowledged by *${escapeMrkdwn(ack.acknowledgedBy)}*`)) + } // A link too long for a button is left out rather than sent: Slack rejects the whole message. const links = Arr.filter(block.links, (link) => link.url.length <= MAX_IMAGE_URL_CHARS).slice( 0, MAX_LINK_BUTTONS, ) - if (links.length > 0) { - blocks.push({ - type: "actions", - elements: Arr.map(links, (link, index) => ({ - type: "button", - text: { type: "plain_text", text: clamp(link.label, MAX_BUTTON_TEXT_CHARS), emoji: true }, - action_id: `maple_link_${index}`, - url: link.url, - ...(link.primary ? { style: "primary" as const } : undefined), - })), + const buttons: Array = Arr.map(links, (link, index) => ({ + type: "button", + text: { type: "plain_text", text: clamp(link.label, MAX_BUTTON_TEXT_CHARS), emoji: true }, + action_id: `maple_link_${index}`, + url: link.url, + ...(link.primary ? { style: "primary" as const } : undefined), + })) + if (ack !== undefined) { + const acknowledged = ack.acknowledgedBy !== null + buttons.push({ + type: "button", + text: { type: "plain_text", text: acknowledged ? "Unacknowledge" : "Acknowledge" }, + action_id: ALERT_ACK_ACTION, + value: chatAlertAckControlId(acknowledged ? "unacknowledge" : "acknowledge", ack.incidentId), }) } + if (buttons.length > 0) blocks.push({ type: "actions", elements: buttons }) const footer = [ ...Arr.map(block.footer, toMrkdwn), ...(block.sentAtMs === null ? [] : [slackDate(block.sentAtMs)]), diff --git a/packages/chat-platform/src/render/blocks.ts b/packages/chat-platform/src/render/blocks.ts index 50209b0133..50869f73f1 100644 --- a/packages/chat-platform/src/render/blocks.ts +++ b/packages/chat-platform/src/render/blocks.ts @@ -124,6 +124,13 @@ export interface ChatAlertLink { readonly primary: boolean } +/** The incident an alert card can acknowledge, and whether somebody already has. */ +export interface ChatAlertAcknowledge { + readonly incidentId: string + /** Who acknowledged it, as a name to show; null while it is unacknowledged. */ + readonly acknowledgedBy: string | null +} + /** * A notification Maple posts on its own — an alert firing or resolving — rather than a turn. * @@ -147,6 +154,8 @@ export interface ChatAlertBlock { readonly footer: ReadonlyArray /** When it was sent, epoch ms, for a connector that can show it in each reader's own timezone. */ readonly sentAtMs: number | null + /** An open incident's ack control; absent on cards with nothing to acknowledge (resolve, test). */ + readonly acknowledge?: ChatAlertAcknowledge } export type ChatBlock = diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index 3b9e5b251e..d34ef496d0 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -162,10 +162,32 @@ export const AlertIncidentEventType = Schema.Literals([ }) export type AlertIncidentEventType = Schema.Schema.Type -export const AlertEventType = Schema.Literals(["trigger", "resolve", "renotify", "test"]).annotate({ +/** The notifications the public API reports; `acknowledge` stays internal (see AlertEventType). */ +export const AlertNotificationEventType = Schema.Literals([ + "trigger", + "resolve", + "renotify", + "test", +]).annotate({ identifier: "@maple/AlertEventType", title: "Alert Event Type", }) +export type AlertNotificationEventType = Schema.Schema.Type + +/** + * Every delivery the queue sends. `acknowledge` (a PagerDuty sync) is kept out of the public API: + * shipped clients decode the event type as a closed enum and would fail on a new value. + */ +export const AlertEventType = Schema.Literals([ + ...AlertNotificationEventType.literals, + "acknowledge", +]).annotate({ + identifier: "@maple/AlertDeliveryEventType", + title: "Alert Delivery Event Type", +}) + +export const isAlertNotificationEventType = (type: AlertEventType): type is AlertNotificationEventType => + type !== "acknowledge" export type AlertEventType = Schema.Schema.Type export const AlertDeliveryStatus = Schema.Literals(["queued", "processing", "success", "failed"]).annotate({ diff --git a/packages/domain/src/http/v2/alert-deliveries.ts b/packages/domain/src/http/v2/alert-deliveries.ts index c6e0672890..9da4bf09ed 100644 --- a/packages/domain/src/http/v2/alert-deliveries.ts +++ b/packages/domain/src/http/v2/alert-deliveries.ts @@ -1,6 +1,11 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api" import { Schema } from "effect" -import { AlertDeliveryStatus, AlertDestinationType, AlertEventType, AlertPersistenceError } from "../alerts" +import { + AlertDeliveryStatus, + AlertDestinationType, + AlertNotificationEventType, + AlertPersistenceError, +} from "../alerts" import { AuthorizationV2 } from "./auth" import { ListOf, ListQuery, Timestamp } from "./envelopes" import { V2ParameterInvalid } from "./errors" @@ -22,7 +27,7 @@ export const V2AlertDelivery = Schema.Struct({ destination_name: Schema.String, destination_type: AlertDestinationType, delivery_key: Schema.String, - event_type: AlertEventType, + event_type: AlertNotificationEventType, attempt_number: Schema.Number.pipe(Schema.check(Schema.isInt(), Schema.isGreaterThan(0))), status: AlertDeliveryStatus, scheduled_at: Timestamp, diff --git a/packages/domain/src/http/v2/alert-incidents.ts b/packages/domain/src/http/v2/alert-incidents.ts index 6b73520577..7666b80273 100644 --- a/packages/domain/src/http/v2/alert-incidents.ts +++ b/packages/domain/src/http/v2/alert-incidents.ts @@ -3,7 +3,7 @@ import { Schema } from "effect" import { AlertIncidentNotFoundError, AlertIncidentResolvedError, AlertPersistenceError } from "../alerts" import { AlertComparator, - AlertEventType, + AlertNotificationEventType, AlertIncidentAckSource, AlertIncidentEventType, AlertIncidentHoldReason, @@ -120,7 +120,7 @@ export const V2AlertIncident = Schema.Struct({ description: "Stable identity of the (rule, group) incident stream, used to deduplicate notifications.", }), - last_delivered_event_type: Schema.NullOr(AlertEventType).annotate({ + last_delivered_event_type: Schema.NullOr(AlertNotificationEventType).annotate({ description: "The most recent notification event delivered for this incident (`trigger`, `resolve`, `renotify`, `test`), or `null`.", }), diff --git a/packages/web-data/src/alerts/form-utils.ts b/packages/web-data/src/alerts/form-utils.ts index 4f5063039c..fd6683a2c3 100644 --- a/packages/web-data/src/alerts/form-utils.ts +++ b/packages/web-data/src/alerts/form-utils.ts @@ -1146,6 +1146,7 @@ export const eventTypeMeta: Record export type DeliveryStatusVariant = Exclude | "outline"