From 9c52485643d69a5a960a319fcbb18a1e21db96ca Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:24:04 +0100 Subject: [PATCH 1/4] feat(alerts): acknowledge incidents from Slack/Discord cards and in PagerDuty Alert cards posted to a chat channel for an open incident now carry an Acknowledge button (Unacknowledge once acked, with "Acknowledged by "). A click reaches the chat-bot relay as an alert_ack:/alert_unack: control that names only the incident (a UUID, strictly decoded); the org always comes from the workspace the click landed in. The relay runs AlertIncidentAckService for that org (linked Maple user where the connector has identity, else the platform display name), then redraws the clicked card from the delivery that posted it, so the chart, values and template stay. A resolved or unknown incident gets a short private note instead. Acknowledging also queues a PagerDuty acknowledge in the same transaction: a new "acknowledge" delivery event type, queued only to the enabled pagerduty destinations of the rule or that already paged the incident, and drained by the alerting dispatcher like any delivery. It acks the current tier's dedup key, and for a critical incident of a tiered rule the warning key too. The dispatcher skips an ack the incident no longer holds (resolved, or an escalation cleared it) and never restarts the reminder clock for one. Unacknowledge sends nothing: PagerDuty has no un-acknowledge. --- apps/chat-bot/src/relay/run.ts | 72 ++++++- apps/chat-bot/src/relay/settle.test.ts | 1 + apps/chat-bot/src/relay/turn.test.ts | 167 ++++++++++++++- apps/chat-bot/src/relay/turn.ts | 105 ++++++++++ .../services/alerts/AlertChatCardService.ts | 191 ++++++++++++++++++ .../alerts/AlertIncidentAck.chat.test.ts | 190 +++++++++++++++++ .../alerts/AlertIncidentAckService.ts | 124 +++++++++++- .../alerts/AlertsService.delivery.test.ts | 118 +++++++++++ .../src/services/alerts/AlertsService.ts | 68 +++---- .../src/services/alerts/delivery/context.ts | 2 + .../src/services/alerts/delivery/dispatch.ts | 8 + .../alerts/delivery/stored-payload.ts | 52 +++++ .../alerts/delivery/transports/chat.test.ts | 22 ++ .../alerts/delivery/transports/chat.ts | 37 +++- .../alerts/delivery/transports/linear.ts | 6 + .../alerts/delivery/transports/pagerduty.ts | 39 +++- .../alerts/delivery/transports/render.test.ts | 40 ++++ .../src/services/push/MobilePushService.ts | 3 + .../chat-platform/src/action-token.test.ts | 41 +++- packages/chat-platform/src/action-token.ts | 35 ++++ .../src/connectors/discord/render.test.ts | 42 +++- .../src/connectors/discord/render.ts | 34 +++- .../src/connectors/slack/render.test.ts | 63 +++++- .../src/connectors/slack/render.ts | 32 ++- packages/chat-platform/src/render/blocks.ts | 9 + packages/domain/src/http/alerts.ts | 8 +- packages/web-data/src/alerts/form-utils.ts | 1 + .../src/alerts/incident-timeline.test.ts | 11 + .../web-data/src/alerts/incident-timeline.ts | 10 +- 29 files changed, 1454 insertions(+), 77 deletions(-) create mode 100644 packages/backend/src/services/alerts/AlertChatCardService.ts create mode 100644 packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts create mode 100644 packages/backend/src/services/alerts/delivery/stored-payload.ts diff --git a/apps/chat-bot/src/relay/run.ts b/apps/chat-bot/src/relay/run.ts index 8c07741cc4..f7f2dddb98 100644 --- a/apps/chat-bot/src/relay/run.ts +++ b/apps/chat-bot/src/relay/run.ts @@ -10,7 +10,13 @@ import type { ChatConnector, ConnectorCredentials, InboundEvent } from "@maple/chat-platform" import { connectors } from "@maple/chat-platform/connectors" import type { ChatConnectorId, OrgId } from "@maple/domain/primitives" -import type { IntegrationsPersistenceError } from "@maple/domain/http" +import { + AlertIncidentAckSource, + AlertIncidentId, + type IntegrationsPersistenceError, +} from "@maple/domain/http" +import { AlertIncidentAckService } from "@maple/backend/services/alerts/AlertIncidentAckService" +import { AlertChatCardService } from "@maple/backend/services/alerts/AlertChatCardService" import { runtimeTelemetry } from "@maple/infra/worker-telemetry" import { ChatSessions, type ChatSessionsApi } from "@maple/backend/platform/bindings" import { type ChatSessionNamespace, chatSessionsLayerIfBound } from "@maple/backend/platform/chat-sessions" @@ -35,6 +41,7 @@ import { relayWithWorkspace, withWorkspace, WorkspaceLookupFailed, + type RelayAlertAckOutcome, type RelayPorts, type ResolvedWorkspace, } from "./turn.ts" @@ -201,6 +208,68 @@ const lookupWorkspace = ( ).pipe(lookupFailed(connectorId, "The chat workspace could not be read")), ) +const decodeAckSource = Schema.decodeUnknownOption(AlertIncidentAckSource) +const decodeIncidentId = Schema.decodeUnknownOption(AlertIncidentId) + +/** + * An alert card's ack click, in one connection: the ack (which also queues the PagerDuty + * acknowledgement), then the card redrawn with its new state. Every failure is an outcome. + */ +const alertAck = + (host: Pick): RelayPorts["alertAck"] => + (request) => { + const source = decodeAckSource(request.connector) + const incidentId = decodeIncidentId(request.incidentId) + if (Option.isNone(source)) return Effect.succeed({ kind: "unavailable" }) + if (Option.isNone(incidentId)) return Effect.succeed({ kind: "not_found" }) + return withDatabase( + host.env, + Effect.gen(function* () { + const acks = yield* AlertIncidentAckService + const cards = yield* AlertChatCardService + const actor = { userId: request.userId, name: request.displayName, source: source.value } + const change = + request.action === "acknowledge" + ? acks.acknowledge(request.orgId, incidentId.value, actor) + : acks.unacknowledge(request.orgId, incidentId.value, actor) + const applied = yield* change.pipe( + Effect.as<"applied" | "resolved" | "not_found">("applied"), + Effect.catchTags({ + "@maple/http/errors/AlertIncidentNotFoundError": () => + Effect.succeed("not_found" as const), + "@maple/http/errors/AlertIncidentResolvedError": () => + Effect.succeed("resolved" as const), + }), + ) + if (applied === "not_found") return { kind: "not_found" } satisfies RelayAlertAckOutcome + // The ack stands even when the card cannot be redrawn; the clicker is told instead. + const card = yield* cards + .cardForMessage(request.orgId, incidentId.value, request.messageId) + .pipe( + Effect.tapError((error) => + Effect.logWarning("An alert card could not be rebuilt").pipe( + Effect.annotateLogs({ "error.type": error._tag }), + ), + ), + Effect.orElseSucceed(() => Option.none()), + ) + return { kind: applied, card } satisfies RelayAlertAckOutcome + }).pipe( + // Built per event: the services hold the event's own connection. + Effect.provide([AlertIncidentAckService.layer, AlertChatCardService.layer]), + ), + ).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.logError("An alert acknowledgement from chat failed").pipe( + Effect.annotateLogs({ "error.type": summarizeCause(cause) }), + Effect.as({ kind: "unavailable" }), + ), + ), + ) + } + /** What a relayed event reads from the graph rather than the object: settings and the chat-session port. */ interface RelayEnvironment { readonly settings: RelaySettings @@ -258,6 +327,7 @@ const ports = ( ownsConversation: host.ownsConversation, rememberConversation: host.rememberConversation, recordTurn: host.recordTurn, + alertAck: alertAck(host), }) /** diff --git a/apps/chat-bot/src/relay/settle.test.ts b/apps/chat-bot/src/relay/settle.test.ts index bedf92a90a..ddb24fb214 100644 --- a/apps/chat-bot/src/relay/settle.test.ts +++ b/apps/chat-bot/src/relay/settle.test.ts @@ -179,6 +179,7 @@ const ports = ( announceUnlinked: Effect.succeed(false), ownsConversation: () => Effect.succeed(false), rememberConversation: () => Effect.void, + alertAck: () => Effect.succeed({ kind: "unavailable" }), // Cloned, as Durable Object storage does: nothing the fiber holds survives it. recordTurn: (checkpoint) => Effect.suspend(() => { diff --git a/apps/chat-bot/src/relay/turn.test.ts b/apps/chat-bot/src/relay/turn.test.ts index 65811e75a6..9f501383ce 100644 --- a/apps/chat-bot/src/relay/turn.test.ts +++ b/apps/chat-bot/src/relay/turn.test.ts @@ -22,6 +22,7 @@ import { type ChatSessionStub, chatSessionClientFromStub } from "@maple/domain/c import { ChatConnectorId, OrgId, UserId } from "@maple/domain/primitives" import { chatActionControlId, + chatAlertAckControlId, chatConnectorId, ChatOutboundError, ConnectorCredentials, @@ -52,7 +53,14 @@ import { } from "effect" import { TestClock } from "effect/testing" import { ConversationNotRecorded, FOLLOW_UP_WINDOW_MS } from "./conversation.ts" -import { relayInboundEvent, relayWithWorkspace, WorkspaceLookupFailed, type RelayPorts } from "./turn.ts" +import { + relayInboundEvent, + relayWithWorkspace, + WorkspaceLookupFailed, + type RelayAlertAckOutcome, + type RelayAlertAckRequest, + type RelayPorts, +} from "./turn.ts" const TESTCHAT = chatConnectorId("testchat") const ORG = Schema.decodeSync(OrgId)("org_1") @@ -271,6 +279,8 @@ interface Host { readonly announced: { count: number } /** How many workspace lookups the events caused — one database connection each. */ readonly lookups: { count: number } + /** Every alert ack the relay asked the host for. */ + readonly alertAcks: Array } const host = ( @@ -288,8 +298,11 @@ const host = ( readonly opened?: ReadonlyArray /** The relay object could not record the conversation the bot just opened. */ readonly rememberFails?: boolean + /** What the alert ack port answers. */ + readonly alertAck?: (request: RelayAlertAckRequest) => RelayAlertAckOutcome }, ): Host => { + const alertAcks: Array = [] const forgotten: Array = [] const charts: Array<{ orgId: OrgId; ref: ChatChartRef }> = [] const opened = new Set(options?.opened ?? []) @@ -301,6 +314,7 @@ const host = ( opened, announced, lookups, + alertAcks, ports: { outbound, supportsIdentity: options?.supportsIdentity ?? false, @@ -355,6 +369,11 @@ const host = ( ) : Effect.sync(() => void opened.add(conversation.conversationKey)), recordTurn: () => Effect.void, + alertAck: (request) => + Effect.sync(() => { + alertAcks.push(request) + return options?.alertAck?.(request) ?? { kind: "unavailable" } + }), }, } } @@ -1278,6 +1297,152 @@ describe("settling an approval somebody clicked", () => { ) }) +describe("acknowledging an alert from its card", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + const ackClick = (action: "acknowledge" | "unacknowledge") => + click({ actionToken: chatAlertAckControlId(action, INCIDENT), messageId: "alert-msg" }) + const redrawn: ReadonlyArray = [ + { + kind: "alert", + color: "#e01e5a", + title: "Checkout errors", + summary: "", + fields: [], + imageUrl: null, + imageAlt: "", + links: [], + footer: [], + sentAtMs: null, + acknowledge: { incidentId: INCIDENT, acknowledgedBy: "Ada" }, + }, + ] + + it.effect("acks in the workspace's org and redraws the clicked card", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "applied", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(deployment.alertAcks).toEqual([ + { + orgId: ORG, + connector: TESTCHAT, + incidentId: INCIDENT, + action: "acknowledge", + messageId: "alert-msg", + userId: null, + displayName: "Ada", + }, + ]) + expect(platform.calls).toHaveLength(1) + expect(platform.calls[0]?.verb).toBe("edit") + expect(platform.calls[0]?.ref.messageId).toBe("alert-msg") + expect(platform.calls[0]?.blocks).toEqual(redrawn) + }), + ) + + it.effect("records the linked Maple user when the connector can name the clicker", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + supportsIdentity: true, + linkedUserId: ADA, + alertAck: () => ({ kind: "applied", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("unacknowledge"), deployment.ports) + + expect(deployment.alertAcks[0]).toMatchObject({ action: "unacknowledge", userId: ADA }) + }), + ) + + it.effect("acks an unlinked clicker under their platform name rather than refusing", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + supportsIdentity: true, + alertAck: () => ({ kind: "applied", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(deployment.alertAcks[0]).toMatchObject({ userId: null, displayName: "Ada" }) + expect(platform.calls[0]?.verb).toBe("edit") + }), + ) + + it.effect("redraws a resolved incident's card and says so privately", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "resolved", card: Option.some(redrawn) }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["edit", "whisper"]) + expect(notices(blocksOf(platform.calls))).toEqual([ + "That incident is already resolved, so there is nothing to acknowledge.", + ]) + }), + ) + + it.effect("tells the clicker when the incident is gone, and edits nothing", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "not_found" }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["whisper"]) + expect(notices(blocksOf(platform.calls))).toEqual(["That incident no longer exists in Maple."]) + }), + ) + + it.effect("confirms privately when there is no card to redraw", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { + alertAck: () => ({ kind: "applied", card: Option.none() }), + }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["whisper"]) + expect(notices(blocksOf(platform.calls))).toEqual(["Acknowledged."]) + }), + ) + + it.effect("asks nothing of the host for a workspace nobody linked", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined, { linked: false }) + + yield* relayInboundEvent(ackClick("acknowledge"), deployment.ports) + + expect(deployment.alertAcks).toEqual([]) + expect(Arr.map(platform.calls, (call) => call.verb)).toEqual(["whisper"]) + }), + ) + + it.effect("treats a malformed ack control as not Maple's", () => + Effect.gen(function* () { + const platform = chat() + const deployment = host(platform.outbound, undefined) + + yield* relayInboundEvent(click({ actionToken: "alert_ack:not-a-uuid" }), deployment.ports) + + expect(deployment.alertAcks).toEqual([]) + expect(platform.calls).toEqual([]) + }), + ) +}) + describe("relaying everything else a connector reports", () => { it.effect("unlinks a workspace the bot was removed from", () => Effect.gen(function* () { diff --git a/apps/chat-bot/src/relay/turn.ts b/apps/chat-bot/src/relay/turn.ts index de2e31651d..46667a4709 100644 --- a/apps/chat-bot/src/relay/turn.ts +++ b/apps/chat-bot/src/relay/turn.ts @@ -14,9 +14,11 @@ import { ConnectorCredentials, decodeChatActionControlId, + decodeChatAlertAckControlId, driveChatTurn, WORKSPACE_CREDENTIALS, type ChatActionRequest, + type ChatAlertAckAction, type ChatBlock, type ChatChartRef, type ChatConversation, @@ -76,6 +78,27 @@ export class WorkspaceLookupFailed extends Schema.TaggedError> } + | { readonly kind: "resolved"; readonly card: Option.Option> } + | { readonly kind: "not_found" } + | { readonly kind: "unavailable" } + export interface RelayPorts { /** The connector's outbound half — how a turn is shown on the platform it was asked on. */ readonly outbound: ChatOutbound @@ -127,6 +150,8 @@ export interface RelayPorts { readonly announceUnlinked: Effect.Effect /** Checkpoint the turn's posted messages, for settling it after an eviction (`./settle.ts`). */ readonly recordTurn: (checkpoint: RelayTurnCheckpoint) => Effect.Effect + /** Acknowledge or unacknowledge an alert incident from its card. Never fails; see the outcome. */ + readonly alertAck: (request: RelayAlertAckRequest) => Effect.Effect } /** @@ -145,6 +170,10 @@ const BUSY_NOTICE = "Still working on the previous message here — ask again on const UNAVAILABLE_NOTICE = "Maple's agent can't be reached from here right now." +const ALERT_RESOLVED_NOTICE = "That incident is already resolved, so there is nothing to acknowledge." +const ALERT_GONE_NOTICE = "That incident no longer exists in Maple." +const ALERT_UNAVAILABLE_NOTICE = "Maple couldn't record that just now. Try again in a moment." + /** The control outlived what it pointed at: a wiped conversation, or a build that changed the log. */ const PROPOSAL_GONE_NOTICE = "That change isn't waiting for a decision any more." @@ -467,6 +496,10 @@ const settleAction = Effect.fn("ChatBotRelay.settleApproval")(function* ( ports: RelayPorts, ) { yield* Effect.annotateCurrentSpan({ "maple.chat.connector": action.connector }) + const alertAck = decodeChatAlertAckControlId(action.actionToken) + if (Option.isSome(alertAck)) { + return yield* settleAlertAck(action, alertAck.value.action, alertAck.value.incidentId, ports) + } const control = decodeChatActionControlId(action.actionToken) // A platform hands back whatever was on the control that was clicked, which includes controls // Maple never rendered. Not ours is not an error. @@ -571,6 +604,78 @@ const settleAction = Effect.fn("ChatBotRelay.settleApproval")(function* ( } }) +/** + * A click on an alert card's ack button. Anyone in the channel may acknowledge: where the connector + * can link accounts the ack is recorded as that Maple user, otherwise under the platform name. + * The org is the workspace's; the incident id off the control is only looked up inside it. + */ +const settleAlertAck = Effect.fn("ChatBotRelay.settleAlertAck")(function* ( + action: InboundAction, + ackAction: ChatAlertAckAction, + incidentId: string, + ports: RelayPorts, +) { + yield* Effect.annotateCurrentSpan({ + "maple.chat.alert_ack.action": ackAction, + "maple.alert.incident_id": incidentId, + }) + const transport = yield* ports.outbound.transport + const clicker = decodeExternalUserId(action.actor.id) + const workspace = yield* Effect.exit( + ports.resolveWorkspace(action.connector, action.workspaceId, Option.getOrUndefined(clicker)), + ) + if (Exit.isFailure(workspace) || Option.isNone(workspace.value)) { + yield* Effect.annotateCurrentSpan({ "maple.chat.alert_ack": "unavailable" }) + return yield* tellClicker(transport, action, notice(ALERT_UNAVAILABLE_NOTICE)) + } + const { orgId, linkedUserId } = workspace.value.value + yield* Effect.annotateCurrentSpan({ orgId }) + + const outcome = yield* ports.alertAck({ + orgId, + connector: action.connector, + incidentId, + action: ackAction, + messageId: action.messageId, + userId: linkedUserId ?? null, + displayName: action.actor.displayName, + }) + yield* Effect.annotateCurrentSpan({ "maple.chat.alert_ack": outcome.kind }) + const redraw = (card: Option.Option>) => + Option.match(card, { + onNone: () => Effect.void, + onSome: (blocks) => + transport.edit({ target: replyTarget(action), messageId: action.messageId }, blocks).pipe( + Effect.tapError((error) => + Effect.logWarning("An acknowledged alert card could not be redrawn").pipe( + Effect.annotateLogs({ "error.type": error.operation }), + ), + ), + Effect.ignore, + ), + }) + switch (outcome.kind) { + case "applied": + // No card to redraw (a digest, an old delivery): say it privately instead. + return yield* Option.isSome(outcome.card) + ? redraw(outcome.card) + : tellClicker( + transport, + action, + notice(ackAction === "acknowledge" ? "Acknowledged." : "Unacknowledged."), + ) + case "resolved": + yield* redraw(outcome.card) + return yield* tellClicker(transport, action, notice(ALERT_RESOLVED_NOTICE)) + case "not_found": + return yield* tellClicker(transport, action, notice(ALERT_GONE_NOTICE)) + case "unavailable": + return yield* tellClicker(transport, action, notice(ALERT_UNAVAILABLE_NOTICE)) + default: + return outcome satisfies never + } +}) + /** * Put the decision on the message that carried the controls. * diff --git a/packages/backend/src/services/alerts/AlertChatCardService.ts b/packages/backend/src/services/alerts/AlertChatCardService.ts new file mode 100644 index 0000000000..878e514646 --- /dev/null +++ b/packages/backend/src/services/alerts/AlertChatCardService.ts @@ -0,0 +1,191 @@ +/** + * Redraws an alert card Maple already posted to a chat channel, with the incident's current + * acknowledgement. The card is rebuilt from the delivery that posted it (its snapshotted payload), + * so an edit keeps the chart, values and template the original showed. + */ +import type { ChatBlock } from "@maple/chat-platform" +import { + AlertComparator, + AlertSeverity, + AlertSignalType, + type AlertEventType, + type AlertIncidentId, + type AlertPersistenceError, + type OrgId, +} from "@maple/domain/http" +import * as PG from "@maple-dev/effect-orm/postgres" +import { AlertDeliveryEvents, AlertIncidents } from "@maple/db/tables" +import { Context, Effect, Layer, Option, Schema, Array as Arr } from "effect" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { makePersistenceError } from "./alert-persistence" +import { renderTitleBody } from "./AlertDeliveryDispatch" +import { resolveSignalDisplay } from "./alert-signal-display" +import { decodeStoredAlertRuleMetadata, selectStoredAlertRules } from "./AlertRuleModel" +import { StoredDeliveryPayloadSchema } from "./delivery/stored-payload" +import { buildChatAlertBlocks, incidentAcknowledgedBy } from "./delivery/transports/chat" + +export interface AlertChatCardServiceApi { + /** + * The card a chat message carries, redrawn now. `None` when the message is not a firing card + * Maple posted for this incident (a digest, another org's message, a delivery too old to redraw). + */ + readonly cardForMessage: ( + orgId: OrgId, + incidentId: AlertIncidentId, + messageId: string, + ) => Effect.Effect>, AlertPersistenceError> +} + +const decodePayload = Schema.decodeUnknownOption(StoredDeliveryPayloadSchema) +const decodeSeverity = Schema.decodeUnknownOption(AlertSeverity) +const decodeSignalType = Schema.decodeUnknownOption(AlertSignalType) +const decodeComparator = Schema.decodeUnknownOption(AlertComparator) + +export class AlertChatCardService extends Context.Service()( + "@maple/backend/services/alerts/AlertChatCardService", + { + make: Effect.gen(function* () { + const database = yield* Database + const dbExecute = makeDbExecute(database, "AlertChatCardService", makePersistenceError) + + const cardForMessage = Effect.fn("AlertChatCardService.cardForMessage")(function* ( + orgId: OrgId, + incidentId: AlertIncidentId, + messageId: string, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.alert.incident_id": incidentId }) + const [deliveries, incidents] = yield* Effect.all( + [ + dbExecute((db) => + db.run( + PG.from(AlertDeliveryEvents) + .select() + .where(($) => [ + $.orgId.eq(orgId), + $.incidentId.eq(incidentId), + $.providerReference.eq(messageId), + $.status.eq("success"), + $.eventType.in_("trigger", "renotify"), + ]) + .orderBy(["scheduledAt", "desc"]) + .limit(1), + ), + ), + dbExecute((db) => + db.run( + PG.from(AlertIncidents) + .select() + .where(($) => [$.orgId.eq(orgId), $.id.eq(incidentId)]) + .limit(1), + ), + ), + ], + { concurrency: "unbounded" }, + ) + const delivery = Arr.head(deliveries) + const incident = Arr.head(incidents) + if (Option.isNone(delivery) || Option.isNone(incident)) { + yield* Effect.annotateCurrentSpan("maple.alert.card_found", false) + return Option.none>() + } + const row = delivery.value + const current = incident.value + const payload = decodePayload(row.payloadJson) + const linkUrl = Option.flatMap(payload, (value) => Option.fromUndefinedOr(value.linkUrl)) + const signalType = decodeSignalType(current.signalType) + const comparator = decodeComparator(current.comparator) + const eventType: AlertEventType = row.eventType === "renotify" ? "renotify" : "trigger" + if ( + Option.isNone(payload) || + Option.isNone(linkUrl) || + Option.isNone(signalType) || + Option.isNone(comparator) + ) { + yield* Effect.annotateCurrentSpan("maple.alert.card_found", false) + return Option.none>() + } + const stored = payload.value + const ruleRows = yield* dbExecute((db) => + db.run( + selectStoredAlertRules() + .where(($) => [$.orgId.eq(orgId), $.id.eq(row.ruleId)]) + .limit(1), + ), + ) + const ruleRow = Arr.head(ruleRows) + // A rule whose stored config no longer decodes still redraws, from the snapshot alone. + const rule = Option.isNone(ruleRow) + ? Option.none() + : yield* Effect.option(decodeStoredAlertRuleMetadata(ruleRow.value)) + const context = { + ruleId: row.ruleId, + ruleName: Option.match(ruleRow, { + onNone: () => stored.rule?.name ?? current.ruleName, + onSome: (value) => value.name, + }), + eventType, + severity: Option.getOrElse( + Option.orElse(decodeSeverity(stored.rule?.severity), () => + decodeSeverity(current.severity), + ), + (): AlertSeverity => "warning", + ), + signalType: signalType.value, + signalDisplay: resolveSignalDisplay({ + signalType: signalType.value, + queryBuilderDraft: Option.match(rule, { + onNone: () => null, + onSome: (value) => value.queryBuilderDraft, + }), + rawQueryReducer: Option.match(ruleRow, { + onNone: () => null, + onSome: (value) => value.reducer, + }), + }), + comparator: comparator.value, + threshold: current.threshold, + thresholdUpper: current.thresholdUpper, + value: stored.observed?.value ?? null, + sampleCount: stored.observed?.sampleCount ?? null, + detail: stored.observed?.detail ?? null, + groupKey: current.groupKey, + windowMinutes: Option.match(ruleRow, { + onNone: () => stored.rule?.windowMinutes ?? 5, + onSome: (value) => value.windowMinutes, + }), + incidentId: current.id, + incidentStatus: current.status, + incidentAcknowledgedBy: incidentAcknowledgedBy(current), + sparkline: stored.chart?.sparkline ?? null, + chartUrl: stored.chart?.url ?? null, + dedupeKey: current.dedupeKey, + template: + stored.template ?? + Option.match(rule, { + onNone: () => null, + onSome: (value) => value.notificationTemplate, + }), + sentAtMs: row.attemptedAt ?? row.scheduledAt, + } + yield* Effect.annotateCurrentSpan("maple.alert.card_found", true) + return Option.some( + buildChatAlertBlocks({ + context, + templated: renderTitleBody( + context, + "chat", + linkUrl.value, + stored.chatUrl ?? linkUrl.value, + ), + linkUrl: linkUrl.value, + }), + ) + }) + + return { cardForMessage } satisfies AlertChatCardServiceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts b/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts new file mode 100644 index 0000000000..dcb2b80c83 --- /dev/null +++ b/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts @@ -0,0 +1,190 @@ +/** + * What an acknowledgement reaches beyond the incident row: the PagerDuty acknowledgements it + * queues, and the chat card it redraws. + */ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { AlertIncidentId, AlertRuleId, OrgId } from "@maple/domain/http" +import { Effect, Layer, Option, Schema, Array as Arr } from "effect" +import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" +import { AlertChatCardService } from "./AlertChatCardService" +import { AlertIncidentAckService, type AlertIncidentActor } from "./AlertIncidentAckService" + +const ORG = Schema.decodeSync(OrgId)("org_ack_chat") +const RULE = Schema.decodeSync(AlertRuleId)("00000000-0000-4000-8000-000000000030") +const INCIDENT = Schema.decodeSync(AlertIncidentId)("00000000-0000-4000-8000-000000000031") +const RESOLVED = Schema.decodeSync(AlertIncidentId)("00000000-0000-4000-8000-000000000032") + +const PD_RULE = "00000000-0000-4000-8000-0000000000d1" +const PD_DISABLED = "00000000-0000-4000-8000-0000000000d2" +const CHAT_DEST = "00000000-0000-4000-8000-0000000000d3" +const PD_PRIOR = "00000000-0000-4000-8000-0000000000d4" +const PD_UNRELATED = "00000000-0000-4000-8000-0000000000d5" + +const MESSAGE_TS = "1762400000.000100" + +const sam: AlertIncidentActor = { userId: null, name: "sam", source: "slack" } + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const makeLayer = (testDb: TestDb) => + Layer.mergeAll(AlertIncidentAckService.layer, AlertChatCardService.layer).pipe( + Layer.provide(testDb.layer), + ) + +const destination = (id: string, name: string, type: string, enabled: boolean) => + `('${id}', '${ORG}', '${name}', '${type}', ${enabled}, '{}', '', '', '', now(), now(), 'u', 'u')` + +const seed = (testDb: TestDb) => + Effect.promise(async () => { + await executeSql( + testDb, + `INSERT INTO alert_destinations + (id, org_id, name, type, enabled, config_json, secret_ciphertext, secret_iv, secret_tag, + created_at, updated_at, created_by, updated_by) + VALUES ${Arr.join( + [ + destination(PD_RULE, "On-call", "pagerduty", true), + destination(PD_DISABLED, "Old on-call", "pagerduty", false), + destination(CHAT_DEST, "#alerts", "chat", true), + destination(PD_PRIOR, "Former on-call", "pagerduty", true), + destination(PD_UNRELATED, "Other team", "pagerduty", true), + ], + ", ", + )}`, + ) + await executeSql( + testDb, + `INSERT INTO alert_rules + (id, org_id, name, severity, signal_type, comparator, threshold, window_minutes, + destination_ids_json, reducer, no_data_behavior, created_at, updated_at, created_by, updated_by) + VALUES ($1, $2, 'Checkout errors', 'warning', 'error_rate', 'gt', 5, 5, $3::jsonb, + 'avg', 'skip', now(), now(), 'u', 'u')`, + [RULE, ORG, JSON.stringify([PD_RULE, PD_DISABLED, CHAT_DEST])], + ) + await executeSql( + testDb, + `INSERT INTO alert_incidents + (id, org_id, rule_id, incident_key, rule_name, group_key, signal_type, severity, status, + comparator, threshold, first_triggered_at, last_triggered_at, resolved_at, + dedupe_key, created_at, updated_at) + VALUES + ($1, $3, $4, 'k-open', 'Checkout errors', 'checkout', 'error_rate', 'warning', 'open', + 'gt', 5, '2026-11-06T09:00:00Z', '2026-11-06T09:10:00Z', null, 'd-open', now(), now()), + ($2, $3, $4, 'k-res', 'Checkout errors', 'checkout', 'error_rate', 'warning', 'resolved', + 'gt', 5, '2026-11-06T08:00:00Z', '2026-11-06T08:10:00Z', '2026-11-06T08:20:00Z', 'd-res', now(), now())`, + [INCIDENT, RESOLVED, ORG, RULE], + ) + const payload = JSON.stringify({ + eventType: "trigger", + incidentId: INCIDENT, + dedupeKey: "d-open", + rule: { + name: "Checkout errors", + severity: "warning", + signalType: "error_rate", + windowMinutes: 5, + }, + observed: { value: 7.5, sampleCount: 1200 }, + template: null, + chart: null, + linkUrl: `https://app.maple.dev/alerts/incidents/${INCIDENT}`, + chatUrl: `https://app.maple.dev/alerts/rules/${RULE}`, + }) + // The card the channel got, and a page the former on-call destination got. + await executeSql( + testDb, + `INSERT INTO alert_delivery_events + (id, org_id, incident_id, rule_id, destination_id, delivery_key, event_type, attempt_number, + status, scheduled_at, attempted_at, provider_reference, payload_json, created_at, updated_at) + VALUES + ('00000000-0000-4000-8000-0000000000e1', $1, $2, $3, $4, 'k1', 'trigger', 1, 'success', + '2026-11-06T09:00:00Z', '2026-11-06T09:00:01Z', $5, $6::jsonb, now(), now()), + ('00000000-0000-4000-8000-0000000000e2', $1, $2, $3, $7, 'k2', 'trigger', 1, 'success', + '2026-11-06T09:00:00Z', '2026-11-06T09:00:01Z', 'd-open', $6::jsonb, now(), now())`, + [ORG, INCIDENT, RULE, CHAT_DEST, MESSAGE_TS, payload, PD_PRIOR], + ) + }) + +const queuedAcks = (testDb: TestDb) => + Effect.promise(() => + testDb.pglite.query<{ destination_id: string; status: string; payload_json: { dedupeKey: string } }>( + `SELECT destination_id, status, payload_json FROM alert_delivery_events + WHERE event_type = 'acknowledge' ORDER BY destination_id`, + ), + ).pipe(Effect.map((result) => result.rows)) + +describe("acknowledging an incident queues PagerDuty acknowledgements", () => { + it.effect("to the enabled PagerDuty destinations of the rule and of the incident's past pages", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const acks = yield* AlertIncidentAckService + + yield* acks.acknowledge(ORG, INCIDENT, sam) + + const queued = yield* queuedAcks(testDb) + assert.deepStrictEqual( + Arr.map(queued, (row) => [row.destination_id, row.status, row.payload_json.dedupeKey]), + [ + [PD_RULE, "queued", "d-open"], + [PD_PRIOR, "queued", "d-open"], + ], + ) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("queues nothing for an unacknowledge or for an ack that changed nothing", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const acks = yield* AlertIncidentAckService + + yield* acks.acknowledge(ORG, INCIDENT, sam) + yield* acks.acknowledge(ORG, INCIDENT, sam) + yield* acks.unacknowledge(ORG, INCIDENT, sam) + + assert.lengthOf(yield* queuedAcks(testDb), 2) + }).pipe(Effect.provide(makeLayer(testDb))) + }) +}) + +describe("redrawing the chat card an ack was clicked on", () => { + it.effect("shows who acknowledged, and the Acknowledge control again once it is undone", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const acks = yield* AlertIncidentAckService + const cards = yield* AlertChatCardService + + yield* acks.acknowledge(ORG, INCIDENT, sam) + const acked = Option.getOrUndefined(yield* cards.cardForMessage(ORG, INCIDENT, MESSAGE_TS)) + const card = acked?.[0] + assert.strictEqual(card?.kind, "alert") + if (card?.kind !== "alert") return + assert.include(card.title, "Checkout errors") + assert.deepStrictEqual(card.acknowledge, { incidentId: INCIDENT, acknowledgedBy: "sam" }) + + yield* acks.unacknowledge(ORG, INCIDENT, sam) + const unacked = Option.getOrUndefined(yield* cards.cardForMessage(ORG, INCIDENT, MESSAGE_TS)) + const again = unacked?.[0] + assert.deepStrictEqual(again?.kind === "alert" ? again.acknowledge : undefined, { + incidentId: INCIDENT, + acknowledgedBy: null, + }) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("finds no card for a message Maple did not post for that incident or org", () => { + const testDb = createTestDb(createdDbs) + return Effect.gen(function* () { + yield* seed(testDb) + const cards = yield* AlertChatCardService + const otherOrg = Schema.decodeSync(OrgId)("org_someone_else") + + assert.isTrue(Option.isNone(yield* cards.cardForMessage(ORG, INCIDENT, "1762400000.999999"))) + assert.isTrue(Option.isNone(yield* cards.cardForMessage(ORG, RESOLVED, MESSAGE_TS))) + assert.isTrue(Option.isNone(yield* cards.cardForMessage(otherOrg, INCIDENT, MESSAGE_TS))) + }).pipe(Effect.provide(makeLayer(testDb))) + }) +}) diff --git a/packages/backend/src/services/alerts/AlertIncidentAckService.ts b/packages/backend/src/services/alerts/AlertIncidentAckService.ts index 5655e744ad..31692d4038 100644 --- a/packages/backend/src/services/alerts/AlertIncidentAckService.ts +++ b/packages/backend/src/services/alerts/AlertIncidentAckService.ts @@ -4,6 +4,8 @@ * carries the history, which the timeline shows and on-call escalation reads. */ import { + AlertDeliveryEventId, + AlertDestinationId, AlertIncidentAckSource, AlertIncidentEventDocument, AlertIncidentEventId, @@ -17,7 +19,16 @@ import { type UserId, } from "@maple/domain/http" import * as PG from "@maple-dev/effect-orm/postgres" -import { AlertIncidentEvents, AlertIncidents } from "@maple/db/tables" +import { + AlertDeliveryEvents, + AlertDestinations, + AlertIncidentEvents, + AlertIncidents, + AlertRules, + type AlertIncidentRow, +} from "@maple/db/tables" +import { makeAlertDeliveryKey } from "@maple/alerting-core" +import type { MapleDb } from "@maple/db/client" import { Context, Effect, Layer, Option, Schema, Array as Arr } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" @@ -62,6 +73,8 @@ export interface AlertIncidentAckServiceApi { } const decodeEventId = Schema.decodeUnknownSync(AlertIncidentEventId) +const decodeDeliveryId = Schema.decodeUnknownSync(AlertDeliveryEventId) +const decodeIdList = Schema.decodeUnknownOption(Schema.Array(AlertDestinationId)) const decodeEventType = Schema.decodeUnknownOption(AlertIncidentEventType) const decodeSource = Schema.decodeUnknownOption(AlertIncidentAckSource) const decodeIso = Schema.decodeUnknownSync(AlertIncidentEventDocument.fields.createdAt) @@ -96,6 +109,105 @@ export class AlertIncidentAckService extends Context.Service< ), ) + /** + * Queue a PagerDuty `acknowledge` for each enabled PagerDuty destination of the rule or that + * this incident already paged. Written in the ack's transaction; the alerting dispatcher sends it. + */ + const queuePagerDutyAcknowledge = ( + db: MapleDb, + incident: AlertIncidentRow, + now: number, + makeUuid: () => string, + ) => + Effect.gen(function* () { + const [ruleRows, priorRows] = yield* Effect.all([ + db.run( + PG.from(AlertRules) + .select(($) => ({ destinationIdsJson: PG.undecoded($.destinationIdsJson) })) + .where(($) => [$.orgId.eq(incident.orgId), $.id.eq(incident.ruleId)]) + .limit(1), + ), + db.run( + PG.from(AlertDeliveryEvents) + .select("destinationId") + .where(($) => [$.orgId.eq(incident.orgId), $.incidentId.eq(incident.id)]), + ), + ]) + // A corrupt stored list must not fail the ack; the incident's own deliveries still count. + const ruleIds = Arr.flatMap(ruleRows, (row) => + Option.getOrElse( + decodeIdList(row.destinationIdsJson), + (): ReadonlyArray => [], + ), + ) + const candidateIds = Arr.dedupe([ + ...ruleIds, + ...Arr.map(priorRows, (row) => row.destinationId), + ]) + if (candidateIds.length === 0) return 0 + const destinations = yield* db.run( + PG.from(AlertDestinations) + .select("id") + .where(($) => [ + $.orgId.eq(incident.orgId), + $.id.in_(...candidateIds), + $.type.eq("pagerduty"), + $.enabled.eq(true), + ]), + ) + yield* Effect.forEach( + destinations, + (destination) => + db.run( + PG.insertInto(AlertDeliveryEvents) + .values({ + id: decodeDeliveryId(makeUuid()), + orgId: incident.orgId, + incidentId: incident.id, + ruleId: incident.ruleId, + destinationId: destination.id, + deliveryKey: makeAlertDeliveryKey( + incident.id, + destination.id, + "acknowledge", + now, + ), + eventType: "acknowledge", + attemptNumber: 1, + status: "queued", + scheduledAt: now, + claimedAt: null, + claimExpiresAt: null, + claimedBy: null, + attemptedAt: null, + providerMessage: null, + providerReference: null, + responseCode: null, + errorMessage: null, + payloadJson: { + eventType: "acknowledge", + incidentId: incident.id, + incidentStatus: incident.status, + dedupeKey: incident.dedupeKey, + peakSeverity: incident.peakSeverity, + rule: { + id: incident.ruleId, + name: incident.ruleName, + severity: incident.severity, + groupKey: incident.groupKey, + signalType: incident.signalType, + }, + }, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing(), + ), + { discard: true }, + ) + return destinations.length + }) + /** * One guarded UPDATE per change, so two people clicking at once write one event: the loser * matches no row and reads back the state the winner left. @@ -146,6 +258,16 @@ export class AlertIncidentAckService extends Context.Service< ]) .returning(), ) + const changedRow = Arr.head(rows) + if (acknowledged && Option.isSome(changedRow)) { + const queued = yield* queuePagerDutyAcknowledge( + db, + changedRow.value, + now, + () => runtime.makeUuid(), + ) + yield* Effect.annotateCurrentSpan("maple.alert.pagerduty_acks_queued", queued) + } if (rows.length > 0) { yield* db.run( PG.insertInto(AlertIncidentEvents).values({ diff --git a/packages/backend/src/services/alerts/AlertsService.delivery.test.ts b/packages/backend/src/services/alerts/AlertsService.delivery.test.ts index dd70c20e1d..a8a9203736 100644 --- a/packages/backend/src/services/alerts/AlertsService.delivery.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.delivery.test.ts @@ -4,6 +4,7 @@ import { AlertDeliveryError, AlertDeliveryRejectedError, ChatAlertDestinationConfig, + PagerDutyAlertDestinationConfig, WebhookAlertDestinationConfig, } from "@maple/domain/http" import { ChatAlertPoster } from "./ChatAlertPoster" @@ -1129,3 +1130,120 @@ describe("AlertsService held incidents", () => { }).pipe(Effect.provide(makeLayer(testDb, makeLivenessStub(state), { fetch: okFetch }))) }) }) + +describe("AlertsService PagerDuty acknowledgements", () => { + const fixedTime = 1_710_000_400_000 + + const run = (acknowledged: boolean) => { + const testDb = createTestDb(trackedDbs) + const bodies: Array = [] + const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => { + bodies.push(requestBodyText(init?.body)) + return new Response("{}", { status: 202 }) + }) as typeof fetch + return Effect.gen(function* () { + yield* TestClock.setTime(fixedTime) + const alerts = yield* AlertsService + const orgId = asOrgId("org_pd_ack") + const userId = asUserId("user_pd_ack") + const destination = yield* alerts.createDestination( + orgId, + userId, + adminRoles, + new PagerDutyAlertDestinationConfig({ + type: "pagerduty", + name: "On-call", + enabled: true, + integrationKey: "abcdefghijklmnopqrstuvwxyz012345", + }), + ) + const rule = yield* createErrorRateRule(alerts, orgId, userId, destination.id) + // Break the rule's stored query so the tick queues nothing of its own. + yield* Effect.promise(() => + executeSql(testDb, "update alert_rules set query_spec_json = $1::jsonb where id = $2", [ + "{}", + rule.id, + ]), + ) + const incidentId = "00000000-0000-4000-8000-000000000401" + yield* Effect.promise(() => + executeSql( + testDb, + `INSERT INTO alert_incidents + (id, org_id, rule_id, incident_key, rule_name, group_key, signal_type, severity, status, + comparator, threshold, first_triggered_at, last_triggered_at, dedupe_key, + last_notified_at, acknowledged_at, acknowledged_by_name, acknowledged_via, + created_at, updated_at) + VALUES ($1, $2, $3, 'k', 'Checkout error rate', '__total__', 'error_rate', 'warning', 'open', + 'gt', 5, now(), now(), 'dedupe-ack', to_timestamp(1000), $4, 'sam', 'slack', now(), now())`, + [incidentId, orgId, rule.id, acknowledged ? new Date(fixedTime - 1000) : null], + ), + ) + bodies.length = 0 + yield* Effect.promise(() => + insertDeliveryEventRow(testDb, { + id: "00000000-0000-4000-8000-000000000402", + orgId, + incidentId, + ruleId: rule.id, + destinationId: destination.id, + deliveryKey: "ack-1", + eventType: "acknowledge", + attemptNumber: 1, + status: "queued", + scheduledAt: fixedTime - 500, + payloadJson: JSON.stringify({ + eventType: "acknowledge", + incidentId, + dedupeKey: "dedupe-ack", + rule: { id: rule.id, name: "Checkout error rate", severity: "warning" }, + }), + }), + ) + yield* alerts.runSchedulerTick() + const row = yield* Effect.promise(() => + queryFirstRow<{ status: string; message: string | null; notified: Date | null }>( + testDb, + `select d.status, d.provider_message as message, i.last_notified_at as notified + from alert_delivery_events d join alert_incidents i on i.id = d.incident_id + where d.delivery_key = 'ack-1'`, + [], + ), + ) + return { bodies, row } + }).pipe( + Effect.provide( + makeLayer(testDb, makeWarehouseStub({ tracesAggregateRows: emptyWarehouseRows }), { + fetch: fetchImpl, + }), + ), + ) + } + + it.effect("sends the queued acknowledge without restarting the reminder clock", () => + Effect.gen(function* () { + const { bodies, row } = yield* run(true) + assert.deepStrictEqual( + Arr.map(bodies, (body) => JSON.parse(body)), + [ + { + routing_key: "abcdefghijklmnopqrstuvwxyz012345", + event_action: "acknowledge", + dedup_key: "dedupe-ack", + }, + ], + ) + assert.strictEqual(row?.status, "success") + assert.strictEqual(row?.notified?.getTime(), 1_000_000) + }), + ) + + it.effect("skips an acknowledge the incident no longer holds, so a fresh page stays loud", () => + Effect.gen(function* () { + const { bodies, row } = yield* run(false) + assert.deepStrictEqual(bodies, []) + assert.strictEqual(row?.status, "success") + assert.strictEqual(row?.message, "Skipped: the incident is no longer acknowledged") + }), + ) +}) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 175e3506d4..c0038b3f67 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -58,7 +58,6 @@ import { type AlertSeverity, AlertSignalType as AlertSignalTypeSchema, AlertValidationError, - AlertNotificationTemplate, type AlertEventType as AlertEventTypeValue, type AlertRuleUpsertRequest, type AlertGroupBy, @@ -185,6 +184,8 @@ import { UPTIME_RULE_MANAGED_MESSAGE, } from "./AlertRuleModel" import { mapSignalUnit, resolveSignalDisplay } from "./alert-signal-display" +import { StoredDeliveryPayloadSchema, type StoredDeliveryPayload } from "./delivery/stored-payload" +import { incidentAcknowledgedBy } from "./delivery/transports/chat" import { summarizeCause } from "@maple/backend/platform/describe-cause" const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)) @@ -391,48 +392,6 @@ const DELIVERY_LEASE_TTL_MS = 30_000 /* Schemas for stored JSON formats */ /* -------------------------------------------------------------------------- */ -const StoredDeliveryPayloadSchema = Schema.StructWithRest( - Schema.Struct({ - event: Schema.optionalKey(Schema.Unknown), - eventType: Schema.optionalKey(Schema.String), - incidentId: Schema.optionalKey(Schema.NullOr(Schema.String)), - incidentStatus: Schema.optionalKey(Schema.String), - dedupeKey: Schema.optionalKey(Schema.String), - rule: Schema.optionalKey( - Schema.Struct({ - id: Schema.optionalKey(Schema.String), - name: Schema.optionalKey(Schema.String), - signalType: Schema.optionalKey(Schema.String), - severity: Schema.optionalKey(Schema.String), - groupKey: Schema.optionalKey(Schema.NullOr(Schema.String)), - comparator: Schema.optionalKey(Schema.String), - threshold: Schema.optionalKey(Schema.Number), - thresholdUpper: Schema.optionalKey(Schema.NullOr(Schema.Number)), - windowMinutes: Schema.optionalKey(Schema.Number), - severities: Schema.optionalKey(Schema.Array(Schema.String)), - }), - ), - peakSeverity: Schema.optionalKey(Schema.NullOr(Schema.String)), - observed: Schema.optionalKey( - Schema.Struct({ - value: Schema.optionalKey(Schema.NullOr(Schema.Number)), - sampleCount: Schema.optionalKey(Schema.NullOr(Schema.Number)), - detail: Schema.optionalKey(Schema.NullOr(Schema.String)), - }), - ), - template: Schema.optionalKey(Schema.NullOr(AlertNotificationTemplate)), - chart: Schema.optionalKey( - Schema.NullOr( - Schema.Struct({ - sparkline: Schema.optionalKey(Schema.String), - url: Schema.optionalKey(Schema.String), - }), - ), - ), - }), - [Schema.Record(Schema.String, Schema.Unknown)], -) - const decodeAlertRuleIdSync = Schema.decodeUnknownSync(AlertRuleDocument.fields.id) const decodeAlertIncidentIdSync = Schema.decodeUnknownSync(AlertIncidentDocument.fields.id) const decodeAlertDeliveryEventIdSync = Schema.decodeUnknownSync(AlertDeliveryEventDocument.fields.id) @@ -531,7 +490,7 @@ const isolate = const toIngestDateTime64 = warehouseDateTime64 -type StoredDeliveryPayloadType = Schema.Schema.Type +type StoredDeliveryPayloadType = StoredDeliveryPayload const parseDeliveryPayload = ( value: unknown, @@ -2401,6 +2360,23 @@ export class AlertsService extends Context.Service db.run( diff --git a/packages/backend/src/services/alerts/delivery/context.ts b/packages/backend/src/services/alerts/delivery/context.ts index a56f10468c..b113115136 100644 --- a/packages/backend/src/services/alerts/delivery/context.ts +++ b/packages/backend/src/services/alerts/delivery/context.ts @@ -57,6 +57,8 @@ export interface DispatchContext { readonly eventType: AlertEventType readonly incidentId: string | null readonly incidentStatus: string + /** Who acknowledged the open incident, as a name to show; absent or null while unacknowledged. */ + readonly incidentAcknowledgedBy?: string | null readonly dedupeKey: string readonly windowMinutes: number readonly value: number | null diff --git a/packages/backend/src/services/alerts/delivery/dispatch.ts b/packages/backend/src/services/alerts/delivery/dispatch.ts index 04094a1150..32320ae2de 100644 --- a/packages/backend/src/services/alerts/delivery/dispatch.ts +++ b/packages/backend/src/services/alerts/delivery/dispatch.ts @@ -33,6 +33,14 @@ export const dispatchDelivery = ( /** `sendEmail` (the platform email channel) and `postChatAlert` (a chat connector). */ deps: EffectTransportDeps, ): Effect.Effect => { + // Only PagerDuty has an acknowledge; the queue never sends one elsewhere, this keeps it so. + if (context.eventType === "acknowledge" && context.secretConfig.type !== "pagerduty") { + return Effect.succeed({ + providerMessage: "Skipped: acknowledgements are only sent to PagerDuty", + providerReference: null, + responseCode: null, + }) + } const runtime: TransportRuntime = { timeoutMs } /** * Resolved once here rather than by each provider: the template is a diff --git a/packages/backend/src/services/alerts/delivery/stored-payload.ts b/packages/backend/src/services/alerts/delivery/stored-payload.ts new file mode 100644 index 0000000000..753a46578f --- /dev/null +++ b/packages/backend/src/services/alerts/delivery/stored-payload.ts @@ -0,0 +1,52 @@ +/** + * The JSON an `alert_delivery_events` row carries: what the event saw, snapshotted when it was + * queued. Read by the dispatcher, and by the chat card refresh to redraw a card it posted. + */ +import { AlertNotificationTemplate } from "@maple/domain/http" +import { Schema } from "effect" + +export const StoredDeliveryPayloadSchema = Schema.StructWithRest( + Schema.Struct({ + event: Schema.optionalKey(Schema.Unknown), + eventType: Schema.optionalKey(Schema.String), + incidentId: Schema.optionalKey(Schema.NullOr(Schema.String)), + incidentStatus: Schema.optionalKey(Schema.String), + dedupeKey: Schema.optionalKey(Schema.String), + rule: Schema.optionalKey( + Schema.Struct({ + id: Schema.optionalKey(Schema.String), + name: Schema.optionalKey(Schema.String), + signalType: Schema.optionalKey(Schema.String), + severity: Schema.optionalKey(Schema.String), + groupKey: Schema.optionalKey(Schema.NullOr(Schema.String)), + comparator: Schema.optionalKey(Schema.String), + threshold: Schema.optionalKey(Schema.Number), + thresholdUpper: Schema.optionalKey(Schema.NullOr(Schema.Number)), + windowMinutes: Schema.optionalKey(Schema.Number), + severities: Schema.optionalKey(Schema.Array(Schema.String)), + }), + ), + peakSeverity: Schema.optionalKey(Schema.NullOr(Schema.String)), + observed: Schema.optionalKey( + Schema.Struct({ + value: Schema.optionalKey(Schema.NullOr(Schema.Number)), + sampleCount: Schema.optionalKey(Schema.NullOr(Schema.Number)), + detail: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), + ), + template: Schema.optionalKey(Schema.NullOr(AlertNotificationTemplate)), + linkUrl: Schema.optionalKey(Schema.String), + chatUrl: Schema.optionalKey(Schema.String), + chart: Schema.optionalKey( + Schema.NullOr( + Schema.Struct({ + sparkline: Schema.optionalKey(Schema.String), + url: Schema.optionalKey(Schema.String), + }), + ), + ), + }), + [Schema.Record(Schema.String, Schema.Unknown)], +) + +export type StoredDeliveryPayload = Schema.Schema.Type diff --git a/packages/backend/src/services/alerts/delivery/transports/chat.test.ts b/packages/backend/src/services/alerts/delivery/transports/chat.test.ts index 2fef43fb6b..487356c2af 100644 --- a/packages/backend/src/services/alerts/delivery/transports/chat.test.ts +++ b/packages/backend/src/services/alerts/delivery/transports/chat.test.ts @@ -117,6 +117,28 @@ describe("chat alert blocks", () => { assert.strictEqual(card.sentAtMs, 1_700_000_000_000) }) + it("offers an open incident's firing card the ack control, with who acknowledged it", () => { + assert.deepStrictEqual(cardOf(buildChatAlertBlocks(inputFor())).acknowledge, { + incidentId: "inc_1", + acknowledgedBy: null, + }) + const acked = cardOf( + buildChatAlertBlocks(inputFor({ context: { ...context, incidentAcknowledgedBy: "Ada" } })), + ) + assert.deepStrictEqual(acked.acknowledge, { incidentId: "inc_1", acknowledgedBy: "Ada" }) + }) + + it("gives no ack control to a resolve, a test or a resolved incident", () => { + for (const overrides of [ + { eventType: "resolve" as const }, + { eventType: "test" as const }, + { incidentStatus: "resolved" }, + ]) { + const card = cardOf(buildChatAlertBlocks(inputFor({ context: { ...context, ...overrides } }))) + assert.isUndefined(card.acknowledge) + } + }) + it("colours a resolve green", () => { const card = cardOf(buildChatAlertBlocks(inputFor({ context: { ...context, eventType: "resolve" } }))) assert.strictEqual(card.color, "#2eb67d") diff --git a/packages/backend/src/services/alerts/delivery/transports/chat.ts b/packages/backend/src/services/alerts/delivery/transports/chat.ts index a03a0a508f..d149df76a2 100644 --- a/packages/backend/src/services/alerts/delivery/transports/chat.ts +++ b/packages/backend/src/services/alerts/delivery/transports/chat.ts @@ -10,7 +10,8 @@ import { alertAccentColor, truncate, } from "../../alert-formatting" -import type { EffectTransport, RenderInput, SecretConfigOf } from "../Transport" +import type { TemplateRenderContext } from "../../alert-formatting" +import type { EffectTransport, ResolvedTitleBody, SecretConfigOf } from "../Transport" type Config = SecretConfigOf<"chat"> @@ -29,10 +30,42 @@ const MAX_TEMPLATED_BODY_CHARS = 1200 * and a footer carrying the sparkline, the incident and the time. A rule's own template, when it * has one, replaces the title and the summary and drops the fields, as it does there. */ -export const buildChatAlertBlocks = (input: RenderInput): ReadonlyArray => { +export interface ChatAlertCardInput { + readonly context: TemplateRenderContext & { readonly incidentAcknowledgedBy?: string | null } + readonly templated: ResolvedTitleBody | null + readonly linkUrl: string +} + +/** The name an open, acknowledged incident shows on its alert cards; null otherwise. */ +export const incidentAcknowledgedBy = ( + incident: { + readonly status: string + readonly acknowledgedAt: number | null + readonly acknowledgedByName: string | null + } | null, +): string | null => + incident === null || incident.status !== "open" || incident.acknowledgedAt === null + ? null + : (incident.acknowledgedByName ?? "someone") + +/** Only a firing card of an open incident can be acknowledged from chat. */ +const ackControl = (context: ChatAlertCardInput["context"]): Pick => + context.incidentId !== null && + context.incidentStatus === "open" && + (context.eventType === "trigger" || context.eventType === "renotify") + ? { + acknowledge: { + incidentId: context.incidentId, + acknowledgedBy: context.incidentAcknowledgedBy ?? null, + }, + } + : {} + +export const buildChatAlertBlocks = (input: ChatAlertCardInput): ReadonlyArray => { const { context, templated, linkUrl } = input const group = displayGroupKey(context.groupKey) const card: Omit = { + ...ackControl(context), kind: "alert", color: alertAccentColor(context.eventType, context.severity), imageUrl: context.chartUrl ?? null, diff --git a/packages/backend/src/services/alerts/delivery/transports/linear.ts b/packages/backend/src/services/alerts/delivery/transports/linear.ts index 4d3c584269..b59b893e57 100644 --- a/packages/backend/src/services/alerts/delivery/transports/linear.ts +++ b/packages/backend/src/services/alerts/delivery/transports/linear.ts @@ -456,6 +456,12 @@ export const linearTransport: EffectTransport = { providerReference: null, responseCode: null, } satisfies DispatchResult + case "acknowledge": + return { + providerMessage: "Skipped: acknowledgements are only sent to PagerDuty", + providerReference: null, + responseCode: null, + } satisfies DispatchResult case "test": { const team = yield* fetchTeam(deps.linear, input) return { diff --git a/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts b/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts index 4a8a9c8bfb..f3c55625cc 100644 --- a/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts +++ b/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts @@ -1,4 +1,4 @@ -import { Duration, Effect, Schema, String as Str } from "effect" +import { Duration, Effect, Schema, Array as Arr, String as Str } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" import type { AlertSeverity } from "@maple/domain/http" import { displayGroupKey, formatAlertHeadline, truncate, uptimeSummaryLine } from "../../alert-formatting" @@ -41,14 +41,31 @@ export const pagerDutyDedupKey = ( ? criticalDedupKey(context.dedupeKey) : context.dedupeKey -const resolveRequest = (integrationKey: string, dedupKey: string): HttpRequestSpec => ({ +const actionRequest = ( + integrationKey: string, + action: "resolve" | "acknowledge", + dedupKey: string, +): HttpRequestSpec => ({ url: EVENTS_URL, headers: { "content-type": "application/json" }, guarded: false, sensitivePath: false, - body: encodeJson({ routing_key: integrationKey, event_action: "resolve", dedup_key: dedupKey }), + body: encodeJson({ routing_key: integrationKey, event_action: action, dedup_key: dedupKey }), }) +/** + * An acknowledgement covers every alert the incident may hold open: the current tier's key, and + * for a critical incident of a tiered rule also the warning key it opened under (a de-escalation + * already resolved the critical key, so a warning incident holds only one). PagerDuty ignores an + * acknowledge for a key it has no open alert for. There is no un-acknowledge in the Events API. + */ +export const pagerDutyAcknowledgeKeys = ( + context: Pick, +): ReadonlyArray => { + const primary = pagerDutyDedupKey(context, keySeverity(context)) + return primary === context.dedupeKey ? [primary] : [primary, context.dedupeKey] +} + /** * PagerDuty Events API v2. The body is machine-facing, so `custom_details` * carries raw numbers rather than the formatted strings the chat providers @@ -62,6 +79,10 @@ export const pagerDutyTransport: HttpTransport = { providerLabel: "PagerDuty", render: (input: RenderInput) => { const { context, templated, linkUrl, chatUrl } = input + if (context.eventType === "acknowledge") { + const [primary = context.dedupeKey] = pagerDutyAcknowledgeKeys(context) + return actionRequest(input.config.integrationKey, "acknowledge", primary) + } return { // Fixed vendor host: nothing user-supplied to guard against, and the // path carries no credential (the routing key rides in the body). @@ -113,6 +134,11 @@ export const pagerDutyTransport: HttpTransport = { // key PagerDuty has never seen is a no-op. followUps: (input) => { const { context } = input + if (context.eventType === "acknowledge") { + return Arr.map(Arr.drop(pagerDutyAcknowledgeKeys(context), 1), (key) => + actionRequest(input.config.integrationKey, "acknowledge", key), + ) + } const mayHoldCriticalKey = criticalHasOwnKey(context) || context.incidentPeakSeverity === "critical" // Judged by where the incident is now, not by the queued event: a warning // reminder delivered after an escalation must leave the critical page open. @@ -120,11 +146,14 @@ export const pagerDutyTransport: HttpTransport = { context.eventType === "resolve" || (context.eventType !== "test" && (context.incidentSeverity ?? context.severity) === "warning") return mayHoldCriticalKey && closesCritical - ? [resolveRequest(input.config.integrationKey, criticalDedupKey(context.dedupeKey))] + ? [actionRequest(input.config.integrationKey, "resolve", criticalDedupKey(context.dedupeKey))] : [] }, ack: (input) => ({ - providerMessage: "Delivered to PagerDuty", + providerMessage: + input.context.eventType === "acknowledge" + ? "Acknowledged in PagerDuty" + : "Delivered to PagerDuty", providerReference: pagerDutyDedupKey(input.context, keySeverity(input.context)), }), } diff --git a/packages/backend/src/services/alerts/delivery/transports/render.test.ts b/packages/backend/src/services/alerts/delivery/transports/render.test.ts index 17e83cc7f3..8776a0be99 100644 --- a/packages/backend/src/services/alerts/delivery/transports/render.test.ts +++ b/packages/backend/src/services/alerts/delivery/transports/render.test.ts @@ -422,3 +422,43 @@ describe("transport render: uptime monitors", () => { ) }) }) + +describe("pagerduty acknowledge", () => { + const ack = (overrides: Partial) => { + const input = inputFor( + { type: "pagerduty" as const, integrationKey: "k" }, + { context: { ...context, eventType: "acknowledge", ...overrides } }, + ) + return Arr.map( + [pagerDutyTransport.render(input), ...(pagerDutyTransport.followUps?.(input) ?? [])], + (spec) => JSON.parse(spec.body), + ) + } + const tiered = { ruleSeverities: ["warning", "critical"] as const } + + it("acknowledges the incident's key with a bare acknowledge event", () => { + assert.deepStrictEqual(ack({ ruleSeverities: ["critical"] }), [ + { routing_key: "k", event_action: "acknowledge", dedup_key: "org_1:rule_1:checkout" }, + ]) + }) + + it("acknowledges both alerts a critical incident of a tiered rule may hold open", () => { + assert.deepStrictEqual( + Arr.map( + ack({ ...tiered, severity: "critical", incidentSeverity: "critical" }), + (body) => `${body.event_action} ${body.dedup_key}`, + ), + ["acknowledge org_1:rule_1:checkout:critical", "acknowledge org_1:rule_1:checkout"], + ) + }) + + it("acknowledges only the warning key once the incident has fallen back to warning", () => { + assert.deepStrictEqual( + Arr.map( + ack({ ...tiered, severity: "critical", incidentSeverity: "warning" }), + (body) => `${body.event_action} ${body.dedup_key}`, + ), + ["acknowledge org_1:rule_1:checkout"], + ) + }) +}) diff --git a/packages/backend/src/services/push/MobilePushService.ts b/packages/backend/src/services/push/MobilePushService.ts index 727f8d5957..48b680108f 100644 --- a/packages/backend/src/services/push/MobilePushService.ts +++ b/packages/backend/src/services/push/MobilePushService.ts @@ -179,6 +179,7 @@ const wantsEvent = (device: MobileDevice, event: IncidentPushEvent): boolean => ? device.preferences.criticalIncidents : device.preferences.warningIncidents case "test": + case "acknowledge": return false } } @@ -354,6 +355,8 @@ export const renderIncidentPush = ( } case "trigger": case "test": + // Never pushed (`wantsEvent` refuses it); rendered like a trigger to keep the switch total. + case "acknowledge": return { alert: { title: `${severity} · ${rule}`, diff --git a/packages/chat-platform/src/action-token.test.ts b/packages/chat-platform/src/action-token.test.ts index 28b739b44b..9cf422f8a0 100644 --- a/packages/chat-platform/src/action-token.test.ts +++ b/packages/chat-platform/src/action-token.test.ts @@ -1,9 +1,11 @@ -import { Option } from "effect" +import { Option, String as Str } from "effect" import { describe, expect, it } from "vitest" import { chatActionControlId, chatActionDecision, + chatAlertAckControlId, decodeChatActionControlId, + decodeChatAlertAckControlId, encodeChatActionToken, } from "./action-token" @@ -50,3 +52,40 @@ describe("chat action control id", () => { expect(Option.getOrUndefined(chatActionDecision(`some-other-app:${token}`))).toBeUndefined() }) }) + +describe("alert ack control id", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + const read = (raw: string) => Option.getOrUndefined(decodeChatAlertAckControlId(raw)) + + it("round-trips both actions with the incident they name", () => { + expect(read(chatAlertAckControlId("acknowledge", INCIDENT))).toEqual({ + action: "acknowledge", + incidentId: INCIDENT, + }) + expect(read(chatAlertAckControlId("unacknowledge", INCIDENT))).toEqual({ + action: "unacknowledge", + incidentId: INCIDENT, + }) + }) + + it("fits the tightest platform's 100-character control id", () => { + expect(chatAlertAckControlId("unacknowledge", INCIDENT).length).toBeLessThanOrEqual(100) + }) + + it.each([ + "", + "alert_ack:", + "alert_ack:not-a-uuid", + `alert_ack:${INCIDENT}x`, + `alert_ack:${Str.toUpperCase(INCIDENT)}`, + `alert_ack:org_other:${INCIDENT}`, + `alert_snooze:${INCIDENT}`, + `approve:${INCIDENT}`, + ])("rejects %j", (raw) => { + expect(read(raw)).toBeUndefined() + }) + + it("is not mistaken for an approval control", () => { + expect(readControl(chatAlertAckControlId("acknowledge", INCIDENT))).toBeUndefined() + }) +}) diff --git a/packages/chat-platform/src/action-token.ts b/packages/chat-platform/src/action-token.ts index 5e4c932926..f08fe410bb 100644 --- a/packages/chat-platform/src/action-token.ts +++ b/packages/chat-platform/src/action-token.ts @@ -79,3 +79,38 @@ export const chatActionDecision = (raw: string): Option.Option Str.startsWith(`${candidate}${DECISION_SEPARATOR}`)(raw), ) + +/** The two things an alert card's button can do to its incident. */ +export type ChatAlertAckAction = "acknowledge" | "unacknowledge" + +export interface ChatAlertAckRequest { + readonly action: ChatAlertAckAction + readonly incidentId: string +} + +const ALERT_ACK_PREFIXES = { + acknowledge: "alert_ack:", + unacknowledge: "alert_unack:", +} as const satisfies Record + +const ALERT_ACK_ACTIONS: ReadonlyArray = ["acknowledge", "unacknowledge"] + +// An incident id is a UUID. Anything else on an ack prefix is forged or corrupt. +const INCIDENT_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + +/** + * An alert card's ack button. Carries the incident only: the org always comes from the workspace + * the click landed in, so a forged id can at most name an incident of that org. + */ +export const chatAlertAckControlId = (action: ChatAlertAckAction, incidentId: string): string => + `${ALERT_ACK_PREFIXES[action]}${incidentId}` + +/** Read an ack button back. `None` for anything that is not exactly one of Maple's. */ +export const decodeChatAlertAckControlId = (raw: string): Option.Option => + Option.flatMap( + Arr.findFirst(ALERT_ACK_ACTIONS, (action) => Str.startsWith(ALERT_ACK_PREFIXES[action])(raw)), + (action) => { + const incidentId = Str.slice(ALERT_ACK_PREFIXES[action].length)(raw) + return INCIDENT_ID_PATTERN.test(incidentId) ? Option.some({ action, incidentId }) : Option.none() + }, + ) diff --git a/packages/chat-platform/src/connectors/discord/render.test.ts b/packages/chat-platform/src/connectors/discord/render.test.ts index 4105460157..74b1a73298 100644 --- a/packages/chat-platform/src/connectors/discord/render.test.ts +++ b/packages/chat-platform/src/connectors/discord/render.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import type { ChatAlertBlock, ChatBlock } from "../../render/blocks" -import type { ChatActionToken } from "../../action-token" +import { decodeChatAlertAckControlId, type ChatActionToken } from "../../action-token" import { EMPTY_CONTENT, MAX_ACTION_ROWS, MAX_CONTENT_CHARS, renderDiscordMessage } from "./render" import { Array as Arr, Option, String as Str } from "effect" @@ -228,3 +228,43 @@ describe("renderDiscordMessage", () => { ]) }) }) + +describe("an alert's acknowledge control on Discord", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + + it("puts an Acknowledge button under an unacknowledged alert", () => { + const payload = renderDiscordMessage([ + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: null } }), + ]) + expect(payload.components).toHaveLength(1) + const button = payload.components[0]?.components[0] + expect(button?.label).toBe("Acknowledge") + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.custom_id ?? ""))).toEqual({ + action: "acknowledge", + incidentId: INCIDENT, + }) + expect(Arr.some(payload.embeds[0]?.fields ?? [], (field) => field.name === "Acknowledged by")).toBe( + false, + ) + }) + + it("names who acknowledged on the embed and offers Unacknowledge", () => { + const payload = renderDiscordMessage([ + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: "Ada" } }), + ]) + const button = payload.components[0]?.components[0] + expect(button?.label).toBe("Unacknowledge") + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.custom_id ?? ""))?.action).toBe( + "unacknowledge", + ) + expect(payload.embeds[0]?.fields?.[0]).toEqual({ + name: "Acknowledged by", + value: "Ada", + inline: false, + }) + }) + + it("adds no component to an alert without the control", () => { + expect(renderDiscordMessage([alert()]).components).toEqual([]) + }) +}) diff --git a/packages/chat-platform/src/connectors/discord/render.ts b/packages/chat-platform/src/connectors/discord/render.ts index 9db810fc2b..385778d1bf 100644 --- a/packages/chat-platform/src/connectors/discord/render.ts +++ b/packages/chat-platform/src/connectors/discord/render.ts @@ -8,8 +8,8 @@ * characters, a message carries at most 10 embeds, an action row at most 5 buttons, and a button's * `custom_id` is 1–100 characters. */ -import { chatActionControlId, type ChatActionToken } from "../../action-token" -import type { ChatAlertBlock, ChatBlock, ChatToolActivity } from "../../render/blocks" +import { chatActionControlId, chatAlertAckControlId, type ChatActionToken } from "../../action-token" +import type { ChatAlertAcknowledge, ChatAlertBlock, ChatBlock, ChatToolActivity } from "../../render/blocks" import { Array as Arr, Option, String as Str } from "effect" /** https://discord.com/developers — Create/Edit Message JSON params, API v10. */ @@ -32,8 +32,8 @@ export interface DiscordEmbed { export interface DiscordButton { readonly type: 2 - /** 3 = success, 4 = danger. */ - readonly style: 3 | 4 + /** 1 = primary, 2 = secondary, 3 = success, 4 = danger. */ + readonly style: 1 | 2 | 3 | 4 readonly label: string readonly custom_id: string } @@ -141,6 +141,9 @@ export const renderDiscordMessage = (blocks: ReadonlyArray): DiscordM lines.push(block.tone === "error" ? `**${block.text}**` : `_${block.text}_`) break case "alert": { + if (block.acknowledge !== undefined && rows.length < MAX_ACTION_ROWS) { + rows.push(alertAckRow(block.acknowledge)) + } const embed = alertEmbed(block) const cost = embedCost(embed) if (embeds.length < MAX_EMBEDS && embedChars + cost <= MAX_EMBED_TOTAL_CHARS) { @@ -189,8 +192,12 @@ const alertEmbed = (block: ChatAlertBlock): DiscordEmbed => { Arr.map(block.links, (link) => `[${link.label}](${link.url})`), " · ", ) + const acknowledgedBy = block.acknowledge?.acknowledgedBy ?? null const fields = Arr.map( [ + ...(acknowledgedBy === null + ? [] + : [{ name: "Acknowledged by", value: acknowledgedBy, inline: false }]), ...Arr.map(block.fields, (field) => ({ name: field.label, value: field.value, inline: true })), ...(links === "" ? [] : [{ name: "Links", value: links, inline: false }]), ].slice(0, MAX_EMBED_FIELDS), @@ -246,6 +253,25 @@ const toolLabel = (tool: ChatToolActivity): string => { return `${tool.label}${detail}${status}` } +/** An alert's ack toggle: Acknowledge while open and unacknowledged, Unacknowledge once acked. */ +const alertAckRow = (ack: ChatAlertAcknowledge): DiscordActionRow => { + const acknowledged = ack.acknowledgedBy !== null + return { + type: 1, + components: [ + { + type: 2, + style: acknowledged ? 2 : 1, + label: acknowledged ? "Unacknowledge" : "Acknowledge", + custom_id: chatAlertAckControlId( + acknowledged ? "unacknowledge" : "acknowledge", + ack.incidentId, + ), + }, + ], + } +} + /** * The two buttons, carrying the driver's action control id through Discord's `custom_id`. * diff --git a/packages/chat-platform/src/connectors/slack/render.test.ts b/packages/chat-platform/src/connectors/slack/render.test.ts index 35fa3f5c40..d8dd4be11e 100644 --- a/packages/chat-platform/src/connectors/slack/render.test.ts +++ b/packages/chat-platform/src/connectors/slack/render.test.ts @@ -4,7 +4,12 @@ */ import { Option, Array as Arr, String as Str } from "effect" import { describe, expect, it } from "vitest" -import { decodeChatActionControlId, encodeChatActionToken, type ChatActionToken } from "../../action-token" +import { + decodeChatActionControlId, + decodeChatAlertAckControlId, + encodeChatActionToken, + type ChatActionToken, +} from "../../action-token" import type { ChatAlertBlock, ChatBlock } from "../../render/blocks" import { APPROVE_ACTION, @@ -300,4 +305,60 @@ describe("the rest of the vocabulary", () => { ?.blocks[1] expect(section?.type === "section" && section.text.text).toBe("<!channel> look") }) + + describe("an alert's acknowledge control", () => { + const INCIDENT = "00000000-0000-4000-8000-0000000000a1" + const ackButton = (block: ChatAlertBlock) => { + const blocks = renderSlackMessage([block]).attachments?.[0]?.blocks ?? [] + const actions = Option.getOrUndefined(Arr.findFirst(blocks, (b) => b.type === "actions")) + return { + blocks, + button: + actions?.type === "actions" + ? Option.getOrUndefined(Arr.findFirst(actions.elements, (b) => b.value !== undefined)) + : undefined, + } + } + + it("offers Acknowledge on an unacknowledged incident, carrying a control the host can read", () => { + const { blocks, button } = ackButton( + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: null } }), + ) + expect(button?.text.text).toBe("Acknowledge") + expect(button?.url).toBeUndefined() + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.value ?? ""))).toEqual({ + action: "acknowledge", + incidentId: INCIDENT, + }) + const contexts = Arr.filter(blocks, (block) => block.type === "context") + expect( + Arr.some( + contexts, + (block) => + block.type === "context" && + Arr.some(block.elements, (element) => Str.includes("Acknowledged")(element.text)), + ), + ).toBe(false) + }) + + it("shows who acknowledged and offers Unacknowledge", () => { + const { blocks, button } = ackButton( + alert({ acknowledge: { incidentId: INCIDENT, acknowledgedBy: "sam_*ops*" } }), + ) + expect(button?.text.text).toBe("Unacknowledge") + expect(Option.getOrUndefined(decodeChatAlertAckControlId(button?.value ?? ""))?.action).toBe( + "unacknowledge", + ) + const texts = Arr.flatMap(blocks, (block) => + block.type === "context" ? Arr.map(block.elements, (element) => element.text) : [], + ) + expect( + Arr.some(texts, (text) => Str.includes("Acknowledged by")(text) && Str.includes("sam")(text)), + ).toBe(true) + }) + + it("renders no ack button on a card without the control", () => { + expect(ackButton(alert()).button).toBeUndefined() + }) + }) }) diff --git a/packages/chat-platform/src/connectors/slack/render.ts b/packages/chat-platform/src/connectors/slack/render.ts index c36e034251..e24d1ea5de 100644 --- a/packages/chat-platform/src/connectors/slack/render.ts +++ b/packages/chat-platform/src/connectors/slack/render.ts @@ -12,7 +12,7 @@ * `section` carries at most 10 `fields` of at most 2000 each, and a link button's `url` is at most * 3000. */ -import { chatActionControlId, type ChatActionToken } from "../../action-token" +import { chatActionControlId, chatAlertAckControlId, type ChatActionToken } from "../../action-token" import type { ChatAlertBlock, ChatBlock, ChatToolActivity } from "../../render/blocks" import { escapeMrkdwn, toMrkdwn } from "./mrkdwn" import { Array as Arr, String as Str } from "effect" @@ -142,6 +142,7 @@ export const MAX_FIELD_CHARS = 2000 export const MAX_LINK_BUTTONS = 5 export const APPROVE_ACTION = "maple_approve" +export const ALERT_ACK_ACTION = "maple_alert_ack" export const DENY_ACTION = "maple_deny" /** A message with nothing in it: Slack rejects a post carrying neither text nor blocks. */ @@ -279,23 +280,32 @@ const alertAttachment = (block: ChatAlertBlock): SlackAttachment => { alt_text: clamp(block.imageAlt === "" ? "Chart" : block.imageAlt, MAX_ALT_TEXT_CHARS), }) } + const ack = block.acknowledge + if (ack !== undefined && ack.acknowledgedBy !== null) { + blocks.push(context(`:white_check_mark: Acknowledged by *${escapeMrkdwn(ack.acknowledgedBy)}*`)) + } // A link too long for a button is left out rather than sent: Slack rejects the whole message. const links = Arr.filter(block.links, (link) => link.url.length <= MAX_IMAGE_URL_CHARS).slice( 0, MAX_LINK_BUTTONS, ) - if (links.length > 0) { - blocks.push({ - type: "actions", - elements: Arr.map(links, (link, index) => ({ - type: "button", - text: { type: "plain_text", text: clamp(link.label, MAX_BUTTON_TEXT_CHARS), emoji: true }, - action_id: `maple_link_${index}`, - url: link.url, - ...(link.primary ? { style: "primary" as const } : undefined), - })), + const buttons: Array = Arr.map(links, (link, index) => ({ + type: "button", + text: { type: "plain_text", text: clamp(link.label, MAX_BUTTON_TEXT_CHARS), emoji: true }, + action_id: `maple_link_${index}`, + url: link.url, + ...(link.primary ? { style: "primary" as const } : undefined), + })) + if (ack !== undefined) { + const acknowledged = ack.acknowledgedBy !== null + buttons.push({ + type: "button", + text: { type: "plain_text", text: acknowledged ? "Unacknowledge" : "Acknowledge" }, + action_id: ALERT_ACK_ACTION, + value: chatAlertAckControlId(acknowledged ? "unacknowledge" : "acknowledge", ack.incidentId), }) } + if (buttons.length > 0) blocks.push({ type: "actions", elements: buttons }) const footer = [ ...Arr.map(block.footer, toMrkdwn), ...(block.sentAtMs === null ? [] : [slackDate(block.sentAtMs)]), diff --git a/packages/chat-platform/src/render/blocks.ts b/packages/chat-platform/src/render/blocks.ts index 50209b0133..50869f73f1 100644 --- a/packages/chat-platform/src/render/blocks.ts +++ b/packages/chat-platform/src/render/blocks.ts @@ -124,6 +124,13 @@ export interface ChatAlertLink { readonly primary: boolean } +/** The incident an alert card can acknowledge, and whether somebody already has. */ +export interface ChatAlertAcknowledge { + readonly incidentId: string + /** Who acknowledged it, as a name to show; null while it is unacknowledged. */ + readonly acknowledgedBy: string | null +} + /** * A notification Maple posts on its own — an alert firing or resolving — rather than a turn. * @@ -147,6 +154,8 @@ export interface ChatAlertBlock { readonly footer: ReadonlyArray /** When it was sent, epoch ms, for a connector that can show it in each reader's own timezone. */ readonly sentAtMs: number | null + /** An open incident's ack control; absent on cards with nothing to acknowledge (resolve, test). */ + readonly acknowledge?: ChatAlertAcknowledge } export type ChatBlock = diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index 57af9712db..37c32ed11b 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -149,7 +149,13 @@ export const AlertIncidentEventType = Schema.Literals([ }) export type AlertIncidentEventType = Schema.Schema.Type -export const AlertEventType = Schema.Literals(["trigger", "resolve", "renotify", "test"]).annotate({ +export const AlertEventType = Schema.Literals([ + "trigger", + "resolve", + "renotify", + "test", + "acknowledge", +]).annotate({ identifier: "@maple/AlertEventType", title: "Alert Event Type", }) diff --git a/packages/web-data/src/alerts/form-utils.ts b/packages/web-data/src/alerts/form-utils.ts index 8826c3ad0e..fbdf41bd85 100644 --- a/packages/web-data/src/alerts/form-utils.ts +++ b/packages/web-data/src/alerts/form-utils.ts @@ -1090,6 +1090,7 @@ export const eventTypeMeta: Record export type DeliveryStatusVariant = Exclude | "outline" diff --git a/packages/web-data/src/alerts/incident-timeline.test.ts b/packages/web-data/src/alerts/incident-timeline.test.ts index 4ae08e547f..39a27791db 100644 --- a/packages/web-data/src/alerts/incident-timeline.test.ts +++ b/packages/web-data/src/alerts/incident-timeline.test.ts @@ -101,6 +101,17 @@ describe("buildIncidentTimeline", () => { expect(failed?.detail).toBe("discord · webhook returned 500") }) + it("names a PagerDuty acknowledgement as one, not as a notification", () => { + const [, sent] = build(makeIncident(), [ + makeDelivery({ + eventType: "acknowledge", + destinationName: "On-call", + destinationType: "pagerduty", + }), + ]) + expect(sent?.title).toBe("Sent acknowledgement to On-call") + }) + it("ends a held incident on the hold instead of a breach", () => { const entries = build(makeIncident({ holdReason: "no_data", heldSince: "2026-10-15T14:40:00.000Z" })) expect(entries.at(-1)?.title).toBe("Waiting on data") diff --git a/packages/web-data/src/alerts/incident-timeline.ts b/packages/web-data/src/alerts/incident-timeline.ts index 754627e41f..3af13c54e7 100644 --- a/packages/web-data/src/alerts/incident-timeline.ts +++ b/packages/web-data/src/alerts/incident-timeline.ts @@ -86,9 +86,13 @@ function deliveryEntry( ? failed ? `Reminder to ${name} failed` : `Reminded ${name}` - : failed - ? `Could not notify ${name}` - : `Notified ${name}` + : event.eventType === "acknowledge" + ? failed + ? `Acknowledgement to ${name} failed` + : `Sent acknowledgement to ${name}` + : failed + ? `Could not notify ${name}` + : `Notified ${name}` const reason = failed ? (event.errorMessage ?? (event.responseCode != null ? `HTTP ${event.responseCode}` : null)) : null From 9bde5ae7844160839e0cd949a2cc8b456ab395d7 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:25:27 +0100 Subject: [PATCH 2/4] fix(alerts): leave a skipped PagerDuty acknowledgement out of the incident timeline --- .../src/alerts/incident-timeline.test.ts | 21 +++++++++++++++++++ .../web-data/src/alerts/incident-timeline.ts | 3 +++ 2 files changed, 24 insertions(+) diff --git a/packages/web-data/src/alerts/incident-timeline.test.ts b/packages/web-data/src/alerts/incident-timeline.test.ts index 6487a35d00..20e813adad 100644 --- a/packages/web-data/src/alerts/incident-timeline.test.ts +++ b/packages/web-data/src/alerts/incident-timeline.test.ts @@ -202,4 +202,25 @@ describe("buildIncidentTimeline", () => { expect(entries[1]).toMatchObject({ title: "Acknowledgement cleared", tone: "warn" }) expect(entries.at(-1)?.detail).toBe("Last value 8.4% · reminders every 30m while open") }) + + it("leaves out an acknowledgement the dispatcher skipped", () => { + const sent = makeDelivery({ + eventType: "acknowledge", + destinationType: "pagerduty", + responseCode: 202, + }) + const skipped = makeDelivery({ + id: "00000000-0000-4000-8000-0000000000e9", + eventType: "acknowledge", + destinationType: "pagerduty", + responseCode: null, + providerMessage: "Skipped: the incident is no longer acknowledged", + }) + expect(Arr.map(build(makeIncident(), [sent]), (entry) => entry.title)).toContain( + "Sent acknowledgement to #alerts", + ) + expect(Arr.map(build(makeIncident(), [skipped]), (entry) => entry.title)).not.toContain( + "Sent acknowledgement to #alerts", + ) + }) }) diff --git a/packages/web-data/src/alerts/incident-timeline.ts b/packages/web-data/src/alerts/incident-timeline.ts index fbff0ecab4..fc9dbf8f26 100644 --- a/packages/web-data/src/alerts/incident-timeline.ts +++ b/packages/web-data/src/alerts/incident-timeline.ts @@ -137,6 +137,9 @@ function deliveryEntry( typeLabel: (type: AlertDestinationType) => string, ): IncidentTimelineEntry | null { if (event.eventType === "test") return null + // The dispatcher skips an acknowledgement an escalation overtook: nothing reached PagerDuty. + if (event.eventType === "acknowledge" && event.status === "success" && event.responseCode === null) + return null const name = event.destinationName const failed = event.status === "failed" const sending = event.status === "queued" || event.status === "processing" From 38780a7124ebea9acd72375d386fd70635e96c25 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:42:43 +0100 Subject: [PATCH 3/4] fix(alerts): keep PagerDuty acknowledgements internal and only send them after a page Review fixes for the PagerDuty acknowledge sync: - The public API keeps its four notification event types. `acknowledge` deliveries are left out of /v2/alerts/deliveries, since shipped clients decode the event type as a closed enum. - An acknowledgement goes only to PagerDuty destinations that already delivered this incident's page, so it never reaches a pager that was not notified or lands before the page itself. - Both dedup keys are acknowledged when the rule pages critical under its own key or the incident ever peaked at critical. - The dispatcher reads the ack right before sending, so an escalation that committed after the claim is not silenced. --- .../src/routes/v2/alert-deliveries.http.ts | 21 +++++++--- .../api/src/routes/v2/alert-incidents.http.ts | 8 +++- apps/api/src/routes/v2/alerts.http.test.ts | 15 ++++++- .../alerts/AlertIncidentAck.chat.test.ts | 19 ++++----- .../alerts/AlertIncidentAckService.ts | 40 ++++++------------- .../services/alerts/AlertReadModelsService.ts | 5 ++- .../src/services/alerts/AlertsService.ts | 25 +++++++++--- .../alerts/delivery/transports/pagerduty.ts | 18 ++++----- .../alerts/delivery/transports/render.test.ts | 21 +++++----- packages/domain/src/http/alerts.ts | 20 +++++++++- .../domain/src/http/v2/alert-deliveries.ts | 9 ++++- .../domain/src/http/v2/alert-incidents.ts | 4 +- .../src/alerts/incident-timeline.test.ts | 32 --------------- .../web-data/src/alerts/incident-timeline.ts | 13 ++---- 14 files changed, 131 insertions(+), 119 deletions(-) diff --git a/apps/api/src/routes/v2/alert-deliveries.http.ts b/apps/api/src/routes/v2/alert-deliveries.http.ts index 2c57ece5da..0676cd6118 100644 --- a/apps/api/src/routes/v2/alert-deliveries.http.ts +++ b/apps/api/src/routes/v2/alert-deliveries.http.ts @@ -1,12 +1,14 @@ import { HttpApiBuilder } from "effect/http-api" -import type { AlertDeliveryEventDocument } from "@maple/domain/http" -import { CurrentTenant } from "@maple/domain/http" +import type { AlertDeliveryEventDocument, AlertNotificationEventType } from "@maple/domain/http" +import { CurrentTenant, isAlertNotificationEventType } from "@maple/domain/http" import type { V2AlertDelivery } from "@maple/domain/http/v2" import { MapleApiV2, paginateOffsetQuery, timestamp, timestampOrNull } from "@maple/domain/http/v2" -import { Effect, Array as Arr } from "effect" +import { Effect, Result, Array as Arr } from "effect" import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService" -const toV2Delivery = (doc: AlertDeliveryEventDocument): V2AlertDelivery => ({ +const toV2Delivery = ( + doc: AlertDeliveryEventDocument & { readonly eventType: AlertNotificationEventType }, +): V2AlertDelivery => ({ id: doc.id, object: "alert_delivery", incident_id: doc.incidentId, @@ -39,10 +41,19 @@ export const HttpV2AlertDeliveriesLive = HttpApiBuilder.group(MapleApiV2, "alert ? { incidentId: query.incident_id } : undefined), ...(query.rule_id !== undefined ? { ruleId: query.rule_id } : undefined), + notificationsOnly: true, limit, offset, }) - .pipe(Effect.map((response) => Arr.map(response.events, toV2Delivery))), + .pipe( + Effect.map((response) => + Arr.filterMap(response.events, (doc) => + isAlertNotificationEventType(doc.eventType) + ? Result.succeed(toV2Delivery({ ...doc, eventType: doc.eventType })) + : Result.failVoid, + ), + ), + ), ) return { object: "list" as const, ...page } }), diff --git a/apps/api/src/routes/v2/alert-incidents.http.ts b/apps/api/src/routes/v2/alert-incidents.http.ts index 9235babcda..cf3b09f7be 100644 --- a/apps/api/src/routes/v2/alert-incidents.http.ts +++ b/apps/api/src/routes/v2/alert-incidents.http.ts @@ -1,6 +1,6 @@ import { HttpApiBuilder } from "effect/http-api" import type { AlertIncidentDocument, AlertIncidentEventDocument } from "@maple/domain/http" -import { CurrentTenant } from "@maple/domain/http" +import { CurrentTenant, isAlertNotificationEventType } from "@maple/domain/http" import type { V2AlertIncident, V2AlertIncidentEvent } from "@maple/domain/http/v2" import { MapleApiV2, paginateOffsetQuery } from "@maple/domain/http/v2" import { Duration, Effect, Option, Array as Arr } from "effect" @@ -31,7 +31,11 @@ const toV2Incident = (doc: AlertIncidentDocument): V2AlertIncident => ({ last_observed_value: doc.lastObservedValue, last_sample_count: doc.lastSampleCount, dedupe_key: doc.dedupeKey, - last_delivered_event_type: doc.lastDeliveredEventType, + // An acknowledgement never becomes the last delivered event; narrow it for the public enum. + last_delivered_event_type: + doc.lastDeliveredEventType !== null && isAlertNotificationEventType(doc.lastDeliveredEventType) + ? doc.lastDeliveredEventType + : null, last_notified_at: doc.lastNotifiedAt, hold_reason: doc.holdReason, held_since: doc.heldSince, diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index 9db81c6bf2..ebd3e436ff 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -430,6 +430,19 @@ describe("v2 alerts over HTTP", () => { )`, ) + // An internal PagerDuty acknowledgement sync is not a notification: the public list leaves it out. + await executeSql( + harness.testDb, + `insert into alert_delivery_events ( + id, org_id, rule_id, destination_id, delivery_key, event_type, + attempt_number, status, scheduled_at, payload_json, created_at, updated_at + ) + select '00000000-0000-4000-9000-0000000000ac', rules.org_id, rules.id, destinations.id, + 'ack-sync', 'acknowledge', 1, 'success', timestamp '2026-08-02 12:00:00+00', '{}'::jsonb, + timestamp '2026-08-02 12:00:00+00', timestamp '2026-08-02 12:00:00+00' + from (select id, org_id from alert_rules where org_id = 'org_alerts_e2e' limit 1) as rules + cross join (select id from alert_destinations where org_id = 'org_alerts_e2e' limit 1) as destinations`, + ) const first = await harness.request("GET", "/v2/alerts/deliveries?limit=60", key.secret) expect(first.status).toBe(200) expect(first.body.data).toHaveLength(60) @@ -451,7 +464,7 @@ describe("v2 alerts over HTTP", () => { expect( Arr.some( [...first.body.data, ...second.body.data], - (item) => item.delivery_key === "foreign-delivery", + (item) => item.delivery_key === "foreign-delivery" || item.delivery_key === "ack-sync", ), ).toBe(false) diff --git a/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts b/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts index dcb2b80c83..0d22a88a74 100644 --- a/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts +++ b/packages/backend/src/services/alerts/AlertIncidentAck.chat.test.ts @@ -91,7 +91,8 @@ const seed = (testDb: TestDb) => linkUrl: `https://app.maple.dev/alerts/incidents/${INCIDENT}`, chatUrl: `https://app.maple.dev/alerts/rules/${RULE}`, }) - // The card the channel got, and a page the former on-call destination got. + // The card the channel got, a page the former on-call destination got, and a page to the + // rule's on-call that is still waiting to go out. await executeSql( testDb, `INSERT INTO alert_delivery_events @@ -101,8 +102,10 @@ const seed = (testDb: TestDb) => ('00000000-0000-4000-8000-0000000000e1', $1, $2, $3, $4, 'k1', 'trigger', 1, 'success', '2026-11-06T09:00:00Z', '2026-11-06T09:00:01Z', $5, $6::jsonb, now(), now()), ('00000000-0000-4000-8000-0000000000e2', $1, $2, $3, $7, 'k2', 'trigger', 1, 'success', - '2026-11-06T09:00:00Z', '2026-11-06T09:00:01Z', 'd-open', $6::jsonb, now(), now())`, - [ORG, INCIDENT, RULE, CHAT_DEST, MESSAGE_TS, payload, PD_PRIOR], + '2026-11-06T09:00:00Z', '2026-11-06T09:00:01Z', 'd-open', $6::jsonb, now(), now()), + ('00000000-0000-4000-8000-0000000000e3', $1, $2, $3, $8, 'k3', 'trigger', 2, 'queued', + '2026-11-06T09:05:00Z', null, null, $6::jsonb, now(), now())`, + [ORG, INCIDENT, RULE, CHAT_DEST, MESSAGE_TS, payload, PD_PRIOR, PD_RULE], ) }) @@ -115,7 +118,7 @@ const queuedAcks = (testDb: TestDb) => ).pipe(Effect.map((result) => result.rows)) describe("acknowledging an incident queues PagerDuty acknowledgements", () => { - it.effect("to the enabled PagerDuty destinations of the rule and of the incident's past pages", () => { + it.effect("only to the PagerDuty destinations this incident already paged", () => { const testDb = createTestDb(createdDbs) return Effect.gen(function* () { yield* seed(testDb) @@ -126,10 +129,8 @@ describe("acknowledging an incident queues PagerDuty acknowledgements", () => { const queued = yield* queuedAcks(testDb) assert.deepStrictEqual( Arr.map(queued, (row) => [row.destination_id, row.status, row.payload_json.dedupeKey]), - [ - [PD_RULE, "queued", "d-open"], - [PD_PRIOR, "queued", "d-open"], - ], + // The rule's on-call has not been paged yet: an ack there would land before the page. + [[PD_PRIOR, "queued", "d-open"]], ) }).pipe(Effect.provide(makeLayer(testDb))) }) @@ -144,7 +145,7 @@ describe("acknowledging an incident queues PagerDuty acknowledgements", () => { yield* acks.acknowledge(ORG, INCIDENT, sam) yield* acks.unacknowledge(ORG, INCIDENT, sam) - assert.lengthOf(yield* queuedAcks(testDb), 2) + assert.lengthOf(yield* queuedAcks(testDb), 1) }).pipe(Effect.provide(makeLayer(testDb))) }) }) diff --git a/packages/backend/src/services/alerts/AlertIncidentAckService.ts b/packages/backend/src/services/alerts/AlertIncidentAckService.ts index 31692d4038..8eed1130ab 100644 --- a/packages/backend/src/services/alerts/AlertIncidentAckService.ts +++ b/packages/backend/src/services/alerts/AlertIncidentAckService.ts @@ -24,7 +24,6 @@ import { AlertDestinations, AlertIncidentEvents, AlertIncidents, - AlertRules, type AlertIncidentRow, } from "@maple/db/tables" import { makeAlertDeliveryKey } from "@maple/alerting-core" @@ -74,7 +73,6 @@ export interface AlertIncidentAckServiceApi { const decodeEventId = Schema.decodeUnknownSync(AlertIncidentEventId) const decodeDeliveryId = Schema.decodeUnknownSync(AlertDeliveryEventId) -const decodeIdList = Schema.decodeUnknownOption(Schema.Array(AlertDestinationId)) const decodeEventType = Schema.decodeUnknownOption(AlertIncidentEventType) const decodeSource = Schema.decodeUnknownOption(AlertIncidentAckSource) const decodeIso = Schema.decodeUnknownSync(AlertIncidentEventDocument.fields.createdAt) @@ -110,8 +108,9 @@ export class AlertIncidentAckService extends Context.Service< ) /** - * Queue a PagerDuty `acknowledge` for each enabled PagerDuty destination of the rule or that - * this incident already paged. Written in the ack's transaction; the alerting dispatcher sends it. + * Queue a PagerDuty `acknowledge` for each enabled PagerDuty destination this incident already + * paged, so it never reaches a pager that was not notified or lands before the page itself. + * Written in the ack's transaction; the alerting dispatcher sends it. */ const queuePagerDutyAcknowledge = ( db: MapleDb, @@ -120,30 +119,17 @@ export class AlertIncidentAckService extends Context.Service< makeUuid: () => string, ) => Effect.gen(function* () { - const [ruleRows, priorRows] = yield* Effect.all([ - db.run( - PG.from(AlertRules) - .select(($) => ({ destinationIdsJson: PG.undecoded($.destinationIdsJson) })) - .where(($) => [$.orgId.eq(incident.orgId), $.id.eq(incident.ruleId)]) - .limit(1), - ), - db.run( - PG.from(AlertDeliveryEvents) - .select("destinationId") - .where(($) => [$.orgId.eq(incident.orgId), $.incidentId.eq(incident.id)]), - ), - ]) - // A corrupt stored list must not fail the ack; the incident's own deliveries still count. - const ruleIds = Arr.flatMap(ruleRows, (row) => - Option.getOrElse( - decodeIdList(row.destinationIdsJson), - (): ReadonlyArray => [], - ), + const paged = yield* db.run( + PG.from(AlertDeliveryEvents) + .select("destinationId") + .where(($) => [ + $.orgId.eq(incident.orgId), + $.incidentId.eq(incident.id), + $.status.eq("success"), + $.eventType.in_("trigger", "renotify"), + ]), ) - const candidateIds = Arr.dedupe([ - ...ruleIds, - ...Arr.map(priorRows, (row) => row.destinationId), - ]) + const candidateIds = Arr.dedupe(Arr.map(paged, (row) => row.destinationId)) if (candidateIds.length === 0) return 0 const destinations = yield* db.run( PG.from(AlertDestinations) diff --git a/packages/backend/src/services/alerts/AlertReadModelsService.ts b/packages/backend/src/services/alerts/AlertReadModelsService.ts index bfcf088c73..283e4cac37 100644 --- a/packages/backend/src/services/alerts/AlertReadModelsService.ts +++ b/packages/backend/src/services/alerts/AlertReadModelsService.ts @@ -86,6 +86,8 @@ export interface ListAlertDeliveryEventsOptions { readonly offset?: number readonly incidentId?: AlertIncidentId readonly ruleId?: AlertRuleId + /** Leave out the internal PagerDuty `acknowledge` syncs, which the public API does not report. */ + readonly notificationsOnly?: boolean } export interface AlertReadModelsServiceApi { @@ -483,7 +485,7 @@ export class AlertReadModelsService extends Context.Service< options: ListAlertDeliveryEventsOptions = {}, ) { yield* Effect.annotateCurrentSpan("orgId", orgId) - const { incidentId, ruleId } = options + const { incidentId, ruleId, notificationsOnly } = options const rows = yield* dbExecute((db) => db.run( PG.from(AlertDeliveryEvents) @@ -492,6 +494,7 @@ export class AlertReadModelsService extends Context.Service< $.orgId.eq(orgId), incidentId === undefined ? undefined : $.incidentId.eq(incidentId), ruleId === undefined ? undefined : $.ruleId.eq(ruleId), + notificationsOnly === true ? $.eventType.neq("acknowledge") : undefined, ]) .orderBy(["createdAt", "desc"], ["id", "desc"]) .limit(options.limit ?? 100) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index bbb3abb1de..dec9a81b51 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -2360,13 +2360,26 @@ export class AlertsService extends Context.Service + db.run( + PG.from(AlertIncidents) + .select("id") + .where(($) => [ + $.orgId.eq(row.orgId), + $.id.eq(ackIncidentId), + $.status.eq("open"), + $.acknowledgedAt.isNotNull(), + ]) + .limit(1), + ), + )).length === 0) + if (isStaleAcknowledgement) { yield* finalizeClaimedDelivery(row.id, claimTime, { status: "success", attemptedAt: claimTime, diff --git a/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts b/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts index f3c55625cc..79061ec3fe 100644 --- a/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts +++ b/packages/backend/src/services/alerts/delivery/transports/pagerduty.ts @@ -54,17 +54,17 @@ const actionRequest = ( }) /** - * An acknowledgement covers every alert the incident may hold open: the current tier's key, and - * for a critical incident of a tiered rule also the warning key it opened under (a de-escalation - * already resolved the critical key, so a warning incident holds only one). PagerDuty ignores an - * acknowledge for a key it has no open alert for. There is no un-acknowledge in the Events API. + * An acknowledgement covers every alert the incident may hold open: its base key, and the critical + * key when the rule pages critical under its own key or the incident ever reached critical (the + * rule may have dropped that tier since). PagerDuty ignores an acknowledge for a key it has no + * open alert for. There is no un-acknowledge in the Events API. */ export const pagerDutyAcknowledgeKeys = ( - context: Pick, -): ReadonlyArray => { - const primary = pagerDutyDedupKey(context, keySeverity(context)) - return primary === context.dedupeKey ? [primary] : [primary, context.dedupeKey] -} + context: Pick, +): ReadonlyArray => + criticalHasOwnKey(context) || context.incidentPeakSeverity === "critical" + ? [context.dedupeKey, criticalDedupKey(context.dedupeKey)] + : [context.dedupeKey] /** * PagerDuty Events API v2. The body is machine-facing, so `custom_details` diff --git a/packages/backend/src/services/alerts/delivery/transports/render.test.ts b/packages/backend/src/services/alerts/delivery/transports/render.test.ts index 8776a0be99..ceb9c72261 100644 --- a/packages/backend/src/services/alerts/delivery/transports/render.test.ts +++ b/packages/backend/src/services/alerts/delivery/transports/render.test.ts @@ -442,23 +442,22 @@ describe("pagerduty acknowledge", () => { ]) }) - it("acknowledges both alerts a critical incident of a tiered rule may hold open", () => { - assert.deepStrictEqual( - Arr.map( - ack({ ...tiered, severity: "critical", incidentSeverity: "critical" }), - (body) => `${body.event_action} ${body.dedup_key}`, - ), - ["acknowledge org_1:rule_1:checkout:critical", "acknowledge org_1:rule_1:checkout"], - ) + it("acknowledges both keys an incident of a tiered rule may hold open, whatever its tier now", () => { + const keys = (overrides: Partial) => + Arr.map(ack(overrides), (body) => `${body.event_action} ${body.dedup_key}`) + const both = ["acknowledge org_1:rule_1:checkout", "acknowledge org_1:rule_1:checkout:critical"] + assert.deepStrictEqual(keys({ ...tiered, severity: "critical", incidentSeverity: "critical" }), both) + // A warning alert stays open beside a critical one; PagerDuty ignores a key with no open alert. + assert.deepStrictEqual(keys({ ...tiered, severity: "warning", incidentSeverity: "warning" }), both) }) - it("acknowledges only the warning key once the incident has fallen back to warning", () => { + it("acknowledges the critical key of an incident that peaked there, after the rule lost its tiers", () => { assert.deepStrictEqual( Arr.map( - ack({ ...tiered, severity: "critical", incidentSeverity: "warning" }), + ack({ ruleSeverities: ["critical"], incidentPeakSeverity: "critical" }), (body) => `${body.event_action} ${body.dedup_key}`, ), - ["acknowledge org_1:rule_1:checkout"], + ["acknowledge org_1:rule_1:checkout", "acknowledge org_1:rule_1:checkout:critical"], ) }) }) diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index 37c32ed11b..e3d03e572b 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -149,16 +149,32 @@ export const AlertIncidentEventType = Schema.Literals([ }) export type AlertIncidentEventType = Schema.Schema.Type -export const AlertEventType = Schema.Literals([ +/** The notifications the public API reports; `acknowledge` stays internal (see AlertEventType). */ +export const AlertNotificationEventType = Schema.Literals([ "trigger", "resolve", "renotify", "test", - "acknowledge", ]).annotate({ identifier: "@maple/AlertEventType", title: "Alert Event Type", }) +export type AlertNotificationEventType = Schema.Schema.Type + +/** + * Every delivery the queue sends. `acknowledge` (a PagerDuty sync) is kept out of the public API: + * shipped clients decode the event type as a closed enum and would fail on a new value. + */ +export const AlertEventType = Schema.Literals([ + ...AlertNotificationEventType.literals, + "acknowledge", +]).annotate({ + identifier: "@maple/AlertDeliveryEventType", + title: "Alert Delivery Event Type", +}) + +export const isAlertNotificationEventType = (type: AlertEventType): type is AlertNotificationEventType => + type !== "acknowledge" export type AlertEventType = Schema.Schema.Type export const AlertDeliveryStatus = Schema.Literals(["queued", "processing", "success", "failed"]).annotate({ diff --git a/packages/domain/src/http/v2/alert-deliveries.ts b/packages/domain/src/http/v2/alert-deliveries.ts index c6e0672890..9da4bf09ed 100644 --- a/packages/domain/src/http/v2/alert-deliveries.ts +++ b/packages/domain/src/http/v2/alert-deliveries.ts @@ -1,6 +1,11 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "effect/http-api" import { Schema } from "effect" -import { AlertDeliveryStatus, AlertDestinationType, AlertEventType, AlertPersistenceError } from "../alerts" +import { + AlertDeliveryStatus, + AlertDestinationType, + AlertNotificationEventType, + AlertPersistenceError, +} from "../alerts" import { AuthorizationV2 } from "./auth" import { ListOf, ListQuery, Timestamp } from "./envelopes" import { V2ParameterInvalid } from "./errors" @@ -22,7 +27,7 @@ export const V2AlertDelivery = Schema.Struct({ destination_name: Schema.String, destination_type: AlertDestinationType, delivery_key: Schema.String, - event_type: AlertEventType, + event_type: AlertNotificationEventType, attempt_number: Schema.Number.pipe(Schema.check(Schema.isInt(), Schema.isGreaterThan(0))), status: AlertDeliveryStatus, scheduled_at: Timestamp, diff --git a/packages/domain/src/http/v2/alert-incidents.ts b/packages/domain/src/http/v2/alert-incidents.ts index 6b73520577..7666b80273 100644 --- a/packages/domain/src/http/v2/alert-incidents.ts +++ b/packages/domain/src/http/v2/alert-incidents.ts @@ -3,7 +3,7 @@ import { Schema } from "effect" import { AlertIncidentNotFoundError, AlertIncidentResolvedError, AlertPersistenceError } from "../alerts" import { AlertComparator, - AlertEventType, + AlertNotificationEventType, AlertIncidentAckSource, AlertIncidentEventType, AlertIncidentHoldReason, @@ -120,7 +120,7 @@ export const V2AlertIncident = Schema.Struct({ description: "Stable identity of the (rule, group) incident stream, used to deduplicate notifications.", }), - last_delivered_event_type: Schema.NullOr(AlertEventType).annotate({ + last_delivered_event_type: Schema.NullOr(AlertNotificationEventType).annotate({ description: "The most recent notification event delivered for this incident (`trigger`, `resolve`, `renotify`, `test`), or `null`.", }), diff --git a/packages/web-data/src/alerts/incident-timeline.test.ts b/packages/web-data/src/alerts/incident-timeline.test.ts index 8f2cf8e01c..6469272efe 100644 --- a/packages/web-data/src/alerts/incident-timeline.test.ts +++ b/packages/web-data/src/alerts/incident-timeline.test.ts @@ -121,17 +121,6 @@ describe("buildIncidentTimeline", () => { expect(failed?.detail).toBe("discord · webhook returned 500") }) - it("names a PagerDuty acknowledgement as one, not as a notification", () => { - const [, sent] = build(makeIncident(), [ - makeDelivery({ - eventType: "acknowledge", - destinationName: "On-call", - destinationType: "pagerduty", - }), - ]) - expect(sent?.title).toBe("Sent acknowledgement to On-call") - }) - it("ends a held incident on the hold instead of a breach", () => { const entries = build(makeIncident({ holdReason: "no_data", heldSince: "2026-10-15T14:40:00.000Z" })) expect(entries.at(-1)?.title).toBe("Waiting on data") @@ -203,27 +192,6 @@ describe("buildIncidentTimeline", () => { expect(entries.at(-1)?.detail).toBe("Last value 8.4% · reminders every 30m while open") }) - it("leaves out an acknowledgement the dispatcher skipped", () => { - const sent = makeDelivery({ - eventType: "acknowledge", - destinationType: "pagerduty", - responseCode: 202, - }) - const skipped = makeDelivery({ - id: "00000000-0000-4000-8000-0000000000e9", - eventType: "acknowledge", - destinationType: "pagerduty", - responseCode: null, - providerMessage: "Skipped: the incident is no longer acknowledged", - }) - expect(Arr.map(build(makeIncident(), [sent]), (entry) => entry.title)).toContain( - "Sent acknowledgement to #alerts", - ) - expect(Arr.map(build(makeIncident(), [skipped]), (entry) => entry.title)).not.toContain( - "Sent acknowledgement to #alerts", - ) - }) - it("keeps the state still in effect last when an ack came after it", () => { const entries = build( makeIncident({ holdReason: "no_data", heldSince: "2026-10-15T14:40:00.000Z" }), diff --git a/packages/web-data/src/alerts/incident-timeline.ts b/packages/web-data/src/alerts/incident-timeline.ts index 25e41a5d9f..3285a52992 100644 --- a/packages/web-data/src/alerts/incident-timeline.ts +++ b/packages/web-data/src/alerts/incident-timeline.ts @@ -137,9 +137,6 @@ function deliveryEntry( typeLabel: (type: AlertDestinationType) => string, ): IncidentTimelineEntry | null { if (event.eventType === "test") return null - // The dispatcher skips an acknowledgement an escalation overtook: nothing reached PagerDuty. - if (event.eventType === "acknowledge" && event.status === "success" && event.responseCode === null) - return null const name = event.destinationName const failed = event.status === "failed" const sending = event.status === "queued" || event.status === "processing" @@ -153,13 +150,9 @@ function deliveryEntry( ? failed ? `Reminder to ${name} failed` : `Reminded ${name}` - : event.eventType === "acknowledge" - ? failed - ? `Acknowledgement to ${name} failed` - : `Sent acknowledgement to ${name}` - : failed - ? `Could not notify ${name}` - : `Notified ${name}` + : failed + ? `Could not notify ${name}` + : `Notified ${name}` const reason = failed ? (event.errorMessage ?? (event.responseCode != null ? `HTTP ${event.responseCode}` : null)) : null From d7ecc60353f2589ae5efa1265420046144f006a8 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:42:54 +0100 Subject: [PATCH 4/4] chore(alerts): drop an unused import --- packages/backend/src/services/alerts/AlertIncidentAckService.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/src/services/alerts/AlertIncidentAckService.ts b/packages/backend/src/services/alerts/AlertIncidentAckService.ts index 8eed1130ab..6d610ab6dd 100644 --- a/packages/backend/src/services/alerts/AlertIncidentAckService.ts +++ b/packages/backend/src/services/alerts/AlertIncidentAckService.ts @@ -5,7 +5,6 @@ */ import { AlertDeliveryEventId, - AlertDestinationId, AlertIncidentAckSource, AlertIncidentEventDocument, AlertIncidentEventId,