diff --git a/apps/web/src/page/alerts/editor/derive.ts b/apps/web/src/page/alerts/editor/derive.ts index 107115cdc4..2ffe94d7a4 100644 --- a/apps/web/src/page/alerts/editor/derive.ts +++ b/apps/web/src/page/alerts/editor/derive.ts @@ -125,17 +125,22 @@ export const validationIssues = (form: RuleForm, isDefault = false): ReadonlyArr return issues } +/** Whether an anomaly rule pages nobody: no destination and no escalation chain on either tier. */ +export const isAnomalyUnrouted = (form: RuleForm): boolean => + form.destinationIds.length === 0 && + form.escalationChainId === null && + (form.escalation?.destinationIds.length ?? 0) === 0 && + (form.escalation?.escalationChainId ?? null) === null + /** * An anomaly rule has no query or timing to get wrong: a name, a scope and someone to page. Only * Maple creates the unscoped default rule, which may stay unrouted (anomaly paging is opt-in); the - * server refuses any other unscoped or unrouted enabled anomaly rule. + * server refuses any other unscoped enabled anomaly rule with no destination or escalation chain. */ const anomalyIssues = (form: RuleForm, isDefault: boolean): ReadonlyArray => [ ...(Str.trim(form.name).length === 0 ? ["Rule name"] : []), ...(!isDefault && isDefaultAnomalyRule(form) ? ["Scope"] : []), - ...(!isDefault && form.destinationIds.length === 0 && (form.escalation?.destinationIds.length ?? 0) === 0 - ? ["Who gets notified"] - : []), + ...(!isDefault && isAnomalyUnrouted(form) ? ["Who gets notified"] : []), ] /** What blocks saving: the form's own issues and a query too slow for its interval. */ diff --git a/apps/web/src/page/alerts/editor/editor-chains.scene.test.ts b/apps/web/src/page/alerts/editor/editor-chains.scene.test.ts index 26d0b8a590..fbbb61e587 100644 --- a/apps/web/src/page/alerts/editor/editor-chains.scene.test.ts +++ b/apps/web/src/page/alerts/editor/editor-chains.scene.test.ts @@ -148,4 +148,33 @@ describe("alert rule editor, escalation chains", () => { expect(routed.form?.escalation?.escalationChainId).toBe(chainUuid(2)) expect(issuesOf(routed)).not.toContain("Critical destinations") }) + + test("a scoped anomaly rule routed only by a critical chain saves", () => { + const start = init({ _tag: "AlertsCreate", search: { template: "anomaly" } }, context) + const anomaly = run(start.model, [ + Message.GeneratedDraftId({ id: "6f1c2e3a-0000-4000-8000-000000000002" }), + Message.CompletedReadClock({ nowMs: NOW }), + Message.UpdatedDestinations({ orgId: ORG, rows: [destinationRow(), ...chainDestinations] }), + Message.UpdatedRules({ orgId: ORG, rows: [ruleRow(2)] }), + Message.CompletedFetchEscalationChains({ chains: CHAINS }), + Message.ChangedField({ field: "name", value: "Checkout anomalies" }), + Message.ToggledAnomalySignal({ signal: "error_rate" }), + ]) + scene( + { update: pageUpdate, view }, + given(anomaly), + themeResolved(), + expectView(role("button", { name: "Create rule" })).toBeDisabled(), + ) + const routed = run(anomaly, [pickChain("critical", chainUuid(1))]) + expect(routed.form?.destinationIds).toEqual([]) + expect(routed.form?.escalation?.destinationIds).toEqual([]) + scene( + { update: pageUpdate, view }, + given(routed), + themeResolved(), + expectView(role("button", { name: "Create rule" })).toBeEnabled(), + ) + expect(names(pageUpdate(routed, Message.ClickedSave()).commands)).toEqual(["SaveAlertRule"]) + }) }) diff --git a/apps/web/src/page/alerts/editor/view.ts b/apps/web/src/page/alerts/editor/view.ts index 421e039618..a604c356c6 100644 --- a/apps/web/src/page/alerts/editor/view.ts +++ b/apps/web/src/page/alerts/editor/view.ts @@ -16,7 +16,7 @@ import { RULE_CHART_TOKENS } from "../shared/rule-chart" import type { RuleForm } from "./form" import { Message } from "./message" import type { Model, Notice } from "./model" -import { isProtectedDefault } from "./derive" +import { isAnomalyUnrouted, isProtectedDefault } from "./derive" import { actionBar } from "./view-action-bar" import { conditionFields } from "./view-condition" import { detailsSection } from "./view-details" @@ -111,11 +111,7 @@ const evaluatedFormOf = stableSlice((form: RuleForm): RuleForm => ({ /** Who gets paged is missing: no destination or escalation chain on a tier that needs one. */ const isNotifyMissingOf = (form: RuleForm, isDefault: boolean): boolean => form.signalType === "anomaly" - ? !isDefault && - form.destinationIds.length === 0 && - form.escalationChainId === null && - (form.escalation?.destinationIds.length ?? 0) === 0 && - (form.escalation?.escalationChainId ?? null) === null + ? !isDefault && isAnomalyUnrouted(form) : (form.destinationIds.length === 0 && form.escalationChainId === null) || deriveTierIssues(form).includes("Critical destinations") diff --git a/packages/backend/src/services/alerts/AlertRuleModel.ts b/packages/backend/src/services/alerts/AlertRuleModel.ts index ced7c86b26..24895e1bd2 100644 --- a/packages/backend/src/services/alerts/AlertRuleModel.ts +++ b/packages/backend/src/services/alerts/AlertRuleModel.ts @@ -48,7 +48,7 @@ import { AlertRules, type AlertRuleRow } from "@maple/db/tables" import { DateTime, Array as Arr, Effect, Match, Result, Schema, String as Str } from "effect" import type { AlertRuntimeApi } from "./AlertRuntime" import type { QueryBuilderDataSource } from "@maple/query-model" -import { validateBreachWindow, validateThresholds } from "./alert-thresholds" +import { anyTierRouted, validateBreachWindow, validateThresholds } from "./alert-thresholds" import { ANOMALY_RULE_DEFAULTS, anomalySeverityCode, isDefaultAnomalyScope } from "./anomaly/anomaly-rule" const StringArraySchema = Schema.Array(Schema.String) @@ -987,16 +987,19 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => { requireDestinations: !options?.forPreview && !anomaly, }), ) - // Only the unscoped default may stay unrouted; a scoped rule exists to page someone. + // Only the unscoped default may stay unrouted; a scoped rule exists to page someone, + // through a destination or an escalation chain. const scoped = !isDefaultAnomalyScope({ serviceNames, environments, anomalySignals }) if ( anomaly && scoped && (request.enabled ?? true) && !options?.forPreview && - Arr.every(requestedThresholds, (tier) => tier.destinationIds.length === 0) + !anyTierRouted(requestedThresholds) ) { - details.push("A scoped anomaly rule needs a destination on at least one tier") + details.push( + "A scoped anomaly rule needs a destination or escalation chain on at least one tier", + ) } } const consecutiveBreachesRequired = anomaly diff --git a/packages/backend/src/services/alerts/AlertsService.anomaly.test.ts b/packages/backend/src/services/alerts/AlertsService.anomaly.test.ts index eb20792424..10130da404 100644 --- a/packages/backend/src/services/alerts/AlertsService.anomaly.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.anomaly.test.ts @@ -6,6 +6,7 @@ import { AlertRuleUpsertRequest, AnomalyIncidentId, ErrorIssueId, + EscalationChainCreateRequest, UNGROUPED_GROUP_KEY, type AlertDestinationId, type AlertThresholds, @@ -34,7 +35,10 @@ import { createWebhookDestination, makeLayer, makeWarehouseStub, + stubOrgMembersService, } from "./alerts-service-test-support" +import { EscalationChainsService } from "./EscalationChainsService" +import { OrgMembersService } from "@maple/backend/services/org/OrgMembersService" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) @@ -67,6 +71,10 @@ const harness = () => { const layer = Layer.mergeAll( makeLayer(testDb, makeWarehouseStub({}), { fetch: fetchImpl, chatSessions, pushes }), AnomalyAlertSource.layer.pipe(Layer.provide(testDb.layer)), + Layer.effect(EscalationChainsService, EscalationChainsService.make).pipe( + Layer.provide(testDb.layer), + Layer.provide(Layer.succeed(OrgMembersService, stubOrgMembersService())), + ), testDb.layer, ) return { testDb, bodies, turns, pushes, layer } @@ -293,6 +301,57 @@ describe("AlertsService anomaly rules", () => { }).pipe(Effect.provide(layer)) }) + it.effect("a critical tier routed only by an escalation chain starts it and pushes", () => { + const { testDb, pushes, layer } = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const chain = yield* EscalationChainsService.use((chains) => + chains.create( + orgId, + userId, + adminRoles, + new EscalationChainCreateRequest({ + name: "Checkout escalation", + steps: [ + { + targets: [{ type: "destination", destinationId: destination.id }], + waitMinutes: 10, + }, + ], + }), + ), + ) + yield* alerts.createRule( + orgId, + userId, + adminRoles, + anomalyRule({ + name: "Checkout anomalies", + serviceNames: ["checkout"], + thresholds: [ + { ...criticalTier(destination.id), destinationIds: [], escalationChainId: chain.id }, + ], + }), + ) + yield* insertAnomaly({ ...errorRateSpike, severity: "critical" }) + + yield* alerts.runSchedulerTick() + + expect((yield* alerts.listIncidents(orgId)).incidents[0]).toMatchObject({ severity: "critical" }) + const runs = yield* Effect.promise(() => + queryFirstRow<{ chain_name: string }>( + testDb, + "select chain_name from alert_escalation_runs where org_id = $1", + [orgId], + ), + ) + assert.strictEqual(runs?.chain_name, "Checkout escalation") + assert.deepStrictEqual(pushedEvents(pushes), ["trigger:critical"]) + }).pipe(Effect.provide(layer)) + }) + it.effect("follows the anomaly from warning to critical and resolves with it", () => { const { testDb, pushes, layer } = harness() return Effect.gen(function* () { diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 0b231a72b7..6caa54e848 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -146,7 +146,7 @@ import { import { ChatSessions } from "@maple/backend/platform/bindings" import { Database } from "@maple/backend/platform/DatabaseLive" import { formatComparator, signalCarriesDetail } from "./alert-formatting" -import { destinationsUpTo, escalationChainFor, ruleAtSeverity } from "./alert-thresholds" +import { escalationChainFor, ruleAtSeverity, tierRoutedUpTo } from "./alert-thresholds" import { applyEscalationPlan, enabledDestinationIds, @@ -1339,13 +1339,10 @@ export class AlertsService extends Context.Service, severity: AlertSeverity): boolean => + destinationsUpTo(thresholds, severity).length > 0 || escalationChainFor(thresholds, severity) !== null + +/** Whether any tier names a destination or an escalation chain. */ +export const anyTierRouted = (thresholds: ReadonlyArray): boolean => + Arr.some( + thresholds, + (tier) => + tier.destinationIds.length > 0 || + (tier.escalationChainId !== undefined && tier.escalationChainId !== null), + ) + interface TieredRuleFields { readonly thresholds: AlertThresholds readonly severity: AlertSeverity diff --git a/packages/backend/src/services/alerts/anomaly/anomaly-rules.test.ts b/packages/backend/src/services/alerts/anomaly/anomaly-rules.test.ts index 72e55e47a9..baded1d477 100644 --- a/packages/backend/src/services/alerts/anomaly/anomaly-rules.test.ts +++ b/packages/backend/src/services/alerts/anomaly/anomaly-rules.test.ts @@ -3,6 +3,7 @@ import { AlertDestinationId, AlertRuleId, AlertRuleUpsertRequest, + EscalationChainId, OrgId, RoleName, UserId, @@ -33,6 +34,7 @@ const ADMIN_ROLES = [asRoleName("org:admin")] const NOW = Date.parse("2026-11-06T12:00:00.000Z") const DEFAULT_RULE_ID = asRuleId("00000000-0000-4000-8000-0000000000aa") const DESTINATION_ID = Schema.decodeSync(AlertDestinationId)("00000000-0000-4000-8000-0000000000d1") +const CHAIN_ID = Schema.decodeSync(EscalationChainId)("00000000-0000-4000-8000-0000000000c1") const createdDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(createdDbs)) @@ -136,7 +138,10 @@ describe("anomaly rule normalization", () => { const scoped = anomalyRequest({ severity: "critical", serviceNames: ["checkout"] }) const refused = yield* Effect.flip(normalizeRule(ORG, scoped)) - assert.include(refused.message, "A scoped anomaly rule needs a destination on at least one tier") + assert.include( + refused.message, + "A scoped anomaly rule needs a destination or escalation chain on at least one tier", + ) const paused = yield* normalizeRule( ORG, new AlertRuleUpsertRequest({ ...scoped, enabled: false }), @@ -147,6 +152,33 @@ describe("anomaly rule normalization", () => { }), ) + it.effect("lets an escalation chain alone route a scoped anomaly rule", () => + Effect.gen(function* () { + const { normalizeRule } = makeAlertRuleNormalizer(yield* makeRuntime) + const rule = yield* normalizeRule( + ORG, + anomalyRequest({ + serviceNames: ["checkout"], + thresholds: [ + { + severity: "critical", + value: 0, + upper: null, + recoveryValue: null, + destinationIds: [], + escalationChainId: CHAIN_ID, + }, + ], + }), + ) + assert.isTrue(rule.enabled) + assert.deepStrictEqual( + Arr.map(rule.thresholds, (tier) => [tier.severity, tier.escalationChainId]), + [["critical", CHAIN_ID]], + ) + }), + ) + it.effect("rejects anomalySignals on any other signal type", () => Effect.gen(function* () { const { normalizeRule } = makeAlertRuleNormalizer(yield* makeRuntime) diff --git a/packages/web-data/src/alerts/diagnosis.test.ts b/packages/web-data/src/alerts/diagnosis.test.ts index c69397c4dc..ca929950df 100644 --- a/packages/web-data/src/alerts/diagnosis.test.ts +++ b/packages/web-data/src/alerts/diagnosis.test.ts @@ -262,6 +262,52 @@ describe("buildDiagnosis", () => { expect(notification.action?.kind).toBe("destinations") }) + it("judges a rule routed only by an escalation chain by its latest delivery", () => { + const chainTier = { + severity: "warning", + value: 0.05, + upper: null, + recoveryValue: null, + destinationIds: [], + escalationChainId: "00000000-0000-4000-8000-00000000cccc", + } + const chained = makeRule({ destinationIds: [], thresholds: [chainTier] }) + const event = (status: string) => + ({ + destinationId: DEST_ID, + status, + responseCode: 500, + errorMessage: "boom", + attemptedAt: iso(60_000), + }) as never + + const unsent = stage(diagnose({ rule: chained }), "notification") + expect(unsent.status).toBe("unknown") + expect(unsent.summary).toBe("Routed to an escalation chain; no destination deliveries yet") + + const failed = stage(diagnose({ rule: chained, deliveryEvents: [event("failed")] }), "notification") + expect(failed.status).toBe("fail") + expect(failed.evidence[0]).toContain("boom") + + const delivered = stage( + diagnose({ rule: chained, deliveryEvents: [event("success")] }), + "notification", + ) + expect(delivered.status).toBe("pass") + expect(delivered.summary).toBe("Routed to an escalation chain; notifications are delivering") + + Arr.forEach(["queued", "processing", "silenced"], (pending) => { + const notConfirmed = stage( + diagnose({ rule: chained, deliveryEvents: [event(pending)] }), + "notification", + ) + expect(notConfirmed.status).toBe("unknown") + expect(notConfirmed.summary).toBe( + `Routed to an escalation chain; the latest delivery is ${pending}`, + ) + }) + }) + it("surfaces per-destination delivery failures with the provider error", () => { const stages = diagnose({ destinations: [ @@ -329,4 +375,22 @@ describe("buildDiagnosis for anomaly rules", () => { stage(diagnose({ rule: anomalyRule({ serviceNames: ["checkout"] }) }), "notification").status, ).toBe("fail") }) + + it("a default rule routed by an escalation chain is not reported as opt-in", () => { + const routed = anomalyRule({ + thresholds: [ + { + severity: "critical", + value: 2, + upper: null, + recoveryValue: null, + destinationIds: [], + escalationChainId: "00000000-0000-4000-8000-00000000cccc", + }, + ], + }) + expect(stage(diagnose({ rule: routed }), "notification").summary).toBe( + "Routed to an escalation chain; no destination deliveries yet", + ) + }) }) diff --git a/packages/web-data/src/alerts/diagnosis.ts b/packages/web-data/src/alerts/diagnosis.ts index e60aa593a1..0c1b7e1dbe 100644 --- a/packages/web-data/src/alerts/diagnosis.ts +++ b/packages/web-data/src/alerts/diagnosis.ts @@ -17,7 +17,7 @@ import { countLabel } from "@maple/ui/lib/format" import type { AlertRuleStateRow } from "../collections/alerts" import { comparatorLabels, formatSignalValue } from "./form-utils" import { isDefaultAnomalyRule } from "./anomaly-rule" -import { staleThresholdMs } from "./rule-status" +import { isRuleRouted, staleThresholdMs } from "./rule-status" import { formatTimestampInTimezone } from "../lib/timezone-format" const evaluationCadence = (rule: AlertRuleDocument): string => { @@ -454,7 +454,7 @@ export function buildDiagnosis(input: DiagnosisInput): DiagnosisStage[] { } /* 7 — Notification delivered */ - if (rule.destinationIds.length === 0) { + if (!isRuleRouted(rule)) { stages.push({ id: "notification", label: "Notification delivered", @@ -463,6 +463,38 @@ export function buildDiagnosis(input: DiagnosisInput): DiagnosisStage[] { evidence: ["Add a Slack/PagerDuty/webhook destination so incidents actually reach someone."], action: { label: "Configure destinations", kind: "destinations" }, }) + } else if (rule.destinationIds.length === 0) { + // Chain-only: judge by the rule's latest destination delivery. Member and schedule pages + // are recorded elsewhere, so no event here does not mean nobody was paged. + const latest = deliveryEvents[0] ?? null + stages.push({ + id: "notification", + label: "Notification delivered", + // Only a success proves delivery; queued, processing or silenced stay unknown. + status: + latest === null || (latest.status !== "failed" && latest.status !== "success") + ? "unknown" + : latest.status === "failed" + ? "fail" + : "pass", + summary: + latest === null + ? "Routed to an escalation chain; no destination deliveries yet" + : latest.status === "failed" + ? "Routed to an escalation chain; the latest delivery failed" + : latest.status === "success" + ? "Routed to an escalation chain; notifications are delivering" + : `Routed to an escalation chain; the latest delivery is ${latest.status}`, + evidence: + latest?.status === "failed" + ? [ + `Last delivery failed${latest.responseCode != null ? ` (HTTP ${latest.responseCode})` : ""}${latest.errorMessage != null ? `: ${latest.errorMessage}` : ""}`, + ] + : [], + ...(latest?.status === "failed" + ? { action: { label: "Review destinations", kind: "destinations" as const } } + : undefined), + }) } else { const destinationById = new Map(Arr.map(destinations, (d) => [d.id as string, d])) const evidence: string[] = [] @@ -528,7 +560,7 @@ const anomalyStages = ( if (stage.id === "query") return [{ ...stage, label: "Detector read" }] if (stage.id === "incident" && stage.status === "pass") return [{ ...stage, summary: "No open anomaly in this rule's scope" }] - if (stage.id === "notification" && rule.destinationIds.length === 0 && isDefaultAnomalyRule(rule)) + if (stage.id === "notification" && !isRuleRouted(rule) && isDefaultAnomalyRule(rule)) return [ { id: "notification", diff --git a/packages/web-data/src/alerts/rule-status.test.ts b/packages/web-data/src/alerts/rule-status.test.ts index af7db636ee..e64d4f49ae 100644 --- a/packages/web-data/src/alerts/rule-status.test.ts +++ b/packages/web-data/src/alerts/rule-status.test.ts @@ -246,6 +246,25 @@ describe("deriveRuleStatus", () => { expect(result.attention.recentDeliveryFailure).toBe(true) expect(needsAttention(result)).toBe(true) }) + + it("counts an escalation chain alone as routing the rule", () => { + const tier = { + severity: "critical", + value: 0.1, + upper: null, + recoveryValue: null, + destinationIds: [], + } + const chained = derive({ + rule: makeRule({ + destinationIds: [], + thresholds: [{ ...tier, escalationChainId: "00000000-0000-4000-8000-00000000cccc" }], + }), + }) + expect(chained.attention.noDestinations).toBe(false) + const unrouted = derive({ rule: makeRule({ destinationIds: [], thresholds: [tier] }) }) + expect(unrouted.attention.noDestinations).toBe(true) + }) }) describe("worstState", () => { diff --git a/packages/web-data/src/alerts/rule-status.ts b/packages/web-data/src/alerts/rule-status.ts index 7b7113da0a..71fb93efdd 100644 --- a/packages/web-data/src/alerts/rule-status.ts +++ b/packages/web-data/src/alerts/rule-status.ts @@ -76,6 +76,14 @@ export function worstState(states: ReadonlyArray): AlertRuleS return best } +/** A destination or an escalation chain on any tier routes the rule. */ +export const isRuleRouted = (rule: AlertRuleDocument): boolean => + rule.destinationIds.length > 0 || + Arr.some( + rule.thresholds, + (tier) => tier.destinationIds.length > 0 || (tier.escalationChainId ?? null) !== null, + ) + /** * Derive the display status for a rule from everything the client knows about * it: the rule document, its live-synced per-group states (may be empty on @@ -94,8 +102,9 @@ export function deriveRuleStatus(input: { const latestDelivery = deliveryEvents[0] ?? null const attention: RuleAttention = { - // Anomaly paging is opt-in: the default anomaly rule starts unrouted and that is normal. - noDestinations: rule.enabled && rule.destinationIds.length === 0 && !isDefaultAnomalyRule(rule), + // A destination or an escalation chain on any tier routes the rule. Anomaly paging is + // opt-in: the default anomaly rule starts unrouted and that is normal. + noDestinations: rule.enabled && !isRuleRouted(rule) && !isDefaultAnomalyRule(rule), recentDeliveryFailure: latestDelivery?.status === "failed", slowedDown: rule.enabled && rule.slowedIntervalMinutes !== null, }