Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions apps/web/src/page/alerts/editor/derive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,17 +125,22 @@ export const validationIssues = (form: RuleForm, isDefault = false): ReadonlyArr
return issues
}

/** Whether an anomaly rule pages nobody: no destination and no escalation chain on either tier. */
export const isAnomalyUnrouted = (form: RuleForm): boolean =>
form.destinationIds.length === 0 &&
form.escalationChainId === null &&
(form.escalation?.destinationIds.length ?? 0) === 0 &&
(form.escalation?.escalationChainId ?? null) === null

/**
* An anomaly rule has no query or timing to get wrong: a name, a scope and someone to page. Only
* Maple creates the unscoped default rule, which may stay unrouted (anomaly paging is opt-in); the
* server refuses any other unscoped or unrouted enabled anomaly rule.
* server refuses any other unscoped enabled anomaly rule with no destination or escalation chain.
*/
const anomalyIssues = (form: RuleForm, isDefault: boolean): ReadonlyArray<string> => [
...(Str.trim(form.name).length === 0 ? ["Rule name"] : []),
...(!isDefault && isDefaultAnomalyRule(form) ? ["Scope"] : []),
...(!isDefault && form.destinationIds.length === 0 && (form.escalation?.destinationIds.length ?? 0) === 0
? ["Who gets notified"]
: []),
...(!isDefault && isAnomalyUnrouted(form) ? ["Who gets notified"] : []),
]

/** What blocks saving: the form's own issues and a query too slow for its interval. */
Expand Down
29 changes: 29 additions & 0 deletions apps/web/src/page/alerts/editor/editor-chains.scene.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,33 @@ describe("alert rule editor, escalation chains", () => {
expect(routed.form?.escalation?.escalationChainId).toBe(chainUuid(2))
expect(issuesOf(routed)).not.toContain("Critical destinations")
})

test("a scoped anomaly rule routed only by a critical chain saves", () => {
const start = init({ _tag: "AlertsCreate", search: { template: "anomaly" } }, context)
const anomaly = run(start.model, [
Message.GeneratedDraftId({ id: "6f1c2e3a-0000-4000-8000-000000000002" }),
Message.CompletedReadClock({ nowMs: NOW }),
Message.UpdatedDestinations({ orgId: ORG, rows: [destinationRow(), ...chainDestinations] }),
Message.UpdatedRules({ orgId: ORG, rows: [ruleRow(2)] }),
Message.CompletedFetchEscalationChains({ chains: CHAINS }),
Message.ChangedField({ field: "name", value: "Checkout anomalies" }),
Message.ToggledAnomalySignal({ signal: "error_rate" }),
])
scene(
{ update: pageUpdate, view },
given(anomaly),
themeResolved(),
expectView(role("button", { name: "Create rule" })).toBeDisabled(),
)
const routed = run(anomaly, [pickChain("critical", chainUuid(1))])
expect(routed.form?.destinationIds).toEqual([])
expect(routed.form?.escalation?.destinationIds).toEqual([])
scene(
{ update: pageUpdate, view },
given(routed),
themeResolved(),
expectView(role("button", { name: "Create rule" })).toBeEnabled(),
)
expect(names(pageUpdate(routed, Message.ClickedSave()).commands)).toEqual(["SaveAlertRule"])
})
})
8 changes: 2 additions & 6 deletions apps/web/src/page/alerts/editor/view.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { RULE_CHART_TOKENS } from "../shared/rule-chart"
import type { RuleForm } from "./form"
import { Message } from "./message"
import type { Model, Notice } from "./model"
import { isProtectedDefault } from "./derive"
import { isAnomalyUnrouted, isProtectedDefault } from "./derive"
import { actionBar } from "./view-action-bar"
import { conditionFields } from "./view-condition"
import { detailsSection } from "./view-details"
Expand Down Expand Up @@ -111,11 +111,7 @@ const evaluatedFormOf = stableSlice((form: RuleForm): RuleForm => ({
/** Who gets paged is missing: no destination or escalation chain on a tier that needs one. */
const isNotifyMissingOf = (form: RuleForm, isDefault: boolean): boolean =>
form.signalType === "anomaly"
? !isDefault &&
form.destinationIds.length === 0 &&
form.escalationChainId === null &&
(form.escalation?.destinationIds.length ?? 0) === 0 &&
(form.escalation?.escalationChainId ?? null) === null
? !isDefault && isAnomalyUnrouted(form)
: (form.destinationIds.length === 0 && form.escalationChainId === null) ||
deriveTierIssues(form).includes("Critical destinations")

Expand Down
11 changes: 7 additions & 4 deletions packages/backend/src/services/alerts/AlertRuleModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ import { AlertRules, type AlertRuleRow } from "@maple/db/tables"
import { DateTime, Array as Arr, Effect, Match, Result, Schema, String as Str } from "effect"
import type { AlertRuntimeApi } from "./AlertRuntime"
import type { QueryBuilderDataSource } from "@maple/query-model"
import { validateBreachWindow, validateThresholds } from "./alert-thresholds"
import { anyTierRouted, validateBreachWindow, validateThresholds } from "./alert-thresholds"
import { ANOMALY_RULE_DEFAULTS, anomalySeverityCode, isDefaultAnomalyScope } from "./anomaly/anomaly-rule"

const StringArraySchema = Schema.Array(Schema.String)
Expand Down Expand Up @@ -987,16 +987,19 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => {
requireDestinations: !options?.forPreview && !anomaly,
}),
)
// Only the unscoped default may stay unrouted; a scoped rule exists to page someone.
// Only the unscoped default may stay unrouted; a scoped rule exists to page someone,
// through a destination or an escalation chain.
const scoped = !isDefaultAnomalyScope({ serviceNames, environments, anomalySignals })
if (
anomaly &&
scoped &&
(request.enabled ?? true) &&
!options?.forPreview &&
Arr.every(requestedThresholds, (tier) => tier.destinationIds.length === 0)
!anyTierRouted(requestedThresholds)
) {
details.push("A scoped anomaly rule needs a destination on at least one tier")
details.push(
"A scoped anomaly rule needs a destination or escalation chain on at least one tier",
)
}
}
const consecutiveBreachesRequired = anomaly
Expand Down
59 changes: 59 additions & 0 deletions packages/backend/src/services/alerts/AlertsService.anomaly.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
AlertRuleUpsertRequest,
AnomalyIncidentId,
ErrorIssueId,
EscalationChainCreateRequest,
UNGROUPED_GROUP_KEY,
type AlertDestinationId,
type AlertThresholds,
Expand Down Expand Up @@ -34,7 +35,10 @@ import {
createWebhookDestination,
makeLayer,
makeWarehouseStub,
stubOrgMembersService,
} from "./alerts-service-test-support"
import { EscalationChainsService } from "./EscalationChainsService"
import { OrgMembersService } from "@maple/backend/services/org/OrgMembersService"

const trackedDbs: TestDb[] = []
afterEach(() => cleanupTestDbs(trackedDbs))
Expand Down Expand Up @@ -67,6 +71,10 @@ const harness = () => {
const layer = Layer.mergeAll(
makeLayer(testDb, makeWarehouseStub({}), { fetch: fetchImpl, chatSessions, pushes }),
AnomalyAlertSource.layer.pipe(Layer.provide(testDb.layer)),
Layer.effect(EscalationChainsService, EscalationChainsService.make).pipe(
Layer.provide(testDb.layer),
Layer.provide(Layer.succeed(OrgMembersService, stubOrgMembersService())),
),
testDb.layer,
)
return { testDb, bodies, turns, pushes, layer }
Expand Down Expand Up @@ -293,6 +301,57 @@ describe("AlertsService anomaly rules", () => {
}).pipe(Effect.provide(layer))
})

it.effect("a critical tier routed only by an escalation chain starts it and pushes", () => {
const { testDb, pushes, layer } = harness()
return Effect.gen(function* () {
yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS)
const alerts = yield* AlertsService
const destination = yield* createWebhookDestination(alerts, orgId, userId)
const chain = yield* EscalationChainsService.use((chains) =>
chains.create(
orgId,
userId,
adminRoles,
new EscalationChainCreateRequest({
name: "Checkout escalation",
steps: [
{
targets: [{ type: "destination", destinationId: destination.id }],
waitMinutes: 10,
},
],
}),
),
)
yield* alerts.createRule(
orgId,
userId,
adminRoles,
anomalyRule({
name: "Checkout anomalies",
serviceNames: ["checkout"],
thresholds: [
{ ...criticalTier(destination.id), destinationIds: [], escalationChainId: chain.id },
],
}),
)
yield* insertAnomaly({ ...errorRateSpike, severity: "critical" })

yield* alerts.runSchedulerTick()

expect((yield* alerts.listIncidents(orgId)).incidents[0]).toMatchObject({ severity: "critical" })
const runs = yield* Effect.promise(() =>
queryFirstRow<{ chain_name: string }>(
testDb,
"select chain_name from alert_escalation_runs where org_id = $1",
[orgId],
),
)
assert.strictEqual(runs?.chain_name, "Checkout escalation")
assert.deepStrictEqual(pushedEvents(pushes), ["trigger:critical"])
}).pipe(Effect.provide(layer))
})

it.effect("follows the anomaly from warning to critical and resolves with it", () => {
const { testDb, pushes, layer } = harness()
return Effect.gen(function* () {
Expand Down
11 changes: 4 additions & 7 deletions packages/backend/src/services/alerts/AlertsService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ import {
import { ChatSessions } from "@maple/backend/platform/bindings"
import { Database } from "@maple/backend/platform/DatabaseLive"
import { formatComparator, signalCarriesDetail } from "./alert-formatting"
import { destinationsUpTo, escalationChainFor, ruleAtSeverity } from "./alert-thresholds"
import { escalationChainFor, ruleAtSeverity, tierRoutedUpTo } from "./alert-thresholds"
import {
applyEscalationPlan,
enabledDestinationIds,
Expand Down Expand Up @@ -1339,13 +1339,10 @@ export class AlertsService extends Context.Service<AlertsService, AlertsServiceA
// breach stopped while telemetry was reduced, not "back to X".
resolvedAfterHold: ResolvedAfterHold | null,
) {
// An anomaly tier with no destinations shows in the app only, so it never buzzes a phone.
// Every push path (trigger, escalate, reminder, resolve, owed page) comes through here.
// An anomaly tier with no destination and no escalation chain shows in the app only, so it
// never buzzes a phone. Every push path (trigger, escalate, reminder, resolve, owed page) comes through here.
const incidentSeverity = decodeAlertSeveritySync(incident.severity)
if (
rule.signalType === "anomaly" &&
destinationsUpTo(rule.thresholds, incidentSeverity).length === 0
) {
if (rule.signalType === "anomaly" && !tierRoutedUpTo(rule.thresholds, incidentSeverity)) {
return
}
const linkUrl = resolveNotificationLinkUrl(rule, incident.groupKey, incident.id)
Expand Down
13 changes: 13 additions & 0 deletions packages/backend/src/services/alerts/alert-thresholds.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,19 @@ export const escalationChainFor = (
})
}

/** Whether an incident at `severity` reaches anyone: a destination or an escalation chain. */
export const tierRoutedUpTo = (thresholds: ReadonlyArray<AlertThreshold>, severity: AlertSeverity): boolean =>
destinationsUpTo(thresholds, severity).length > 0 || escalationChainFor(thresholds, severity) !== null

/** Whether any tier names a destination or an escalation chain. */
export const anyTierRouted = (thresholds: ReadonlyArray<AlertThreshold>): boolean =>
Arr.some(
thresholds,
(tier) =>
tier.destinationIds.length > 0 ||
(tier.escalationChainId !== undefined && tier.escalationChainId !== null),
)

interface TieredRuleFields {
readonly thresholds: AlertThresholds
readonly severity: AlertSeverity
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
AlertDestinationId,
AlertRuleId,
AlertRuleUpsertRequest,
EscalationChainId,
OrgId,
RoleName,
UserId,
Expand Down Expand Up @@ -33,6 +34,7 @@ const ADMIN_ROLES = [asRoleName("org:admin")]
const NOW = Date.parse("2026-11-06T12:00:00.000Z")
const DEFAULT_RULE_ID = asRuleId("00000000-0000-4000-8000-0000000000aa")
const DESTINATION_ID = Schema.decodeSync(AlertDestinationId)("00000000-0000-4000-8000-0000000000d1")
const CHAIN_ID = Schema.decodeSync(EscalationChainId)("00000000-0000-4000-8000-0000000000c1")

const createdDbs: TestDb[] = []
afterEach(() => cleanupTestDbs(createdDbs))
Expand Down Expand Up @@ -136,7 +138,10 @@ describe("anomaly rule normalization", () => {

const scoped = anomalyRequest({ severity: "critical", serviceNames: ["checkout"] })
const refused = yield* Effect.flip(normalizeRule(ORG, scoped))
assert.include(refused.message, "A scoped anomaly rule needs a destination on at least one tier")
assert.include(
refused.message,
"A scoped anomaly rule needs a destination or escalation chain on at least one tier",
)
const paused = yield* normalizeRule(
ORG,
new AlertRuleUpsertRequest({ ...scoped, enabled: false }),
Expand All @@ -147,6 +152,33 @@ describe("anomaly rule normalization", () => {
}),
)

it.effect("lets an escalation chain alone route a scoped anomaly rule", () =>
Effect.gen(function* () {
const { normalizeRule } = makeAlertRuleNormalizer(yield* makeRuntime)
const rule = yield* normalizeRule(
ORG,
anomalyRequest({
serviceNames: ["checkout"],
thresholds: [
{
severity: "critical",
value: 0,
upper: null,
recoveryValue: null,
destinationIds: [],
escalationChainId: CHAIN_ID,
},
],
}),
)
assert.isTrue(rule.enabled)
assert.deepStrictEqual(
Arr.map(rule.thresholds, (tier) => [tier.severity, tier.escalationChainId]),
[["critical", CHAIN_ID]],
)
}),
)

it.effect("rejects anomalySignals on any other signal type", () =>
Effect.gen(function* () {
const { normalizeRule } = makeAlertRuleNormalizer(yield* makeRuntime)
Expand Down
64 changes: 64 additions & 0 deletions packages/web-data/src/alerts/diagnosis.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,52 @@ describe("buildDiagnosis", () => {
expect(notification.action?.kind).toBe("destinations")
})

it("judges a rule routed only by an escalation chain by its latest delivery", () => {
const chainTier = {
severity: "warning",
value: 0.05,
upper: null,
recoveryValue: null,
destinationIds: [],
escalationChainId: "00000000-0000-4000-8000-00000000cccc",
}
const chained = makeRule({ destinationIds: [], thresholds: [chainTier] })
const event = (status: string) =>
({
destinationId: DEST_ID,
status,
responseCode: 500,
errorMessage: "boom",
attemptedAt: iso(60_000),
}) as never

const unsent = stage(diagnose({ rule: chained }), "notification")
expect(unsent.status).toBe("unknown")
expect(unsent.summary).toBe("Routed to an escalation chain; no destination deliveries yet")

const failed = stage(diagnose({ rule: chained, deliveryEvents: [event("failed")] }), "notification")
expect(failed.status).toBe("fail")
expect(failed.evidence[0]).toContain("boom")

const delivered = stage(
diagnose({ rule: chained, deliveryEvents: [event("success")] }),
"notification",
)
expect(delivered.status).toBe("pass")
expect(delivered.summary).toBe("Routed to an escalation chain; notifications are delivering")

Arr.forEach(["queued", "processing", "silenced"], (pending) => {
const notConfirmed = stage(
diagnose({ rule: chained, deliveryEvents: [event(pending)] }),
"notification",
)
expect(notConfirmed.status).toBe("unknown")
expect(notConfirmed.summary).toBe(
`Routed to an escalation chain; the latest delivery is ${pending}`,
)
})
})

it("surfaces per-destination delivery failures with the provider error", () => {
const stages = diagnose({
destinations: [
Expand Down Expand Up @@ -329,4 +375,22 @@ describe("buildDiagnosis for anomaly rules", () => {
stage(diagnose({ rule: anomalyRule({ serviceNames: ["checkout"] }) }), "notification").status,
).toBe("fail")
})

it("a default rule routed by an escalation chain is not reported as opt-in", () => {
const routed = anomalyRule({
thresholds: [
{
severity: "critical",
value: 2,
upper: null,
recoveryValue: null,
destinationIds: [],
escalationChainId: "00000000-0000-4000-8000-00000000cccc",
},
],
})
expect(stage(diagnose({ rule: routed }), "notification").summary).toBe(
"Routed to an escalation chain; no destination deliveries yet",
)
})
})
Loading
Loading