From 74ff30e6fd828f38cdc7525254a435de22d450a2 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:09:07 +0100 Subject: [PATCH 01/11] wip(slos): burn-rate alert rule per SLO --- .../v2/alchemy-provider.integration.test.ts | 4 + apps/api/src/routes/v2/alerts.http.test.ts | 4 + apps/api/src/routes/v2/slos.http.ts | 34 ++- .../src/services/alerts/AlertRuleModel.ts | 33 ++- .../src/services/alerts/AlertRulesService.ts | 16 +- .../src/services/alerts/AlertsService.ts | 26 +- .../alerts/alerts-service-test-support.ts | 6 + .../src/services/slos/SloAlertSource.ts | 76 +++++ .../backend/src/services/slos/SlosService.ts | 268 +++++++++++++++--- .../src/services/slos/slo-alert-rule.ts | 128 +++++++++ packages/domain/src/http/alerts.ts | 8 + packages/domain/src/http/v2/slos.ts | 37 ++- 12 files changed, 570 insertions(+), 70 deletions(-) create mode 100644 packages/backend/src/services/slos/SloAlertSource.ts create mode 100644 packages/backend/src/services/slos/slo-alert-rule.ts diff --git a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts index 3ecd966a73..caac85a016 100644 --- a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts +++ b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts @@ -44,6 +44,7 @@ import { DashboardPersistenceService } from "@maple/backend/services/dashboards/ import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService" import { AlertRuntime, AlertsService } from "@maple/backend/services/alerts/AlertsService" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" +import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource" import { AlertDestinationsService } from "@maple/backend/services/alerts/AlertDestinationsService" import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService" import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService" @@ -186,6 +187,9 @@ const makeHarness = () => { alertReadModelsLive, alertRulesLive, UptimeAlertSource.layer.pipe(Layer.provide(testDb.layer)), + Layer.effect(SloAlertSource, SloAlertSource.make).pipe( + Layer.provide(Layer.mergeAll(testDb.layer, warehouseLive)), + ), ), ), ) diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index 0c49db26a0..178cb2729c 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -41,6 +41,7 @@ import { DashboardPersistenceService } from "@maple/backend/services/dashboards/ import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService" import { AlertRuntime, AlertsService } from "@maple/backend/services/alerts/AlertsService" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" +import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource" import { AlertDestinationsService } from "@maple/backend/services/alerts/AlertDestinationsService" import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService" import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService" @@ -190,6 +191,9 @@ const makeHarness = ( alertReadModelsLive, alertRulesLive, UptimeAlertSource.layer.pipe(Layer.provide(testDb.layer)), + Layer.effect(SloAlertSource, SloAlertSource.make).pipe( + Layer.provide(Layer.mergeAll(testDb.layer, warehouseLive)), + ), ), ), ) diff --git a/apps/api/src/routes/v2/slos.http.ts b/apps/api/src/routes/v2/slos.http.ts index 1dd770efae..1b4550cf08 100644 --- a/apps/api/src/routes/v2/slos.http.ts +++ b/apps/api/src/routes/v2/slos.http.ts @@ -8,7 +8,7 @@ import { sloSliToWire, V2InsufficientPermissions, } from "@maple/domain/http/v2" -import type { V2Slo } from "@maple/domain/http/v2" +import type { V2Slo, V2SloAlertingParams } from "@maple/domain/http/v2" import { Effect, Array as Arr } from "effect" import { auditDiff } from "@/routes/v2/audit-changes" import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService" @@ -28,14 +28,44 @@ export const toV2Slo = (slo: SloRecord): V2Slo => ({ target_percent: slo.targetPercent, window_days: slo.windowDays, alert_rule_id: slo.alertRuleId, + alerting: { + enabled: slo.alerting.enabled, + critical_destination_ids: slo.alerting.criticalDestinationIds, + warning_destination_ids: slo.alerting.warningDestinationIds, + }, created_at: isoTimestamp(slo.createdAt), updated_at: isoTimestamp(slo.updatedAt), }) const sloAuditDiff = auditDiff({ - fields: ["name", "description", "service_name", "environments", "sli", "target_percent", "window_days"], + fields: [ + "name", + "description", + "service_name", + "environments", + "sli", + "target_percent", + "window_days", + "alerting", + ], }) +/** Wire alerting params to the service's, keeping only the fields the caller sent. */ +const alertingFromWire = (alerting: V2SloAlertingParams | undefined) => + alerting === undefined + ? undefined + : { + alerting: { + ...(alerting.enabled === undefined ? undefined : { enabled: alerting.enabled }), + ...(alerting.critical_destination_ids === undefined + ? undefined + : { criticalDestinationIds: alerting.critical_destination_ids }), + ...(alerting.warning_destination_ids === undefined + ? undefined + : { warningDestinationIds: alerting.warning_destination_ids }), + }, + } + const requireManage = Effect.gen(function* () { const tenant = yield* CurrentTenant.Context yield* Policy.check(tenant.roles, "alerts:manage", (denial) => diff --git a/packages/backend/src/services/alerts/AlertRuleModel.ts b/packages/backend/src/services/alerts/AlertRuleModel.ts index 94923a0bc4..cb5d721581 100644 --- a/packages/backend/src/services/alerts/AlertRuleModel.ts +++ b/packages/backend/src/services/alerts/AlertRuleModel.ts @@ -39,6 +39,8 @@ import { type AlertThresholds, type QueryBuilderQueryDraftPayload, type OrgId, + isManagedSignalType, + type ManagedSignalType, } from "@maple/domain/http" import * as PG from "@maple-dev/effect-orm/postgres" import { AlertRules, type AlertRuleRow } from "@maple/db/tables" @@ -229,12 +231,17 @@ const planGroupingTokens = ( return groupBy } -/** Why the generic rule APIs refuse an `uptime` rule. */ -export const UPTIME_RULE_MANAGED_MESSAGE = - "uptime rules belong to an uptime monitor; create, edit or delete the monitor instead" +/** Why the generic rule APIs refuse a rule another resource owns. */ +export const managedRuleMessage = (signalType: ManagedSignalType): string => + signalType === "uptime" + ? "uptime rules belong to an uptime monitor; create, edit or delete the monitor instead" + : "slo_burn_rate rules belong to an SLO; create, edit or delete the SLO instead" -/** The stored plan of every `uptime` rule: no query, the evaluator reads the monitor's state. */ -export const UPTIME_RULE_PLAN = new CompiledAlertQueryPlan({ +/** + * The stored plan of every managed rule (`uptime`, `slo_burn_rate`): no query, the evaluator + * reads the owning resource instead. + */ +export const MANAGED_RULE_PLAN = new CompiledAlertQueryPlan({ kind: "spec", query: null, rawSql: null, @@ -292,9 +299,9 @@ export const uptimeRuleFields = (monitor: { rawQuerySql: null, destinationIdsJson: [...monitor.destinationIds], querySpecJson: null, - reducer: UPTIME_RULE_PLAN.reducer, + reducer: MANAGED_RULE_PLAN.reducer, sampleCountStrategy: null, - noDataBehavior: UPTIME_RULE_PLAN.noDataBehavior, + noDataBehavior: MANAGED_RULE_PLAN.noDataBehavior, } as const } @@ -539,8 +546,8 @@ export const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(functi sampleCountStrategy: null, noDataBehavior, }) - } else if (rule.signalType === "uptime") { - return yield* makeAlertValidationError(UPTIME_RULE_MANAGED_MESSAGE) + } else if (isManagedSignalType(rule.signalType)) { + return yield* makeAlertValidationError(managedRuleMessage(rule.signalType)) } else { return yield* makeAlertValidationError(`Unsupported signal type: ${rule.signalType}`) } @@ -574,8 +581,8 @@ const parseCompiledPlan = ( component: "compiled_plan", cause, }) - // An uptime rule reads its monitor's state, not the warehouse: no query to decode. - if (row.signalType === "uptime") return Effect.succeed(UPTIME_RULE_PLAN) + // A managed rule reads its owner (a monitor's state, an SLO's burn), never a stored query. + if (isManagedSignalType(row.signalType)) return Effect.succeed(MANAGED_RULE_PLAN) if (row.signalType === "raw_query") { if (row.rawQuerySql == null) { return Effect.fail(invalid("Stored raw alert is missing its SQL query", "rawQuerySql is null")) @@ -868,8 +875,8 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => { request: AlertRuleUpsertRequest, options?: { readonly forPreview?: boolean }, ): Effect.fn.Return { - if (request.signalType === "uptime") { - return yield* invalidRuleError([UPTIME_RULE_MANAGED_MESSAGE]) + if (isManagedSignalType(request.signalType)) { + return yield* invalidRuleError([managedRuleMessage(request.signalType)]) } const name = Str.trim(request.name) // A builder query's draft carries its own scope and never reads the diff --git a/packages/backend/src/services/alerts/AlertRulesService.ts b/packages/backend/src/services/alerts/AlertRulesService.ts index 66009f1b59..e694d52b93 100644 --- a/packages/backend/src/services/alerts/AlertRulesService.ts +++ b/packages/backend/src/services/alerts/AlertRulesService.ts @@ -15,6 +15,8 @@ import { type AlertRuleUpsertRequest, type OrgId, type UserId, + isManagedSignalType, + MANAGED_SIGNAL_TYPES, } from "@maple/domain/http" import * as Orm from "@maple-dev/effect-orm/database" import * as PG from "@maple-dev/effect-orm/postgres" @@ -39,7 +41,7 @@ import { rowToRuleDocument, selectStoredAlertRules, storedRuleCondition, - UPTIME_RULE_MANAGED_MESSAGE, + managedRuleMessage, type RuleEvaluationState, } from "./AlertRuleModel" import { tierConditionChanged } from "./alert-thresholds" @@ -252,11 +254,11 @@ export const makeAlertRulePersistence = (options: { const activeRows = yield* db.run( PG.from(AlertRules) .select("id") - // Monitors' backing rules are capped by the monitor allowance instead. + // Managed rules (monitors, SLOs) are not counted against the rule cap. .where(($) => [ $.orgId.eq(orgId), $.enabled.eq(true), - $.signalType.neq("uptime"), + $.signalType.notIn(...MANAGED_SIGNAL_TYPES), ]), ) const alreadyActive = @@ -364,7 +366,7 @@ export const makeAlertRulePersistence = (options: { db.run( // Monitors' backing rules are managed through the monitor, so the rule list hides them. selectStoredAlertRules() - .where(($) => [$.orgId.eq(orgId), $.signalType.neq("uptime")]) + .where(($) => [$.orgId.eq(orgId), $.signalType.notIn(...MANAGED_SIGNAL_TYPES)]) .orderBy(["createdAt", "desc"], ["id", "desc"]), ), ) @@ -414,9 +416,9 @@ export const makeAlertRulePersistence = (options: { ) { yield* requireAlertsPermission(roles) const row = yield* requireRuleRow(orgId, ruleId) - // Hidden from the generic rule APIs: a monitor's rule is deleted with its monitor. - if (row.signalType === "uptime") { - return yield* new AlertRuleNotFoundError({ message: UPTIME_RULE_MANAGED_MESSAGE, ruleId }) + // Hidden from the generic rule APIs: a managed rule is deleted with its owner. + if (isManagedSignalType(row.signalType)) { + return yield* new AlertRuleNotFoundError({ message: managedRuleMessage(row.signalType), ruleId }) } const deleted = yield* dbExecute((db) => db.transaction(deleteRuleRows(db, orgId, ruleId))) const txid = readTxid(deleted) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 132b1b55e2..56305ec41f 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -72,6 +72,7 @@ import { RoleName, WarehouseUpstreamError, type UserId, + isManagedSignalType, } from "@maple/domain/http" import * as PG from "@maple-dev/effect-orm/postgres" import * as Orm from "@maple-dev/effect-orm/database" @@ -163,6 +164,8 @@ import { makeAlertDestinationDelivery, parseAlertDestinationEncryptionKey } from import { AlertReadModelsService, type AlertReadModelsServiceApi } from "./AlertReadModelsService" import { AlertRulesService, makeAlertRulePersistence, type AlertRulesServiceApi } from "./AlertRulesService" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" +import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource" +import { evaluateSloBurn } from "@maple/backend/services/slos/slo-alert-rule" import { openForAlertIncident, resolveForAlertIncident, @@ -180,7 +183,7 @@ import { selectStoredAlertRules, type NormalizedRule, type StoredAlertRuleRow, - UPTIME_RULE_MANAGED_MESSAGE, + managedRuleMessage, } from "./AlertRuleModel" import { mapSignalUnit, resolveSignalDisplay } from "./alert-signal-display" import { summarizeCause } from "@maple/backend/platform/describe-cause" @@ -621,6 +624,7 @@ export class AlertsService extends Context.Service + Effect.fail(makePersistenceError(error)), + ), + ) + return Option.match(observation, { + onNone: () => [], + onSome: (windows) => [{ evaluation: evaluateSloBurn(windows), groupKey: UNGROUPED_GROUP_KEY }], + }) + } const startMs = endMs - rule.windowMinutes * 60_000 const plan = rule.compiledPlan const source = yield* planEvaluateSource(plan, rule) @@ -1566,8 +1585,8 @@ export class AlertsService extends Context.Service Effect.Effect>, SloPersistenceError | WarehouseReadError> +} + +const toPersistenceError = (error: unknown) => + new SloPersistenceError({ + message: error instanceof Error ? error.message : "SLO persistence failed", + cause: error, + }) + +/** + * The alert evaluator's view of SLOs: its own service, with only the database and the + * warehouse, so the alerting worker takes on nothing else of the SLO API. + */ +export class SloAlertSource extends Context.Service()( + "@maple/backend/services/slos/SloAlertSource", + { + make: Effect.gen(function* () { + const database = yield* Database + const dbExecute = makeDbExecute(database, "SloAlertSource", toPersistenceError) + const warehouse = yield* WarehouseQueryService + + const observe = Effect.fn("SloAlertSource.observe")(function* ( + orgId: OrgId, + ruleId: AlertRuleId, + endMs: number, + ) { + const rows = yield* dbExecute((db) => + db.run( + PG.from(Slos) + .select("id", "orgId", "serviceName", "environmentsJson", "sliJson", "targetPercent", "windowDays") + .where(($) => [$.orgId.eq(orgId), $.alertRuleId.eq(ruleId)]) + .limit(1), + ), + ) + const slo = rows[0] + if (slo === undefined) return Option.none>() + yield* Effect.annotateCurrentSpan({ "maple.slo.id": slo.id }) + const windows = yield* readBurnWindows( + systemTenant(orgId), + { + orgId, + serviceName: slo.serviceName, + environments: slo.environmentsJson, + sli: slo.sliJson, + targetPercent: slo.targetPercent, + windowDays: slo.windowDays, + }, + endMs, + ).pipe(Effect.provideService(WarehouseQueryService, warehouse)) + return Option.some(windows) + }) + + return { observe } satisfies SloAlertSourceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(WarehouseQueryService.layer)) +} diff --git a/packages/backend/src/services/slos/SlosService.ts b/packages/backend/src/services/slos/SlosService.ts index 184d258227..db79dde85a 100644 --- a/packages/backend/src/services/slos/SlosService.ts +++ b/packages/backend/src/services/slos/SlosService.ts @@ -1,13 +1,22 @@ import { randomUUID } from "node:crypto" +import * as Orm from "@maple-dev/effect-orm/database" import * as PG from "@maple-dev/effect-orm/postgres" -import { Slos, type SloRow } from "@maple/db/tables" +import { AlertDestinations, AlertRuleClaims, AlertRules, Slos } from "@maple/db/tables" import { SloNotFoundError, SloPersistenceError, SloValidationError, + AlertThresholds, type WarehouseReadError, } from "@maple/domain/http" -import { type AlertRuleId, type OrgId, SloId, SloWindowDays, type UserId } from "@maple/domain/primitives" +import { + type AlertDestinationId, + AlertRuleId, + type OrgId, + SloId, + SloWindowDays, + type UserId, +} from "@maple/domain/primitives" import { budgetRemaining, sliRatio, @@ -16,13 +25,15 @@ import { type SloHistoryPoint, type SloSli, } from "@maple/domain/slo" -import { Array as Arr, Clock, Context, Effect, Layer, Schema, String as Str } from "effect" +import { Array as Arr, Clock, Context, Effect, Layer, Option, Schema, String as Str } from "effect" import type { MapleDb } from "@maple/db/client" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" import type { TenantContext } from "@maple/backend/services/auth/AuthService" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { deleteRuleRows } from "@maple/backend/services/alerts/AlertRulesService" import { readBudgetHistory, readBurnWindows, readWindowTotals } from "./slo-reads" +import { sloRuleFields, sloRuleName, type SloRuleInput } from "./slo-alert-rule" export interface SloRecord { readonly id: SloId @@ -35,10 +46,20 @@ export interface SloRecord { readonly targetPercent: number readonly windowDays: SloWindowDays readonly alertRuleId: AlertRuleId | null + readonly alerting: SloAlerting readonly createdAt: number readonly updatedAt: number } +/** The SLO's burn-rate alert: on or off, and who each tier notifies. */ +export interface SloAlerting { + readonly enabled: boolean + /** Notified when the fast burn (critical, page) fires. */ + readonly criticalDestinationIds: ReadonlyArray + /** Notified when the slow burn (warning, ticket) fires; a critical incident notifies them too. */ + readonly warningDestinationIds: ReadonlyArray +} + export interface CreateSloInput { readonly name: string readonly description?: string | null @@ -47,6 +68,7 @@ export interface CreateSloInput { readonly sli: SloSli readonly targetPercent: number readonly windowDays?: SloWindowDays + readonly alerting?: Partial readonly createdBy: UserId } @@ -58,6 +80,7 @@ export interface UpdateSloInput { readonly sli?: SloSli readonly targetPercent?: number readonly windowDays?: SloWindowDays + readonly alerting?: Partial readonly updatedBy: UserId } @@ -133,7 +156,55 @@ const validateWindow = (value: number) => const normalizeEnvironments = (environments: ReadonlyArray) => Arr.dedupe(Arr.filter(Arr.map(environments, Str.trim), Str.isNonEmpty)) -const toRecord = (row: Omit): SloRecord => ({ +/** Every SLO with its burn-rate rule, which carries the alerting settings. */ +const selectSlos = () => + PG.from(Slos, "s") + .leftJoin(AlertRules, "r", (slo, rule) => slo.alertRuleId.eq(rule.id).and(slo.orgId.eq(rule.orgId))) + .select(($) => ({ + id: $.id, + orgId: $.orgId, + name: $.name, + description: $.description, + serviceName: $.serviceName, + environmentsJson: $.environmentsJson, + sliJson: $.sliJson, + targetPercent: $.targetPercent, + windowDays: $.windowDays, + alertRuleId: $.alertRuleId, + ruleEnabled: $.r.enabled, + ruleThresholdsJson: $.r.thresholdsJson, + createdAt: $.createdAt, + updatedAt: $.updatedAt, + })) + +interface SloSelectRow { + readonly id: SloId + readonly orgId: OrgId + readonly name: string + readonly description: string | null + readonly serviceName: string + readonly environmentsJson: ReadonlyArray + readonly sliJson: SloSli + readonly targetPercent: number + readonly windowDays: SloWindowDays + readonly alertRuleId: AlertRuleId | null + readonly ruleEnabled: boolean | null + /** Untyped jsonb: read through `decodeThresholds`. */ + readonly ruleThresholdsJson: unknown + readonly createdAt: number + readonly updatedAt: number +} + +const decodeThresholds = Schema.decodeUnknownOption(AlertThresholds) + +/** A tier's destinations; none when the rule is missing (an SLO saved before burn alerts). */ +const tierDestinations = (row: SloSelectRow, severity: "critical" | "warning") => + decodeThresholds(row.ruleThresholdsJson).pipe( + Option.flatMap((tiers) => Arr.findFirst(tiers, (tier) => tier.severity === severity)), + Option.match({ onNone: (): ReadonlyArray => [], onSome: (tier) => tier.destinationIds }), + ) + +const toRecord = (row: SloSelectRow): SloRecord => ({ id: row.id, orgId: row.orgId, name: row.name, @@ -144,25 +215,51 @@ const toRecord = (row: Omit): SloRecord => ({ targetPercent: row.targetPercent, windowDays: row.windowDays, alertRuleId: row.alertRuleId, + alerting: { + enabled: row.ruleEnabled ?? false, + criticalDestinationIds: tierDestinations(row, "critical"), + warningDestinationIds: tierDestinations(row, "warning"), + }, createdAt: row.createdAt, updatedAt: row.updatedAt, }) -const selectSlos = () => - PG.from(Slos).select( - "id", - "orgId", - "name", - "description", - "serviceName", - "environmentsJson", - "sliJson", - "targetPercent", - "windowDays", - "alertRuleId", - "createdAt", - "updatedAt", +/** The per-org lock alert rule writes take, so a rule name check and its insert are atomic. */ +const lockOrgRules = (db: MapleDb, orgId: OrgId) => + db.execute(Orm.sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) + +/** Checks a burn-rate rule write: its destinations exist and its name is free among alert rules. */ +const claimRuleWrite = Effect.fnUntraced(function* ( + db: MapleDb, + orgId: OrgId, + ruleId: AlertRuleId, + rule: SloRuleInput, +) { + const destinationIds = Arr.dedupe([...rule.criticalDestinationIds, ...rule.warningDestinationIds]) + if (destinationIds.length > 0) { + const found = yield* db.run( + PG.from(AlertDestinations) + .select("id") + .where(($) => [$.orgId.eq(orgId), $.id.in_(...destinationIds)]), + ) + const known = new Set(Arr.map(found, (row) => row.id)) + const missing = Arr.findFirst(destinationIds, (id) => !known.has(id)) + if (Option.isSome(missing)) return yield* invalid(`Unknown alert destination ${missing.value}`) + } + const ruleName = sloRuleName(rule.name) + const sameName = yield* db.run( + PG.from(AlertRules) + .select("id") + .where(($) => [$.orgId.eq(orgId), $.name.eq(ruleName), $.id.neq(ruleId)]) + .limit(1), ) + if (sameName.length > 0) { + return yield* invalid(`An alert rule named '${ruleName}' already exists; rename it or pick another SLO name`) + } +}) + +const sameIds = (a: ReadonlyArray, b: ReadonlyArray) => + a.length === b.length && Arr.every(a, (id, index) => id === b[index]) /** Names are unique per org: the list, incidents and notifications identify an SLO by it. */ const ensureNameFree = Effect.fnUntraced(function* (db: MapleDb, orgId: OrgId, name: string, sloId: SloId) { @@ -222,15 +319,38 @@ export class SlosService extends Context.Service()( yield* Effect.annotateCurrentSpan({ orgId, "maple.slo.sli": input.sli._tag }) const name = yield* requireText("name", input.name) const serviceName = yield* requireText("service_name", input.serviceName) - const sli = input.sli const targetPercent = yield* validateTarget(input.targetPercent) const windowDays = yield* validateWindow(input.windowDays ?? DEFAULT_SLO_WINDOW_DAYS) + const environments = normalizeEnvironments(input.environments ?? []) const now = yield* Clock.currentTimeMillis const id = SloId.make(randomUUID()) + const ruleId = AlertRuleId.make(randomUUID()) + const rule: SloRuleInput = { + name, + serviceName, + environments, + enabled: input.alerting?.enabled ?? true, + criticalDestinationIds: Arr.dedupe(input.alerting?.criticalDestinationIds ?? []), + warningDestinationIds: Arr.dedupe(input.alerting?.warningDestinationIds ?? []), + } yield* dbExecute((db) => db.transaction( Effect.gen(function* () { + yield* lockOrgRules(db, orgId) yield* ensureNameFree(db, orgId, name, id) + yield* claimRuleWrite(db, orgId, ruleId, rule) + yield* db.run( + PG.insertInto(AlertRules).values({ + id: ruleId, + orgId, + ...sloRuleFields(rule), + lastScheduledAt: null, + createdAt: now, + updatedAt: now, + createdBy: input.createdBy, + updatedBy: input.createdBy, + }), + ) yield* db.run( PG.insertInto(Slos).values({ id, @@ -238,11 +358,11 @@ export class SlosService extends Context.Service()( name, description: input.description ?? null, serviceName, - environmentsJson: normalizeEnvironments(input.environments ?? []), - sliJson: sli, + environmentsJson: environments, + sliJson: input.sli, targetPercent, windowDays, - alertRuleId: null, + alertRuleId: ruleId, createdBy: input.createdBy, updatedBy: input.createdBy, createdAt: now, @@ -270,54 +390,116 @@ export class SlosService extends Context.Service()( input.serviceName === undefined ? undefined : yield* requireText("service_name", input.serviceName) - const sli = input.sli const targetPercent = input.targetPercent === undefined ? undefined : yield* validateTarget(input.targetPercent) const windowDays = input.windowDays === undefined ? undefined : yield* validateWindow(input.windowDays) + const environments = + input.environments === undefined ? undefined : normalizeEnvironments(input.environments) const now = yield* Clock.currentTimeMillis - const updated = yield* dbExecute((db) => + yield* dbExecute((db) => db.transaction( Effect.gen(function* () { + // Lock, then re-read: the rule is rebuilt from a row no other write changes + // before commit, and a concurrent delete either finished first or waits. + yield* lockOrgRules(db, orgId) + const lockedRows = yield* db.run( + selectSlos() + .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) + .limit(1), + ) + const lockedRow = lockedRows[0] + if (lockedRow === undefined) return yield* notFound(sloId) + const locked = toRecord(lockedRow) if (name !== undefined) yield* ensureNameFree(db, orgId, name, sloId) - return yield* db.run( + const rule: SloRuleInput = { + name: name ?? locked.name, + serviceName: serviceName ?? locked.serviceName, + environments: environments ?? locked.environments, + // An SLO saved before burn alerts existed gets its rule switched on. + enabled: input.alerting?.enabled ?? (locked.alertRuleId === null || locked.alerting.enabled), + criticalDestinationIds: + input.alerting?.criticalDestinationIds === undefined + ? locked.alerting.criticalDestinationIds + : Arr.dedupe(input.alerting.criticalDestinationIds), + warningDestinationIds: + input.alerting?.warningDestinationIds === undefined + ? locked.alerting.warningDestinationIds + : Arr.dedupe(input.alerting.warningDestinationIds), + } + // The rule is written only when it changes: alert_rules is synced to browsers. + const ruleChanged = + locked.alertRuleId === null || + rule.name !== locked.name || + rule.serviceName !== locked.serviceName || + !sameIds(rule.environments, locked.environments) || + rule.enabled !== locked.alerting.enabled || + !sameIds(rule.criticalDestinationIds, locked.alerting.criticalDestinationIds) || + !sameIds(rule.warningDestinationIds, locked.alerting.warningDestinationIds) + const ruleId = locked.alertRuleId ?? AlertRuleId.make(randomUUID()) + if (ruleChanged) { + yield* claimRuleWrite(db, orgId, ruleId, rule) + if (locked.alertRuleId === null) { + yield* db.run( + PG.insertInto(AlertRules).values({ + id: ruleId, + orgId, + ...sloRuleFields(rule), + lastScheduledAt: null, + createdAt: now, + updatedAt: now, + createdBy: input.updatedBy, + updatedBy: input.updatedBy, + }), + ) + } else { + // Ends a tick's claim on the old config, as a rule edit does. + yield* db.run(PG.deleteFrom(AlertRuleClaims).where(($) => [$.ruleId.eq(ruleId)])) + yield* db.run( + PG.update(AlertRules) + .set({ ...sloRuleFields(rule), updatedAt: now, updatedBy: input.updatedBy }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(ruleId)]), + ) + } + } + yield* db.run( PG.update(Slos) .set({ + ...(locked.alertRuleId === null ? { alertRuleId: ruleId } : undefined), ...(name === undefined ? undefined : { name }), - ...(input.description === undefined - ? undefined - : { description: input.description }), + ...(input.description === undefined ? undefined : { description: input.description }), ...(serviceName === undefined ? undefined : { serviceName }), - ...(input.environments === undefined - ? undefined - : { - environmentsJson: normalizeEnvironments( - input.environments, - ), - }), - ...(sli === undefined ? undefined : { sliJson: sli }), + ...(environments === undefined ? undefined : { environmentsJson: environments }), + ...(input.sli === undefined ? undefined : { sliJson: input.sli }), ...(targetPercent === undefined ? undefined : { targetPercent }), ...(windowDays === undefined ? undefined : { windowDays }), updatedBy: input.updatedBy, updatedAt: now, }) - .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) - .returning("id"), + .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]), ) }), ), ) - if (updated.length === 0) return yield* notFound(sloId) return yield* selectOne(orgId, sloId) }) const remove = Effect.fn("SlosService.delete")(function* (orgId: OrgId, sloId: SloId) { yield* Effect.annotateCurrentSpan({ orgId, "maple.slo.id": sloId }) + // The burn-rate rule goes with the SLO, through the rule delete's own cascade. const removed = yield* dbExecute((db) => - db.run( - PG.deleteFrom(Slos) - .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) - .returning("id"), + db.transaction( + Effect.gen(function* () { + yield* lockOrgRules(db, orgId) + const rows = yield* db.run( + PG.deleteFrom(Slos) + .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) + .returning("id", "alertRuleId"), + ) + const ruleId = rows[0]?.alertRuleId ?? null + if (ruleId !== null) yield* deleteRuleRows(db, orgId, ruleId) + return rows + }), ), ) if (removed.length === 0) return yield* notFound(sloId) diff --git a/packages/backend/src/services/slos/slo-alert-rule.ts b/packages/backend/src/services/slos/slo-alert-rule.ts new file mode 100644 index 0000000000..a0463d5d61 --- /dev/null +++ b/packages/backend/src/services/slos/slo-alert-rule.ts @@ -0,0 +1,128 @@ +// The hidden `slo_burn_rate` alert rule behind each SLO, and what one evaluation of it observes. +import type { AlertEvaluation } from "@maple/alerting-core" +import type { AlertDestinationId, AlertThresholds } from "@maple/domain/http" +import { + SLO_BURN_TIERS, + SLO_BURN_LOOKBACK_MINUTES, + tierBreached, + type SloBurnTier, + type SloBurnWindow, +} from "@maple/domain/slo" +import { Array as Arr, Option } from "effect" +import { MANAGED_RULE_PLAN } from "@maple/backend/services/alerts/AlertRuleModel" + +const [CRITICAL_TIER, WARNING_TIER] = SLO_BURN_TIERS + +/** The rule's name: it titles incidents and notifications, so it says what is burning. */ +export const sloRuleName = (sloName: string) => `SLO burn: ${sloName}` + +export interface SloRuleInput { + readonly name: string + readonly serviceName: string + readonly environments: ReadonlyArray + readonly enabled: boolean + readonly criticalDestinationIds: ReadonlyArray + readonly warningDestinationIds: ReadonlyArray +} + +/** + * The `alert_rules` columns of an SLO's burn-rate rule: a warning tier (ticket) and a critical + * tier (page) at their burn factors, checked every minute. One breached check opens and one + * healthy check resolves: the short windows already keep it from flapping. + */ +export const sloRuleFields = (slo: SloRuleInput) => { + const thresholds: AlertThresholds = [ + { + severity: "warning", + value: WARNING_TIER.factor, + upper: null, + recoveryValue: null, + destinationIds: [...slo.warningDestinationIds], + }, + { + severity: "critical", + value: CRITICAL_TIER.factor, + upper: null, + recoveryValue: null, + destinationIds: [...slo.criticalDestinationIds], + }, + ] + return { + name: sloRuleName(slo.name), + notes: null, + notificationTemplateJson: null, + enabled: slo.enabled, + severity: "critical", + serviceNamesJson: [slo.serviceName], + excludeServiceNamesJson: null, + environmentsJson: slo.environments.length === 0 ? null : [...slo.environments], + tagsJson: null, + groupBy: null, + signalType: "slo_burn_rate", + comparator: "gte", + threshold: WARNING_TIER.factor, + thresholdUpper: null, + thresholdsJson: thresholds, + windowMinutes: SLO_BURN_LOOKBACK_MINUTES, + evaluationIntervalMinutes: 1, + slowedIntervalMinutes: null, + slowedAt: null, + minimumSampleCount: 0, + consecutiveBreachesRequired: 1, + breachesOutOf: null, + consecutiveHealthyRequired: 1, + renotifyIntervalMinutes: 60, + apdexThresholdMs: null, + queryBuilderDraftJson: null, + rawQuerySql: null, + destinationIdsJson: Arr.dedupe([...slo.criticalDestinationIds, ...slo.warningDestinationIds]), + querySpecJson: null, + reducer: MANAGED_RULE_PLAN.reducer, + sampleCountStrategy: null, + noDataBehavior: MANAGED_RULE_PLAN.noDataBehavior, + } as const +} + +const rateOf = (windows: ReadonlyArray, minutes: number) => + Arr.findFirst(windows, (window) => window.windowMinutes === minutes) + +const formatRate = (rate: number) => `${rate >= 100 ? Math.round(rate) : Math.round(rate * 10) / 10}x` + +const describeTier = (tier: SloBurnTier, windows: ReadonlyArray) => { + const long = Option.match(rateOf(windows, tier.longWindowMinutes), { onNone: () => 0, onSome: (w) => w.burnRate }) + const short = Option.match(rateOf(windows, tier.shortWindowMinutes), { + onNone: () => 0, + onSome: (w) => w.burnRate, + }) + const label = (minutes: number) => (minutes >= 60 ? `${minutes / 60}h` : `${minutes}m`) + return { + long, + text: `burning the error budget at ${formatRate(long)} over ${label(tier.longWindowMinutes)} and ${formatRate(short)} over ${label(tier.shortWindowMinutes)} (${tier.severity} at ${tier.factor}x)`, + } +} + +/** + * One check of an SLO's burn-rate rule. Each tier is judged on its own pair of windows, so the + * levels are set here rather than by comparing one value to every tier. The value is the + * long-window burn of the highest tier that fires, else the 1h burn. + */ +export const evaluateSloBurn = (windows: ReadonlyArray): AlertEvaluation => { + const levels = Arr.map([WARNING_TIER, CRITICAL_TIER], (tier) => ({ + severity: tier.severity, + status: tierBreached(tier, windows) ? ("breach" as const) : ("clear" as const), + })) + const firing = Arr.findFirst([CRITICAL_TIER, WARNING_TIER], (tier) => tierBreached(tier, windows)) + const headline = describeTier(Option.getOrElse(firing, () => CRITICAL_TIER), windows) + const hour = Option.getOrUndefined(rateOf(windows, CRITICAL_TIER.longWindowMinutes)) + return { + status: Option.isSome(firing) ? "breached" : "healthy", + levels, + value: headline.long, + sampleCount: Math.round(hour?.total ?? 0), + threshold: WARNING_TIER.factor, + thresholdUpper: null, + comparator: "gte", + reason: Option.isSome(firing) ? `SLO ${headline.text}` : `SLO within budget: ${headline.text}`, + derivedFromNoData: false, + } +} diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index 419fb3f8b0..667dca4fc9 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -47,12 +47,20 @@ export const AlertSignalType = Schema.Literals([ "raw_query", /** The hidden rule behind an uptime monitor; managed only through the monitor. */ "uptime", + /** The hidden burn-rate rule behind an SLO; managed only through the SLO. */ + "slo_burn_rate", ]).annotate({ identifier: "@maple/AlertSignalType", title: "Alert Signal Type", }) export type AlertSignalType = Schema.Schema.Type +/** Signal types whose rule another resource owns: hidden from and refused by the generic rule APIs. */ +export const MANAGED_SIGNAL_TYPES = ["uptime", "slo_burn_rate"] as const satisfies ReadonlyArray +export type ManagedSignalType = (typeof MANAGED_SIGNAL_TYPES)[number] +export const isManagedSignalType = (signalType: string): signalType is ManagedSignalType => + signalType === "uptime" || signalType === "slo_burn_rate" + export const AlertGroupByDimension = Schema.String.check(Schema.isMinLength(1), Schema.isTrimmed()).annotate({ identifier: "@maple/AlertGroupByDimension", title: "Alert Group By Dimension", diff --git a/packages/domain/src/http/v2/slos.ts b/packages/domain/src/http/v2/slos.ts index eab5f29d42..89a4169743 100644 --- a/packages/domain/src/http/v2/slos.ts +++ b/packages/domain/src/http/v2/slos.ts @@ -9,7 +9,7 @@ import { V2InsufficientPermissions, V2ParameterInvalid } from "./errors" import { V2WarehouseReadErrors } from "./query-errors" import { publicErrors } from "./public-error" import { PublicId, PublicIdPrefixes } from "./public-id" -import { AlertRulePublicId } from "./resource-ids" +import { AlertDestinationPublicId, AlertRulePublicId } from "./resource-ids" /** `slo_…` public ID ⇄ internal `SloId` (raw UUID). */ export const SloPublicId = PublicId(PublicIdPrefixes.slo, SloId) @@ -69,6 +69,35 @@ const windowField = SloWindowDays.annotate({ examples: [30], }) +const destinationIdsField = (description: string) => Schema.Array(AlertDestinationPublicId).annotate({ description }) + +export const V2SloAlerting = Schema.Struct({ + enabled: Schema.Boolean.annotate({ + description: "Whether the burn-rate alert runs. Incidents open on the SLO's alert rule.", + }), + critical_destination_ids: destinationIdsField( + "Notified when the budget burns fast: 14.4x the sustainable rate over both 1 hour and 5 minutes (page).", + ), + warning_destination_ids: destinationIdsField( + "Notified when the budget burns slowly: 6x over both 6 hours and 30 minutes (ticket). A critical incident notifies these too.", + ), +}).annotate({ + identifier: "SloAlerting", + title: "SLO burn-rate alerting", + description: "Multiwindow, multi-burn-rate alerting on the SLO's error budget.", +}) + +export const V2SloAlertingParams = Schema.Struct({ + enabled: Schema.optionalKey(V2SloAlerting.fields.enabled), + critical_destination_ids: Schema.optionalKey(V2SloAlerting.fields.critical_destination_ids), + warning_destination_ids: Schema.optionalKey(V2SloAlerting.fields.warning_destination_ids), +}).annotate({ + identifier: "SloAlertingParams", + title: "SLO burn-rate alerting parameters", + description: "Omitted fields keep their value; a new SLO alerts with no destinations until you add some.", +}) +export type V2SloAlertingParams = Schema.Schema.Type + const sloExample = { id: "slo_YofPTrK9782DWwcnXhpcCw", object: "slo", @@ -79,7 +108,8 @@ const sloExample = { sli: { type: "availability" }, target_percent: 99.9, window_days: 30, - alert_rule_id: null, + alert_rule_id: "alrt_YofPTrK9782DWwcnXhpcCw", + alerting: { enabled: true, critical_destination_ids: ["dest_oybbpTBhtSFGShMjjLiCrh"], warning_destination_ids: [] }, created_at: "2026-07-01T12:00:00.000Z", updated_at: "2026-07-15T09:12:00.000Z", } as const @@ -100,6 +130,7 @@ export const V2Slo = Schema.Struct({ alert_rule_id: Schema.NullOr(AlertRulePublicId).annotate({ description: "The burn-rate alert rule backing this SLO: its incidents carry this rule id.", }), + alerting: V2SloAlerting, created_at: Timestamp.annotate({ description: "When the SLO was created." }), updated_at: Timestamp.annotate({ description: "When the SLO was last updated." }), }).annotate({ @@ -119,6 +150,7 @@ export const V2SloCreateParams = Schema.Struct({ sli: V2SloSli, target_percent: targetField, window_days: Schema.optionalKey(windowField.annotate({ description: "Defaults to 30." })), + alerting: Schema.optionalKey(V2SloAlertingParams), }).annotate({ identifier: "SloCreateParams", title: "SLO create parameters", @@ -143,6 +175,7 @@ export const V2SloUpdateParams = Schema.Struct({ sli: Schema.optionalKey(V2SloSli), target_percent: Schema.optionalKey(targetField), window_days: Schema.optionalKey(windowField), + alerting: Schema.optionalKey(V2SloAlertingParams), }).annotate({ identifier: "SloUpdateParams", title: "SLO update parameters", From 2c287c56fc7a6fb1593b6fa5462a43e82dc1c8dc Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:13:51 +0100 Subject: [PATCH 02/11] feat(slos): multiwindow burn-rate alerts through the alert engine Each SLO owns a hidden slo_burn_rate alert rule, written with the SLO in one transaction. The evaluator reads the SLO's 5m/30m/1h/6h burn and judges each tier on its own pair of windows: critical at 14.4x over 1h and 5m, warning at 6x over 6h and 30m. Incidents, destinations and delivery are the engine's. Uptime and SLO rules share one managed-rule check (isManagedSignalType) that hides them from the rule list and cap and refuses generic edits. --- apps/api/src/routes/v2/slos.http.test.ts | 13 ++ apps/api/src/routes/v2/slos.http.ts | 2 + .../services/alerts/AlertsService.slo.test.ts | 176 ++++++++++++++++++ .../src/services/alerts/AlertsService.ts | 4 +- .../src/services/slos/SloAlertSource.ts | 10 +- .../backend/src/services/slos/SlosService.ts | 40 +++- .../src/services/slos/slo-alert-rule.ts | 10 +- packages/domain/src/http/alerts.ts | 5 +- packages/domain/src/http/v2/slos.ts | 9 +- packages/web-data/src/alerts/form-utils.ts | 4 + packages/web-data/src/alerts/overview.ts | 11 +- 11 files changed, 265 insertions(+), 19 deletions(-) create mode 100644 packages/backend/src/services/alerts/AlertsService.slo.test.ts diff --git a/apps/api/src/routes/v2/slos.http.test.ts b/apps/api/src/routes/v2/slos.http.test.ts index 5de1d0fa1a..0c0838ce89 100644 --- a/apps/api/src/routes/v2/slos.http.test.ts +++ b/apps/api/src/routes/v2/slos.http.test.ts @@ -178,6 +178,18 @@ describe("v2 slos", () => { window_days: 30, }) const id: string = created.body.id + // Every SLO gets its burn-rate rule, alerting with no destinations until some are added. + expect(created.body.alert_rule_id).toMatch(/^alrt_/) + expect(created.body.alerting).toEqual({ + enabled: true, + critical_destination_ids: [], + warning_destination_ids: [], + }) + const paused = await harness.request("PATCH", `/v2/slos/${id}`, token, { + alerting: { enabled: false }, + }) + expect(paused.body.alerting.enabled).toBe(false) + expect(paused.body.alert_rule_id).toBe(created.body.alert_rule_id) const listed = await harness.request("GET", "/v2/slos", token) expect(listed.body.data.map((slo: { readonly id: string }) => slo.id)).toEqual([id]) @@ -195,6 +207,7 @@ describe("v2 slos", () => { target_percent: 99, window_days: 7, environments: [], + alerting: { enabled: false }, }) const retrieved = await harness.request("GET", `/v2/slos/${id}`, token) diff --git a/apps/api/src/routes/v2/slos.http.ts b/apps/api/src/routes/v2/slos.http.ts index 1b4550cf08..84faac9a3c 100644 --- a/apps/api/src/routes/v2/slos.http.ts +++ b/apps/api/src/routes/v2/slos.http.ts @@ -111,6 +111,7 @@ export const HttpV2SlosLive = HttpApiBuilder.group(MapleApiV2, "slos", (handlers ...(payload.window_days === undefined ? undefined : { windowDays: payload.window_days }), + ...alertingFromWire(payload.alerting), createdBy: tenant.userId, }) yield* recordHttpAudit("slo.created", { @@ -142,6 +143,7 @@ export const HttpV2SlosLive = HttpApiBuilder.group(MapleApiV2, "slos", (handlers ...(payload.window_days === undefined ? undefined : { windowDays: payload.window_days }), + ...alertingFromWire(payload.alerting), updatedBy: tenant.userId, }) yield* recordHttpAudit("slo.updated", { diff --git a/packages/backend/src/services/alerts/AlertsService.slo.test.ts b/packages/backend/src/services/alerts/AlertsService.slo.test.ts new file mode 100644 index 0000000000..df708ff998 --- /dev/null +++ b/packages/backend/src/services/alerts/AlertsService.slo.test.ts @@ -0,0 +1,176 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { Clock, Duration, Effect, Layer, Option, Array as Arr } from "effect" +import { TestClock } from "effect/testing" +import { compiledQueryOf } from "@maple/query-engine/execution" +import { formatWarehouseDateTime } from "@maple/query-engine" +import { AlertDestinationId } from "@maple/domain/primitives" +import { AlertsService } from "./AlertsService" +import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/platform/test-pglite" +import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { SlosService } from "@maple/backend/services/slos/SlosService" +import { + DEFAULT_CLOCK_EPOCH_MS, + adminRoles, + asOrgId, + asUserId, + createWebhookDestination, + makeLayer, +} from "./alerts-service-test-support" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +/** Bad events per 1000 in every minute the burn read sees; mutated as a test moves on. */ +interface Traffic { + badPerMille: number +} + +/** + * A warehouse answering the SLO burn read with 400 minutes of uniform traffic ending at the + * test clock, so every burn window sees the same bad share. + */ +const sloWarehouse = (traffic: Traffic) => + makeWarehouseServiceStub({ + compiledQuery: (_tenant, compiled, options) => + Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis + const minute = Math.floor(nowMs / 60_000) * 60_000 + const rows = + options?.context === "sloBurnWindows" + ? Arr.makeBy(400, (index) => ({ + bucket: formatWarehouseDateTime(minute - index * 60_000), + total: 1000, + good: 1000 - traffic.badPerMille, + })) + : [] + return yield* compiledQueryOf(compiled).decodeRows(rows).pipe(Effect.orDie) + }), + }) + +const harness = (traffic: Traffic) => { + const testDb = createTestDb(trackedDbs) + const warehouse = sloWarehouse(traffic) + const slos = Layer.effect(SlosService, SlosService.make).pipe( + Layer.provide(Layer.mergeAll(testDb.layer, Layer.succeed(WarehouseQueryService, warehouse))), + ) + return { testDb, layer: Layer.mergeAll(makeLayer(testDb, warehouse), slos) } +} + +const orgId = asOrgId("org_slo_alerts") +const userId = asUserId("user_slo_alerts") + +const createSlo = Effect.fn(function* (destinationId: AlertDestinationId | undefined) { + const slos = yield* SlosService + return yield* slos.create(orgId, { + name: "Checkout availability", + serviceName: "checkout", + environments: ["production"], + sli: { _tag: "availability" }, + targetPercent: 99.9, + alerting: + destinationId === undefined + ? {} + : { criticalDestinationIds: [destinationId], warningDestinationIds: [destinationId] }, + createdBy: userId, + }) +}) + +describe("AlertsService SLO burn-rate rules", () => { + it.effect("a fast burn pages, and a recovered short window resolves it", () => { + const traffic: Traffic = { badPerMille: 20 } + const { layer } = harness(traffic) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const slo = yield* createSlo(destination.id) + assert.isNotNull(slo.alertRuleId) + + // 2% bad at 99.9% burns 20x in every window: past both tiers, so critical. + yield* alerts.runSchedulerTick() + const opened = (yield* alerts.listIncidents(orgId)).incidents + assert.lengthOf(opened, 1) + assert.strictEqual(opened[0]?.signalType, "slo_burn_rate") + assert.strictEqual(opened[0]?.severity, "critical") + assert.strictEqual(opened[0]?.ruleId, slo.alertRuleId) + assert.strictEqual(opened[0]?.ruleName, "SLO burn: Checkout availability") + + traffic.badPerMille = 0 + yield* TestClock.adjust(Duration.minutes(1)) + yield* alerts.runSchedulerTick() + const resolved = (yield* alerts.listIncidents(orgId)).incidents + assert.strictEqual(resolved[0]?.status, "resolved") + }).pipe(Effect.provide(layer)) + }) + + it.effect("a slow burn opens a warning, not a page", () => { + // 0.8% bad burns 8x: over the 6x ticket factor, under the 14.4x page factor. + const { layer } = harness({ badPerMille: 8 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + const opened = (yield* alerts.listIncidents(orgId)).incidents + assert.lengthOf(opened, 1) + assert.strictEqual(opened[0]?.severity, "warning") + }).pipe(Effect.provide(layer)) + }) + + it.effect("a burn within budget opens nothing", () => { + const { layer } = harness({ badPerMille: 1 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + assert.lengthOf((yield* alerts.listIncidents(orgId)).incidents, 0) + }).pipe(Effect.provide(layer)) + }) + + it.effect("the rule is hidden from the rule APIs and deleted with its SLO", () => { + const { layer, testDb } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const slos = yield* SlosService + const slo = yield* createSlo(undefined) + const ruleId = Option.getOrThrow(Option.fromNullOr(slo.alertRuleId)) + + assert.lengthOf((yield* alerts.listRules(orgId)).rules, 0) + const refused = yield* alerts.deleteRule(orgId, adminRoles, ruleId).pipe(Effect.flip) + assert.strictEqual(refused._tag, "@maple/http/errors/AlertRuleNotFoundError") + + // Switching alerting off disables the rule; the SLO keeps it. + const paused = yield* slos.update(orgId, slo.id, { + alerting: { enabled: false }, + updatedBy: userId, + }) + assert.isFalse(paused.alerting.enabled) + + yield* slos.delete(orgId, slo.id) + const remaining = yield* Effect.promise(() => + queryFirstRow<{ count: number }>( + testDb, + "select count(*)::int as count from alert_rules where id = $1", + [ruleId], + ), + ) + assert.strictEqual(remaining?.count, 0) + }).pipe(Effect.provide(layer)) + }) + + it.effect("refuses an unknown destination and keeps the SLO unsaved", () => { + const { layer } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const slos = yield* SlosService + const refused = yield* createSlo( + AlertDestinationId.make("00000000-0000-4000-8000-000000000000"), + ).pipe(Effect.flip) + assert.strictEqual(refused._tag, "@maple/http/errors/SloValidationError") + assert.lengthOf(yield* slos.list(orgId), 0) + }).pipe(Effect.provide(layer)) + }) +}) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 56305ec41f..4fc833a93a 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -1124,7 +1124,9 @@ export class AlertsService extends Context.Service [], - onSome: (windows) => [{ evaluation: evaluateSloBurn(windows), groupKey: UNGROUPED_GROUP_KEY }], + onSome: (windows) => [ + { evaluation: evaluateSloBurn(windows), groupKey: UNGROUPED_GROUP_KEY }, + ], }) } const startMs = endMs - rule.windowMinutes * 60_000 diff --git a/packages/backend/src/services/slos/SloAlertSource.ts b/packages/backend/src/services/slos/SloAlertSource.ts index fdaf13ad34..85d4f98cb0 100644 --- a/packages/backend/src/services/slos/SloAlertSource.ts +++ b/packages/backend/src/services/slos/SloAlertSource.ts @@ -45,7 +45,15 @@ export class SloAlertSource extends Context.Service db.run( PG.from(Slos) - .select("id", "orgId", "serviceName", "environmentsJson", "sliJson", "targetPercent", "windowDays") + .select( + "id", + "orgId", + "serviceName", + "environmentsJson", + "sliJson", + "targetPercent", + "windowDays", + ) .where(($) => [$.orgId.eq(orgId), $.alertRuleId.eq(ruleId)]) .limit(1), ), diff --git a/packages/backend/src/services/slos/SlosService.ts b/packages/backend/src/services/slos/SlosService.ts index db79dde85a..10c4ff664b 100644 --- a/packages/backend/src/services/slos/SlosService.ts +++ b/packages/backend/src/services/slos/SlosService.ts @@ -201,7 +201,10 @@ const decodeThresholds = Schema.decodeUnknownOption(AlertThresholds) const tierDestinations = (row: SloSelectRow, severity: "critical" | "warning") => decodeThresholds(row.ruleThresholdsJson).pipe( Option.flatMap((tiers) => Arr.findFirst(tiers, (tier) => tier.severity === severity)), - Option.match({ onNone: (): ReadonlyArray => [], onSome: (tier) => tier.destinationIds }), + Option.match({ + onNone: (): ReadonlyArray => [], + onSome: (tier) => tier.destinationIds, + }), ) const toRecord = (row: SloSelectRow): SloRecord => ({ @@ -254,7 +257,9 @@ const claimRuleWrite = Effect.fnUntraced(function* ( .limit(1), ) if (sameName.length > 0) { - return yield* invalid(`An alert rule named '${ruleName}' already exists; rename it or pick another SLO name`) + return yield* invalid( + `An alert rule named '${ruleName}' already exists; rename it or pick another SLO name`, + ) } }) @@ -417,7 +422,9 @@ export class SlosService extends Context.Service()( serviceName: serviceName ?? locked.serviceName, environments: environments ?? locked.environments, // An SLO saved before burn alerts existed gets its rule switched on. - enabled: input.alerting?.enabled ?? (locked.alertRuleId === null || locked.alerting.enabled), + enabled: + input.alerting?.enabled ?? + (locked.alertRuleId === null || locked.alerting.enabled), criticalDestinationIds: input.alerting?.criticalDestinationIds === undefined ? locked.alerting.criticalDestinationIds @@ -434,7 +441,10 @@ export class SlosService extends Context.Service()( rule.serviceName !== locked.serviceName || !sameIds(rule.environments, locked.environments) || rule.enabled !== locked.alerting.enabled || - !sameIds(rule.criticalDestinationIds, locked.alerting.criticalDestinationIds) || + !sameIds( + rule.criticalDestinationIds, + locked.alerting.criticalDestinationIds, + ) || !sameIds(rule.warningDestinationIds, locked.alerting.warningDestinationIds) const ruleId = locked.alertRuleId ?? AlertRuleId.make(randomUUID()) if (ruleChanged) { @@ -454,10 +464,16 @@ export class SlosService extends Context.Service()( ) } else { // Ends a tick's claim on the old config, as a rule edit does. - yield* db.run(PG.deleteFrom(AlertRuleClaims).where(($) => [$.ruleId.eq(ruleId)])) + yield* db.run( + PG.deleteFrom(AlertRuleClaims).where(($) => [$.ruleId.eq(ruleId)]), + ) yield* db.run( PG.update(AlertRules) - .set({ ...sloRuleFields(rule), updatedAt: now, updatedBy: input.updatedBy }) + .set({ + ...sloRuleFields(rule), + updatedAt: now, + updatedBy: input.updatedBy, + }) .where(($) => [$.orgId.eq(orgId), $.id.eq(ruleId)]), ) } @@ -465,11 +481,17 @@ export class SlosService extends Context.Service()( yield* db.run( PG.update(Slos) .set({ - ...(locked.alertRuleId === null ? { alertRuleId: ruleId } : undefined), + ...(locked.alertRuleId === null + ? { alertRuleId: ruleId } + : undefined), ...(name === undefined ? undefined : { name }), - ...(input.description === undefined ? undefined : { description: input.description }), + ...(input.description === undefined + ? undefined + : { description: input.description }), ...(serviceName === undefined ? undefined : { serviceName }), - ...(environments === undefined ? undefined : { environmentsJson: environments }), + ...(environments === undefined + ? undefined + : { environmentsJson: environments }), ...(input.sli === undefined ? undefined : { sliJson: input.sli }), ...(targetPercent === undefined ? undefined : { targetPercent }), ...(windowDays === undefined ? undefined : { windowDays }), diff --git a/packages/backend/src/services/slos/slo-alert-rule.ts b/packages/backend/src/services/slos/slo-alert-rule.ts index a0463d5d61..951030200a 100644 --- a/packages/backend/src/services/slos/slo-alert-rule.ts +++ b/packages/backend/src/services/slos/slo-alert-rule.ts @@ -89,7 +89,10 @@ const rateOf = (windows: ReadonlyArray, minutes: number) => const formatRate = (rate: number) => `${rate >= 100 ? Math.round(rate) : Math.round(rate * 10) / 10}x` const describeTier = (tier: SloBurnTier, windows: ReadonlyArray) => { - const long = Option.match(rateOf(windows, tier.longWindowMinutes), { onNone: () => 0, onSome: (w) => w.burnRate }) + const long = Option.match(rateOf(windows, tier.longWindowMinutes), { + onNone: () => 0, + onSome: (w) => w.burnRate, + }) const short = Option.match(rateOf(windows, tier.shortWindowMinutes), { onNone: () => 0, onSome: (w) => w.burnRate, @@ -112,7 +115,10 @@ export const evaluateSloBurn = (windows: ReadonlyArray): AlertEva status: tierBreached(tier, windows) ? ("breach" as const) : ("clear" as const), })) const firing = Arr.findFirst([CRITICAL_TIER, WARNING_TIER], (tier) => tierBreached(tier, windows)) - const headline = describeTier(Option.getOrElse(firing, () => CRITICAL_TIER), windows) + const headline = describeTier( + Option.getOrElse(firing, () => CRITICAL_TIER), + windows, + ) const hour = Option.getOrUndefined(rateOf(windows, CRITICAL_TIER.longWindowMinutes)) return { status: Option.isSome(firing) ? "breached" : "healthy", diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index 667dca4fc9..a7357cfae1 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -56,7 +56,10 @@ export const AlertSignalType = Schema.Literals([ export type AlertSignalType = Schema.Schema.Type /** Signal types whose rule another resource owns: hidden from and refused by the generic rule APIs. */ -export const MANAGED_SIGNAL_TYPES = ["uptime", "slo_burn_rate"] as const satisfies ReadonlyArray +export const MANAGED_SIGNAL_TYPES = [ + "uptime", + "slo_burn_rate", +] as const satisfies ReadonlyArray export type ManagedSignalType = (typeof MANAGED_SIGNAL_TYPES)[number] export const isManagedSignalType = (signalType: string): signalType is ManagedSignalType => signalType === "uptime" || signalType === "slo_burn_rate" diff --git a/packages/domain/src/http/v2/slos.ts b/packages/domain/src/http/v2/slos.ts index 89a4169743..eb0e9b94bb 100644 --- a/packages/domain/src/http/v2/slos.ts +++ b/packages/domain/src/http/v2/slos.ts @@ -69,7 +69,8 @@ const windowField = SloWindowDays.annotate({ examples: [30], }) -const destinationIdsField = (description: string) => Schema.Array(AlertDestinationPublicId).annotate({ description }) +const destinationIdsField = (description: string) => + Schema.Array(AlertDestinationPublicId).annotate({ description }) export const V2SloAlerting = Schema.Struct({ enabled: Schema.Boolean.annotate({ @@ -109,7 +110,11 @@ const sloExample = { target_percent: 99.9, window_days: 30, alert_rule_id: "alrt_YofPTrK9782DWwcnXhpcCw", - alerting: { enabled: true, critical_destination_ids: ["dest_oybbpTBhtSFGShMjjLiCrh"], warning_destination_ids: [] }, + alerting: { + enabled: true, + critical_destination_ids: ["dest_oybbpTBhtSFGShMjjLiCrh"], + warning_destination_ids: [], + }, created_at: "2026-07-01T12:00:00.000Z", updated_at: "2026-07-15T09:12:00.000Z", } as const diff --git a/packages/web-data/src/alerts/form-utils.ts b/packages/web-data/src/alerts/form-utils.ts index 8826c3ad0e..4af730e08a 100644 --- a/packages/web-data/src/alerts/form-utils.ts +++ b/packages/web-data/src/alerts/form-utils.ts @@ -130,6 +130,7 @@ export const signalLabels: Record = { builder_query: "Query builder", raw_query: "Raw SQL", uptime: "Uptime check", + slo_burn_rate: "SLO burn rate", } satisfies Record export const RAW_QUERY_REDUCER_LABELS: Record = { @@ -185,6 +186,9 @@ export function formatSignalValue(signalType: AlertSignalType, value: number | n // An uptime rule observes 1 while its monitor is down, else 0. case "uptime": return value >= 1 ? "Down" : "Up" + // An SLO rule observes how many times faster than sustainable the budget burns. + case "slo_burn_rate": + return `${value.toFixed(1)}x` } } diff --git a/packages/web-data/src/alerts/overview.ts b/packages/web-data/src/alerts/overview.ts index d436d15247..1eed696d2d 100644 --- a/packages/web-data/src/alerts/overview.ts +++ b/packages/web-data/src/alerts/overview.ts @@ -1,4 +1,9 @@ -import type { AlertDeliveryEventDocument, AlertIncidentDocument, AlertRuleDocument } from "@maple/domain/http" +import { + isManagedSignalType, + type AlertDeliveryEventDocument, + type AlertIncidentDocument, + type AlertRuleDocument, +} from "@maple/domain/http" import type { AlertRuleStateRow } from "../collections/alerts" import { deriveRuleStatus, type DerivedRuleStatus, needsAttention } from "./rule-status" @@ -69,8 +74,8 @@ export const deriveOverview = (inputs: OverviewInputs): AlertsOverviewReady => { const { now } = inputs // ISO timestamps order lexicographically; newest first, as the server lists do. const byIsoDesc = (a: string, b: string) => (a < b ? 1 : a > b ? -1 : 0) - // Uptime monitors own their backing rules: the rule list hides them, their incidents stay. - const rules = Arr.filter(inputs.rules, (rule) => rule.signalType !== "uptime").sort((a, b) => + // Monitors and SLOs own their backing rules: the rule list hides them, their incidents stay. + const rules = Arr.filter(inputs.rules, (rule) => !isManagedSignalType(rule.signalType)).sort((a, b) => byIsoDesc(a.updatedAt, b.updatedAt), ) const incidentCutoff = now - INCIDENT_WINDOW_MS From d63a299cab49c7da5d6764fe95054190441453f0 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:24:54 +0100 Subject: [PATCH 03/11] fix(slos): hold burn alerts through outages, cap SLOs, review fixes - A tier whose short window saw no events holds its state; with every tier held the check is skipped as no data, so an outage cannot resolve a page. - Notifications carry the check's own sentence (both windows and the tier). - At most 50 SLOs per org; each one's rule reads the warehouse every minute. - An edit restores a rule whose row is gone. - Shared persistence mapper and org rule lock; history reads one hour less. --- .../services/alerts/AlertDeliveryDispatch.ts | 3 + .../services/alerts/AlertsService.slo.test.ts | 47 ++++++++++++++- .../src/services/alerts/AlertsService.ts | 4 +- .../src/services/alerts/alert-formatting.ts | 8 ++- .../src/services/alerts/alert-persistence.ts | 11 +++- .../services/alerts/alert-signal-display.ts | 1 + .../src/services/slos/SloAlertSource.ts | 9 +-- .../backend/src/services/slos/SlosService.ts | 44 +++++++------- .../src/services/slos/slo-alert-rule.ts | 60 ++++++++++++------- .../src/services/slos/slo-persistence.ts | 8 +++ .../backend/src/services/slos/slo-reads.ts | 2 +- packages/domain/src/slo.ts | 36 +++++++---- 12 files changed, 165 insertions(+), 68 deletions(-) create mode 100644 packages/backend/src/services/slos/slo-persistence.ts diff --git a/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts b/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts index c4e0d1c487..908d77b2ba 100644 --- a/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts +++ b/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts @@ -82,6 +82,9 @@ type SummaryLineContext = Pick< */ export const buildSummaryLine = (context: SummaryLineContext, em: (value: string) => string): string => { if (context.signalType === "uptime") return em(uptimeSummaryLine(context)) + // An SLO's check names both windows and the tier, which one value and window cannot. + if (context.signalType === "slo_burn_rate" && context.detail != null && context.eventType !== "test") + return em(context.detail) const signal = formatSignalLabel(context) const observed = formatSignalMetric(context.value, signalDisplayOf(context)) const window = formatWindow(context.windowMinutes) diff --git a/packages/backend/src/services/alerts/AlertsService.slo.test.ts b/packages/backend/src/services/alerts/AlertsService.slo.test.ts index df708ff998..f1eb7c1e1c 100644 --- a/packages/backend/src/services/alerts/AlertsService.slo.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.slo.test.ts @@ -5,7 +5,13 @@ import { compiledQueryOf } from "@maple/query-engine/execution" import { formatWarehouseDateTime } from "@maple/query-engine" import { AlertDestinationId } from "@maple/domain/primitives" import { AlertsService } from "./AlertsService" -import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/platform/test-pglite" +import { + cleanupTestDbs, + createTestDb, + executeSql, + queryFirstRow, + type TestDb, +} from "@maple/backend/platform/test-pglite" import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { SlosService } from "@maple/backend/services/slos/SlosService" @@ -24,6 +30,8 @@ afterEach(() => cleanupTestDbs(trackedDbs)) /** Bad events per 1000 in every minute the burn read sees; mutated as a test moves on. */ interface Traffic { badPerMille: number + /** No spans at all: the service is down or ingest stalled. */ + silent?: boolean } /** @@ -37,7 +45,7 @@ const sloWarehouse = (traffic: Traffic) => const nowMs = yield* Clock.currentTimeMillis const minute = Math.floor(nowMs / 60_000) * 60_000 const rows = - options?.context === "sloBurnWindows" + options?.context === "sloBurnWindows" && traffic.silent !== true ? Arr.makeBy(400, (index) => ({ bucket: formatWarehouseDateTime(minute - index * 60_000), total: 1000, @@ -104,6 +112,41 @@ describe("AlertsService SLO burn-rate rules", () => { }).pipe(Effect.provide(layer)) }) + it.effect("a page stays open while the service sends no spans at all", () => { + const traffic: Traffic = { badPerMille: 20 } + const { layer } = harness(traffic) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + assert.strictEqual((yield* alerts.listIncidents(orgId)).incidents[0]?.status, "open") + + // An outage zeroes every window: that is no data, not a recovery. + traffic.silent = true + yield* TestClock.adjust(Duration.minutes(1)) + yield* alerts.runSchedulerTick() + const held = (yield* alerts.listIncidents(orgId)).incidents + assert.strictEqual(held[0]?.status, "open") + assert.strictEqual(held[0]?.severity, "critical") + }).pipe(Effect.provide(layer)) + }) + + it.effect("an edit brings back a rule whose row is gone", () => { + const { layer, testDb } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const slos = yield* SlosService + const slo = yield* createSlo(undefined) + yield* Effect.promise(() => + executeSql(testDb, "delete from alert_rules where id = $1", [slo.alertRuleId]), + ) + const updated = yield* slos.update(orgId, slo.id, { targetPercent: 99.5, updatedBy: userId }) + assert.strictEqual(updated.alertRuleId, slo.alertRuleId) + assert.isTrue(updated.alerting.enabled) + }).pipe(Effect.provide(layer)) + }) + it.effect("a slow burn opens a warning, not a page", () => { // 0.8% bad burns 8x: over the 6x ticket factor, under the 14.4x page factor. const { layer } = harness({ badPerMille: 8 }) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 4fc833a93a..f9e9093137 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -1319,7 +1319,7 @@ export class AlertsService extends Context.Service): str export const formatObservedSummary = (context: ObservedContext): string => context.signalType === "uptime" ? uptimeObserved(context) - : context.value === null - ? `No data (threshold ${formatThresholdSummary(context)})` - : `${formatSignalMetric(context.value, signalDisplayOf(context))} ${formatThresholdSummary(context)}` + : context.signalType === "slo_burn_rate" && context.detail != null + ? context.detail + : context.value === null + ? `No data (threshold ${formatThresholdSummary(context)})` + : `${formatSignalMetric(context.value, signalDisplayOf(context))} ${formatThresholdSummary(context)}` /** * A notification's subject line without its emoji: `checkout-api is down` for an uptime rule, diff --git a/packages/backend/src/services/alerts/alert-persistence.ts b/packages/backend/src/services/alerts/alert-persistence.ts index 6916a783c8..7af309539f 100644 --- a/packages/backend/src/services/alerts/alert-persistence.ts +++ b/packages/backend/src/services/alerts/alert-persistence.ts @@ -1,4 +1,6 @@ -import { AlertPersistenceError } from "@maple/domain/http" +import * as Orm from "@maple-dev/effect-orm/database" +import type { MapleDb } from "@maple/db/client" +import { AlertPersistenceError, type OrgId } from "@maple/domain/http" import { makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" /** Shared `unknown -> AlertPersistenceError` mapper for every alerts service. */ @@ -6,3 +8,10 @@ export const makePersistenceError = makePersistenceErrorMapper( AlertPersistenceError, "Alert persistence failed", ) + +/** + * The per-org lock every alert rule write takes inside its transaction, so a name check or a + * destination check and the write that follows are atomic. + */ +export const lockOrgAlertRules = (db: MapleDb, orgId: OrgId) => + db.execute(Orm.sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) diff --git a/packages/backend/src/services/alerts/alert-signal-display.ts b/packages/backend/src/services/alerts/alert-signal-display.ts index 4a9587ccc6..008fe4dfbd 100644 --- a/packages/backend/src/services/alerts/alert-signal-display.ts +++ b/packages/backend/src/services/alerts/alert-signal-display.ts @@ -45,6 +45,7 @@ const PRESET_SIGNALS: Record = { apdex: { label: "Apdex", unit: "apdex" }, throughput: { label: "Throughput", unit: "rpm" }, uptime: { label: "Uptime check", unit: "plain" }, + slo_burn_rate: { label: "SLO burn rate", unit: "plain" }, } satisfies Record /** diff --git a/packages/backend/src/services/slos/SloAlertSource.ts b/packages/backend/src/services/slos/SloAlertSource.ts index 85d4f98cb0..eb731d6a02 100644 --- a/packages/backend/src/services/slos/SloAlertSource.ts +++ b/packages/backend/src/services/slos/SloAlertSource.ts @@ -6,6 +6,7 @@ import type { SloBurnWindow } from "@maple/domain/slo" import { Context, Effect, Layer, Option } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { makeSloPersistenceError } from "./slo-persistence" import { systemTenant } from "@maple/backend/services/alerts/system-tenant" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { readBurnWindows } from "./slo-reads" @@ -19,12 +20,6 @@ export interface SloAlertSourceApi { ) => Effect.Effect>, SloPersistenceError | WarehouseReadError> } -const toPersistenceError = (error: unknown) => - new SloPersistenceError({ - message: error instanceof Error ? error.message : "SLO persistence failed", - cause: error, - }) - /** * The alert evaluator's view of SLOs: its own service, with only the database and the * warehouse, so the alerting worker takes on nothing else of the SLO API. @@ -34,7 +29,7 @@ export class SloAlertSource extends Context.Service - new SloPersistenceError({ - message: error instanceof Error ? error.message : "SLO persistence failed", - cause: error, - }) +/** Each SLO's rule reads the warehouse every minute; the cap bounds that per org. */ +export const MAX_SLOS_PER_ORG = 50 const invalid = (message: string) => new SloValidationError({ message }) const notFound = (sloId: SloId) => new SloNotFoundError({ sloId, message: "SLO not found" }) @@ -227,10 +225,6 @@ const toRecord = (row: SloSelectRow): SloRecord => ({ updatedAt: row.updatedAt, }) -/** The per-org lock alert rule writes take, so a rule name check and its insert are atomic. */ -const lockOrgRules = (db: MapleDb, orgId: OrgId) => - db.execute(Orm.sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) - /** Checks a burn-rate rule write: its destinations exist and its name is free among alert rules. */ const claimRuleWrite = Effect.fnUntraced(function* ( db: MapleDb, @@ -282,7 +276,7 @@ export class SlosService extends Context.Service()( { make: Effect.gen(function* () { const database = yield* Database - const dbExecute = makeDbExecute(database, "SlosService", toPersistenceError) + const dbExecute = makeDbExecute(database, "SlosService", makeSloPersistenceError) const warehouse = yield* WarehouseQueryService const withWarehouse = Effect.provideService(WarehouseQueryService, warehouse) const nowMinute = Effect.map(Clock.currentTimeMillis, (now) => Math.floor(now / 60_000) * 60_000) @@ -341,7 +335,17 @@ export class SlosService extends Context.Service()( yield* dbExecute((db) => db.transaction( Effect.gen(function* () { - yield* lockOrgRules(db, orgId) + yield* lockOrgAlertRules(db, orgId) + const [existing] = yield* db.run( + PG.from(Slos) + .select(() => ({ count: PG.count() })) + .where(($) => [$.orgId.eq(orgId)]), + ) + if (Number(existing?.count ?? 0) >= MAX_SLOS_PER_ORG) { + return yield* invalid( + `An organization can have at most ${MAX_SLOS_PER_ORG} SLOs`, + ) + } yield* ensureNameFree(db, orgId, name, id) yield* claimRuleWrite(db, orgId, ruleId, rule) yield* db.run( @@ -407,7 +411,7 @@ export class SlosService extends Context.Service()( Effect.gen(function* () { // Lock, then re-read: the rule is rebuilt from a row no other write changes // before commit, and a concurrent delete either finished first or waits. - yield* lockOrgRules(db, orgId) + yield* lockOrgAlertRules(db, orgId) const lockedRows = yield* db.run( selectSlos() .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) @@ -416,15 +420,15 @@ export class SlosService extends Context.Service()( const lockedRow = lockedRows[0] if (lockedRow === undefined) return yield* notFound(sloId) const locked = toRecord(lockedRow) + // No rule yet (saved before burn alerts), or its row is gone: write one. + const ruleMissing = locked.alertRuleId === null || lockedRow.ruleEnabled === null if (name !== undefined) yield* ensureNameFree(db, orgId, name, sloId) const rule: SloRuleInput = { name: name ?? locked.name, serviceName: serviceName ?? locked.serviceName, environments: environments ?? locked.environments, - // An SLO saved before burn alerts existed gets its rule switched on. - enabled: - input.alerting?.enabled ?? - (locked.alertRuleId === null || locked.alerting.enabled), + // A missing rule comes back switched on. + enabled: input.alerting?.enabled ?? (ruleMissing || locked.alerting.enabled), criticalDestinationIds: input.alerting?.criticalDestinationIds === undefined ? locked.alerting.criticalDestinationIds @@ -436,7 +440,7 @@ export class SlosService extends Context.Service()( } // The rule is written only when it changes: alert_rules is synced to browsers. const ruleChanged = - locked.alertRuleId === null || + ruleMissing || rule.name !== locked.name || rule.serviceName !== locked.serviceName || !sameIds(rule.environments, locked.environments) || @@ -449,7 +453,7 @@ export class SlosService extends Context.Service()( const ruleId = locked.alertRuleId ?? AlertRuleId.make(randomUUID()) if (ruleChanged) { yield* claimRuleWrite(db, orgId, ruleId, rule) - if (locked.alertRuleId === null) { + if (ruleMissing) { yield* db.run( PG.insertInto(AlertRules).values({ id: ruleId, @@ -512,7 +516,7 @@ export class SlosService extends Context.Service()( const removed = yield* dbExecute((db) => db.transaction( Effect.gen(function* () { - yield* lockOrgRules(db, orgId) + yield* lockOrgAlertRules(db, orgId) const rows = yield* db.run( PG.deleteFrom(Slos) .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) diff --git a/packages/backend/src/services/slos/slo-alert-rule.ts b/packages/backend/src/services/slos/slo-alert-rule.ts index 951030200a..5f2f8c1994 100644 --- a/packages/backend/src/services/slos/slo-alert-rule.ts +++ b/packages/backend/src/services/slos/slo-alert-rule.ts @@ -4,7 +4,9 @@ import type { AlertDestinationId, AlertThresholds } from "@maple/domain/http" import { SLO_BURN_TIERS, SLO_BURN_LOOKBACK_MINUTES, + burnWindowOf, tierBreached, + tierObserved, type SloBurnTier, type SloBurnWindow, } from "@maple/domain/slo" @@ -83,20 +85,13 @@ export const sloRuleFields = (slo: SloRuleInput) => { } as const } -const rateOf = (windows: ReadonlyArray, minutes: number) => - Arr.findFirst(windows, (window) => window.windowMinutes === minutes) - const formatRate = (rate: number) => `${rate >= 100 ? Math.round(rate) : Math.round(rate * 10) / 10}x` const describeTier = (tier: SloBurnTier, windows: ReadonlyArray) => { - const long = Option.match(rateOf(windows, tier.longWindowMinutes), { - onNone: () => 0, - onSome: (w) => w.burnRate, - }) - const short = Option.match(rateOf(windows, tier.shortWindowMinutes), { - onNone: () => 0, - onSome: (w) => w.burnRate, - }) + const rate = (minutes: number) => + Option.match(burnWindowOf(windows, minutes), { onNone: () => 0, onSome: (window) => window.burnRate }) + const long = rate(tier.longWindowMinutes) + const short = rate(tier.shortWindowMinutes) const label = (minutes: number) => (minutes >= 60 ? `${minutes / 60}h` : `${minutes}m`) return { long, @@ -106,29 +101,50 @@ const describeTier = (tier: SloBurnTier, windows: ReadonlyArray) /** * One check of an SLO's burn-rate rule. Each tier is judged on its own pair of windows, so the - * levels are set here rather than by comparing one value to every tier. The value is the - * long-window burn of the highest tier that fires, else the 1h burn. + * levels are set here rather than by comparing one value to every tier. A tier whose short + * window saw no events holds its state; with every tier held the check is skipped as no data. + * The value is the long-window burn of the highest tier that fires, else the 1h burn. */ export const evaluateSloBurn = (windows: ReadonlyArray): AlertEvaluation => { - const levels = Arr.map([WARNING_TIER, CRITICAL_TIER], (tier) => ({ + const tiers = [WARNING_TIER, CRITICAL_TIER] + const levels = Arr.map(tiers, (tier) => ({ severity: tier.severity, - status: tierBreached(tier, windows) ? ("breach" as const) : ("clear" as const), + status: !tierObserved(tier, windows) + ? ("hold" as const) + : tierBreached(tier, windows) + ? ("breach" as const) + : ("clear" as const), })) - const firing = Arr.findFirst([CRITICAL_TIER, WARNING_TIER], (tier) => tierBreached(tier, windows)) + const hour = Option.getOrUndefined(burnWindowOf(windows, CRITICAL_TIER.longWindowMinutes)) + const base = { + sampleCount: Math.round(hour?.total ?? 0), + threshold: WARNING_TIER.factor, + thresholdUpper: null, + comparator: "gte" as const, + derivedFromNoData: false, + } + if (Arr.every(levels, (level) => level.status === "hold")) { + return { + ...base, + status: "skipped", + value: null, + reason: "No events in the SLO's short burn windows", + skipReason: "no_data", + levels: [], + } + } + const firing = Arr.findFirst([CRITICAL_TIER, WARNING_TIER], (tier) => + Arr.some(levels, (level) => level.severity === tier.severity && level.status === "breach"), + ) const headline = describeTier( Option.getOrElse(firing, () => CRITICAL_TIER), windows, ) - const hour = Option.getOrUndefined(rateOf(windows, CRITICAL_TIER.longWindowMinutes)) return { + ...base, status: Option.isSome(firing) ? "breached" : "healthy", levels, value: headline.long, - sampleCount: Math.round(hour?.total ?? 0), - threshold: WARNING_TIER.factor, - thresholdUpper: null, - comparator: "gte", reason: Option.isSome(firing) ? `SLO ${headline.text}` : `SLO within budget: ${headline.text}`, - derivedFromNoData: false, } } diff --git a/packages/backend/src/services/slos/slo-persistence.ts b/packages/backend/src/services/slos/slo-persistence.ts new file mode 100644 index 0000000000..f2e62ec5bb --- /dev/null +++ b/packages/backend/src/services/slos/slo-persistence.ts @@ -0,0 +1,8 @@ +import { SloPersistenceError } from "@maple/domain/http" +import { makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" + +/** Shared `unknown -> SloPersistenceError` mapper for the SLO services. */ +export const makeSloPersistenceError = makePersistenceErrorMapper( + SloPersistenceError, + "SLO persistence failed", +) diff --git a/packages/backend/src/services/slos/slo-reads.ts b/packages/backend/src/services/slos/slo-reads.ts index f431b6e885..8622ed13e7 100644 --- a/packages/backend/src/services/slos/slo-reads.ts +++ b/packages/backend/src/services/slos/slo-reads.ts @@ -95,7 +95,7 @@ export const readBudgetHistory = Effect.fn("SloReads.budgetHistory")(function* ( tenant, CH.compile(CH.sloBucketsQuery(queryOpts(slo), "hour"), { orgId: slo.orgId, - startTime: formatWarehouseDateTime(Math.ceil(endMs / HOUR_MS) * HOUR_MS - 2 * windowMs), + startTime: formatWarehouseDateTime(Math.ceil(endMs / HOUR_MS) * HOUR_MS - 2 * windowMs + HOUR_MS), endTime: formatWarehouseDateTime(endMs), bucketSeconds: 3600, }), diff --git a/packages/domain/src/slo.ts b/packages/domain/src/slo.ts index 78dac660ad..94c305630b 100644 --- a/packages/domain/src/slo.ts +++ b/packages/domain/src/slo.ts @@ -68,14 +68,29 @@ export interface SloBurnWindow extends SloCounts { readonly burnRate: number } +/** The window of the given length, if read. */ +export const burnWindowOf = ( + windows: ReadonlyArray, + minutes: number, +): Option.Option => Arr.findFirst(windows, (window) => window.windowMinutes === minutes) + +const burnOf = (windows: ReadonlyArray, minutes: number): number => + Option.match(burnWindowOf(windows, minutes), { onNone: () => 0, onSome: (window) => window.burnRate }) + /** Whether a tier fires on the given window burns. */ -export const tierBreached = (tier: SloBurnTier, windows: ReadonlyArray): boolean => { - const rateOf = (minutes: number) => - Arr.findFirst(windows, (window) => window.windowMinutes === minutes).pipe( - Option.match({ onNone: () => 0, onSome: (window) => window.burnRate }), - ) - return rateOf(tier.longWindowMinutes) >= tier.factor && rateOf(tier.shortWindowMinutes) >= tier.factor -} +export const tierBreached = (tier: SloBurnTier, windows: ReadonlyArray): boolean => + burnOf(windows, tier.longWindowMinutes) >= tier.factor && + burnOf(windows, tier.shortWindowMinutes) >= tier.factor + +/** + * Whether a tier can be judged: its short window saw events. An empty short window is an + * outage or an ingest gap as often as a quiet service, so it neither fires nor resolves. + */ +export const tierObserved = (tier: SloBurnTier, windows: ReadonlyArray): boolean => + Option.match(burnWindowOf(windows, tier.shortWindowMinutes), { + onNone: () => false, + onSome: (window) => window.total > 0, + }) /** A bucket of events starting at `startMs`. */ export interface SloBucket extends SloCounts { @@ -114,7 +129,7 @@ export interface SloHistoryPoint { } /** - * The budget chart from buckets covering two SLO windows: one point per bucket of the last + * The budget chart from buckets covering two SLO windows less one bucket: one point per bucket of the last * window, each reading the SLO window that ends with it. Buckets are `bucketMs` wide and * aligned to it; the last point is `endMs`, inside the newest (partial) bucket. */ @@ -126,7 +141,8 @@ export const budgetHistory = (input: { readonly targetPercent: number }): ReadonlyArray => { const slotsPerWindow = Math.round(input.windowMs / input.bucketMs) - const slotCount = 2 * slotsPerWindow + // The oldest point's window starts one bucket after the first slot's start. + const slotCount = 2 * slotsPerWindow - 1 const alignedEnd = Math.ceil(input.endMs / input.bucketMs) * input.bucketMs const origin = alignedEnd - slotCount * input.bucketMs const slots = Arr.makeBy(slotCount, () => ({ total: 0, good: 0 })) @@ -147,7 +163,7 @@ export const budgetHistory = (input: { return { total: end.total - start.total, good: end.good - start.good } } return Arr.makeBy(slotsPerWindow, (index) => { - const slot = slotsPerWindow + index + const slot = slotsPerWindow - 1 + index const last = between(slot, slot + 1) return { atMs: Math.min(origin + (slot + 1) * input.bucketMs, input.endMs), From 8783e35f3bacd887fd1ba5aac5b70c5186ffba97 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:56:37 +0100 Subject: [PATCH 04/11] refactor(slos): tier state, window labels and rule name in @maple/domain/slo The UI and MCP read the same breach/hold/clear verdict the alert evaluator uses, and find an SLO's name in its rule name with one helper. --- .../backend/src/services/slos/SlosService.ts | 3 ++- .../backend/src/services/slos/slo-alert-rule.ts | 17 +++++------------ packages/domain/src/slo.ts | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/services/slos/SlosService.ts b/packages/backend/src/services/slos/SlosService.ts index c9cb1657f5..a70acc8ea9 100644 --- a/packages/backend/src/services/slos/SlosService.ts +++ b/packages/backend/src/services/slos/SlosService.ts @@ -22,6 +22,7 @@ import { SloTargetPercent, type SloBurnWindow, type SloHistoryPoint, + sloRuleName, type SloSli, } from "@maple/domain/slo" import { Array as Arr, Clock, Context, Effect, Layer, Option, Schema, String as Str } from "effect" @@ -34,7 +35,7 @@ import { WarehouseQueryService } from "@maple/backend/services/warehouse/Warehou import { deleteRuleRows } from "@maple/backend/services/alerts/AlertRulesService" import { lockOrgAlertRules } from "@maple/backend/services/alerts/alert-persistence" import { readBudgetHistory, readBurnWindows, readWindowTotals } from "./slo-reads" -import { sloRuleFields, sloRuleName, type SloRuleInput } from "./slo-alert-rule" +import { sloRuleFields, type SloRuleInput } from "./slo-alert-rule" export interface SloRecord { readonly id: SloId diff --git a/packages/backend/src/services/slos/slo-alert-rule.ts b/packages/backend/src/services/slos/slo-alert-rule.ts index 5f2f8c1994..91bcaf60d7 100644 --- a/packages/backend/src/services/slos/slo-alert-rule.ts +++ b/packages/backend/src/services/slos/slo-alert-rule.ts @@ -5,8 +5,9 @@ import { SLO_BURN_TIERS, SLO_BURN_LOOKBACK_MINUTES, burnWindowOf, - tierBreached, - tierObserved, + sloRuleName, + tierState, + windowLabel, type SloBurnTier, type SloBurnWindow, } from "@maple/domain/slo" @@ -15,9 +16,6 @@ import { MANAGED_RULE_PLAN } from "@maple/backend/services/alerts/AlertRuleModel const [CRITICAL_TIER, WARNING_TIER] = SLO_BURN_TIERS -/** The rule's name: it titles incidents and notifications, so it says what is burning. */ -export const sloRuleName = (sloName: string) => `SLO burn: ${sloName}` - export interface SloRuleInput { readonly name: string readonly serviceName: string @@ -92,10 +90,9 @@ const describeTier = (tier: SloBurnTier, windows: ReadonlyArray) Option.match(burnWindowOf(windows, minutes), { onNone: () => 0, onSome: (window) => window.burnRate }) const long = rate(tier.longWindowMinutes) const short = rate(tier.shortWindowMinutes) - const label = (minutes: number) => (minutes >= 60 ? `${minutes / 60}h` : `${minutes}m`) return { long, - text: `burning the error budget at ${formatRate(long)} over ${label(tier.longWindowMinutes)} and ${formatRate(short)} over ${label(tier.shortWindowMinutes)} (${tier.severity} at ${tier.factor}x)`, + text: `burning the error budget at ${formatRate(long)} over ${windowLabel(tier.longWindowMinutes)} and ${formatRate(short)} over ${windowLabel(tier.shortWindowMinutes)} (${tier.severity} at ${tier.factor}x)`, } } @@ -109,11 +106,7 @@ export const evaluateSloBurn = (windows: ReadonlyArray): AlertEva const tiers = [WARNING_TIER, CRITICAL_TIER] const levels = Arr.map(tiers, (tier) => ({ severity: tier.severity, - status: !tierObserved(tier, windows) - ? ("hold" as const) - : tierBreached(tier, windows) - ? ("breach" as const) - : ("clear" as const), + status: tierState(tier, windows), })) const hour = Option.getOrUndefined(burnWindowOf(windows, CRITICAL_TIER.longWindowMinutes)) const base = { diff --git a/packages/domain/src/slo.ts b/packages/domain/src/slo.ts index 94c305630b..9d6d7b231f 100644 --- a/packages/domain/src/slo.ts +++ b/packages/domain/src/slo.ts @@ -92,6 +92,22 @@ export const tierObserved = (tier: SloBurnTier, windows: ReadonlyArray window.total > 0, }) +/** How a tier reads now: firing, holding its state (no events in its short window), or clear. */ +export type SloTierState = "breach" | "hold" | "clear" + +export const tierState = (tier: SloBurnTier, windows: ReadonlyArray): SloTierState => + !tierObserved(tier, windows) ? "hold" : tierBreached(tier, windows) ? "breach" : "clear" + +/** `5m`, `30m`, `1h`, `6h`. */ +export const windowLabel = (minutes: number): string => (minutes >= 60 ? `${minutes / 60}h` : `${minutes}m`) + +/** The hidden burn-rate rule's name: it titles incidents and notifications. */ +export const SLO_RULE_NAME_PREFIX = "SLO burn: " +export const sloRuleName = (sloName: string): string => `${SLO_RULE_NAME_PREFIX}${sloName}` +/** The SLO name inside a burn-rate rule's name, or `null` for any other rule name. */ +export const sloNameFromRuleName = (ruleName: string): string | null => + ruleName.startsWith(SLO_RULE_NAME_PREFIX) ? ruleName.slice(SLO_RULE_NAME_PREFIX.length) : null + /** A bucket of events starting at `startMs`. */ export interface SloBucket extends SloCounts { readonly startMs: number From 491af50d9d0b1228ab5b786b1fdcd82f7d50ce61 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 01:56:46 +0100 Subject: [PATCH 05/11] style(slos): effect String helpers in sloNameFromRuleName --- packages/domain/src/slo.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/domain/src/slo.ts b/packages/domain/src/slo.ts index 9d6d7b231f..1cdf831371 100644 --- a/packages/domain/src/slo.ts +++ b/packages/domain/src/slo.ts @@ -1,4 +1,4 @@ -import { Array as Arr, Option, Schema } from "effect" +import { Array as Arr, Option, Schema, String as Str } from "effect" import { SloLatencyThresholdMs } from "./primitives" // Service level objectives: what counts as a good event (the SLI), the target share of good @@ -106,7 +106,7 @@ export const SLO_RULE_NAME_PREFIX = "SLO burn: " export const sloRuleName = (sloName: string): string => `${SLO_RULE_NAME_PREFIX}${sloName}` /** The SLO name inside a burn-rate rule's name, or `null` for any other rule name. */ export const sloNameFromRuleName = (ruleName: string): string | null => - ruleName.startsWith(SLO_RULE_NAME_PREFIX) ? ruleName.slice(SLO_RULE_NAME_PREFIX.length) : null + Str.startsWith(SLO_RULE_NAME_PREFIX)(ruleName) ? Str.slice(SLO_RULE_NAME_PREFIX.length)(ruleName) : null /** A bucket of events starting at `startMs`. */ export interface SloBucket extends SloCounts { From bff51a8a5e3178f54e3f2fc781da8a99c254e90e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 02:51:44 +0100 Subject: [PATCH 06/11] fix(slos): a recovery message names every burn tier The healthy reason described only the critical tier, so a resolved warning reported the 1h and 5m windows instead of the 6h and 30m it fired on. --- .../src/services/slos/slo-alert-rule.test.ts | 40 +++++++++++++++++++ .../src/services/slos/slo-alert-rule.ts | 12 +++++- 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 packages/backend/src/services/slos/slo-alert-rule.test.ts diff --git a/packages/backend/src/services/slos/slo-alert-rule.test.ts b/packages/backend/src/services/slos/slo-alert-rule.test.ts new file mode 100644 index 0000000000..825a7fafb3 --- /dev/null +++ b/packages/backend/src/services/slos/slo-alert-rule.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest" +import { Array as Arr } from "effect" +import { SLO_BURN_WINDOWS_MINUTES, type SloBurnWindow } from "@maple/domain/slo" +import { evaluateSloBurn } from "./slo-alert-rule" + +/** Every burn window at the given rates, keyed by window length; 1000 events each. */ +const windows = (rates: Record): ReadonlyArray => + Arr.map(SLO_BURN_WINDOWS_MINUTES, (windowMinutes) => ({ + windowMinutes, + total: 1000, + good: 1000, + burnRate: rates[windowMinutes] ?? 0, + })) + +describe("evaluateSloBurn", () => { + it("names the firing tier's windows", () => { + const evaluation = evaluateSloBurn(windows({ 5: 8, 30: 8, 60: 7, 360: 7 })) + expect(evaluation.status).toBe("breached") + expect(evaluation.reason).toBe( + "SLO burning the error budget at 7x over 6h and 8x over 30m (warning at 6x)", + ) + }) + + it("names every tier on recovery, so a resolved warning reports its own windows", () => { + const evaluation = evaluateSloBurn(windows({ 5: 0.2, 30: 0.3, 60: 0.4, 360: 0.5 })) + expect(evaluation.status).toBe("healthy") + expect(evaluation.reason).toBe( + "SLO within budget: 0.4x over 1h and 0.2x over 5m (critical at 14.4x); 0.5x over 6h and 0.3x over 30m (warning at 6x)", + ) + }) + + it("skips as no data when every short window is empty", () => { + const empty = Arr.map(windows({}), (window) => + window.windowMinutes === 5 || window.windowMinutes === 30 + ? { ...window, total: 0, good: 0 } + : window, + ) + expect(evaluateSloBurn(empty)).toMatchObject({ status: "skipped", skipReason: "no_data", levels: [] }) + }) +}) diff --git a/packages/backend/src/services/slos/slo-alert-rule.ts b/packages/backend/src/services/slos/slo-alert-rule.ts index 91bcaf60d7..5cf620eab2 100644 --- a/packages/backend/src/services/slos/slo-alert-rule.ts +++ b/packages/backend/src/services/slos/slo-alert-rule.ts @@ -90,9 +90,11 @@ const describeTier = (tier: SloBurnTier, windows: ReadonlyArray) Option.match(burnWindowOf(windows, minutes), { onNone: () => 0, onSome: (window) => window.burnRate }) const long = rate(tier.longWindowMinutes) const short = rate(tier.shortWindowMinutes) + const rates = `${formatRate(long)} over ${windowLabel(tier.longWindowMinutes)} and ${formatRate(short)} over ${windowLabel(tier.shortWindowMinutes)}` return { long, - text: `burning the error budget at ${formatRate(long)} over ${windowLabel(tier.longWindowMinutes)} and ${formatRate(short)} over ${windowLabel(tier.shortWindowMinutes)} (${tier.severity} at ${tier.factor}x)`, + text: `burning the error budget at ${rates} (${tier.severity} at ${tier.factor}x)`, + rates: `${rates} (${tier.severity} at ${tier.factor}x)`, } } @@ -138,6 +140,12 @@ export const evaluateSloBurn = (windows: ReadonlyArray): AlertEva status: Option.isSome(firing) ? "breached" : "healthy", levels, value: headline.long, - reason: Option.isSome(firing) ? `SLO ${headline.text}` : `SLO within budget: ${headline.text}`, + // A recovery names every tier, so a resolved warning reports the windows it fired on. + reason: Option.isSome(firing) + ? `SLO ${headline.text}` + : `SLO within budget: ${Arr.join( + Arr.map([CRITICAL_TIER, WARNING_TIER], (tier) => describeTier(tier, windows).rates), + "; ", + )}`, } } From b4fc336e9a5a0716626c71088b36f641617892fd Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 02:53:01 +0100 Subject: [PATCH 07/11] chore(slos): regenerate the iOS OpenAPI spec --- apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index e8c0b0320f..4cc5df05c0 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -6971,7 +6971,8 @@ "throughput", "builder_query", "raw_query", - "uptime" + "uptime", + "slo_burn_rate" ], "examples": [ "error_rate" From 692cb65cc1f990d2494aef94a67a01a6a7dd7cab Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 03:13:28 +0100 Subject: [PATCH 08/11] fix(ios): label the slo_burn_rate signal type --- apps/ios/Maple/Components/AlertFormatting.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/ios/Maple/Components/AlertFormatting.swift b/apps/ios/Maple/Components/AlertFormatting.swift index 63dfe78165..f4c708f254 100644 --- a/apps/ios/Maple/Components/AlertFormatting.swift +++ b/apps/ios/Maple/Components/AlertFormatting.swift @@ -46,6 +46,8 @@ struct SignalDisplay: Codable { case .rawQuery: self.init(label: "Raw query", unit: .plain) // A monitor's hidden rule: down is 1, up is 0, so the value reads as a plain number. case .uptime: self.init(label: "Uptime check", unit: .plain) + // An SLO's hidden rule: the value is how many times faster than sustainable the budget burns. + case .sloBurnRate: self.init(label: "SLO burn rate", unit: .plain) } } From c865aad7a9cd2600d3dd300da6b7f5e35bd3ad0e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 03:22:18 +0100 Subject: [PATCH 09/11] fix(slos): burn rates read as 14.4x in notifications and on iOS; tests for the cap and a held tier A burn signal unit on the server and on iOS keeps the multiplier and one decimal, matching the web. isManagedSignalType reads MANAGED_SIGNAL_TYPES. --- .../Maple/Components/AlertFormatting.swift | 5 ++++- .../services/alerts/AlertsService.slo.test.ts | 22 ++++++++++++++++++- .../src/services/alerts/alert-formatting.ts | 1 + .../services/alerts/alert-signal-display.ts | 6 +++-- .../src/services/slos/slo-alert-rule.test.ts | 13 +++++++++++ packages/domain/src/http/alerts.ts | 2 +- packages/web-data/src/alerts/overview.test.ts | 18 +++++++++++---- 7 files changed, 58 insertions(+), 9 deletions(-) diff --git a/apps/ios/Maple/Components/AlertFormatting.swift b/apps/ios/Maple/Components/AlertFormatting.swift index f4c708f254..bdbeb3db82 100644 --- a/apps/ios/Maple/Components/AlertFormatting.swift +++ b/apps/ios/Maple/Components/AlertFormatting.swift @@ -15,6 +15,8 @@ enum SignalUnit: Codable { case apdex case count case plain + /// An SLO burn rate: a multiple of the sustainable rate, shown as `14.4x` like the web. + case burnRate func format(_ value: Double?) -> String { guard let value, value.isFinite else { return Format.placeholder } @@ -24,6 +26,7 @@ enum SignalUnit: Codable { case .perMinute: return Format.count(value) + "/min" case .apdex: return String(format: "%.2f", value) case .count, .plain: return Format.count(value) + case .burnRate: return String(format: "%.1fx", value) } } } @@ -47,7 +50,7 @@ struct SignalDisplay: Codable { // A monitor's hidden rule: down is 1, up is 0, so the value reads as a plain number. case .uptime: self.init(label: "Uptime check", unit: .plain) // An SLO's hidden rule: the value is how many times faster than sustainable the budget burns. - case .sloBurnRate: self.init(label: "SLO burn rate", unit: .plain) + case .sloBurnRate: self.init(label: "SLO burn rate", unit: .burnRate) } } diff --git a/packages/backend/src/services/alerts/AlertsService.slo.test.ts b/packages/backend/src/services/alerts/AlertsService.slo.test.ts index f1eb7c1e1c..c6a68e8aff 100644 --- a/packages/backend/src/services/alerts/AlertsService.slo.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.slo.test.ts @@ -14,7 +14,7 @@ import { } from "@maple/backend/platform/test-pglite" import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" -import { SlosService } from "@maple/backend/services/slos/SlosService" +import { MAX_SLOS_PER_ORG, SlosService } from "@maple/backend/services/slos/SlosService" import { DEFAULT_CLOCK_EPOCH_MS, adminRoles, @@ -204,6 +204,26 @@ describe("AlertsService SLO burn-rate rules", () => { }).pipe(Effect.provide(layer)) }) + it.effect("caps an org at MAX_SLOS_PER_ORG SLOs", () => { + const { layer } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const slos = yield* SlosService + const create = (index: number) => + slos.create(orgId, { + name: `SLO ${index}`, + serviceName: "checkout", + sli: { _tag: "availability" }, + targetPercent: 99, + createdBy: userId, + }) + yield* Effect.forEach(Arr.range(1, MAX_SLOS_PER_ORG), create, { discard: true }) + const refused = yield* create(MAX_SLOS_PER_ORG + 1).pipe(Effect.flip) + assert.strictEqual(refused._tag, "@maple/http/errors/SloValidationError") + assert.lengthOf(yield* slos.list(orgId), MAX_SLOS_PER_ORG) + }).pipe(Effect.provide(layer)) + }) + it.effect("refuses an unknown destination and keeps the SLO unsaved", () => { const { layer } = harness({ badPerMille: 0 }) return Effect.gen(function* () { diff --git a/packages/backend/src/services/alerts/alert-formatting.ts b/packages/backend/src/services/alerts/alert-formatting.ts index ca3f8aeee0..8585710c62 100644 --- a/packages/backend/src/services/alerts/alert-formatting.ts +++ b/packages/backend/src/services/alerts/alert-formatting.ts @@ -133,6 +133,7 @@ export const formatSignalMetric = (value: number | null, display: SignalDisplay) Match.when("apdex", () => round(metric, 3)), Match.when("rpm", () => `${grouped(metric)} rpm`), Match.whenOr("count", "plain", () => grouped(metric)), + Match.when("burn", () => `${round(metric, 1)}x`), Match.exhaustive, ), }) diff --git a/packages/backend/src/services/alerts/alert-signal-display.ts b/packages/backend/src/services/alerts/alert-signal-display.ts index 008fe4dfbd..c20b58a016 100644 --- a/packages/backend/src/services/alerts/alert-signal-display.ts +++ b/packages/backend/src/services/alerts/alert-signal-display.ts @@ -19,7 +19,8 @@ import { AGGREGATIONS_BY_SOURCE } from "@maple/query-engine/query-builder" import type { ChartUnit } from "@maple/widgets/chart/static-chart" import { Array as Arr, String as Str } from "effect" -export type SignalUnit = "ratio" | "ms" | "rpm" | "apdex" | "count" | "plain" +/** `burn`: an SLO burn rate, a multiple of the sustainable rate shown as `14.4x`. */ +export type SignalUnit = "ratio" | "ms" | "rpm" | "apdex" | "count" | "plain" | "burn" export interface SignalDisplay { /** Human-readable name of the measured quantity, e.g. `sum(db.query.duration)`. */ @@ -45,7 +46,7 @@ const PRESET_SIGNALS: Record = { apdex: { label: "Apdex", unit: "apdex" }, throughput: { label: "Throughput", unit: "rpm" }, uptime: { label: "Uptime check", unit: "plain" }, - slo_burn_rate: { label: "SLO burn rate", unit: "plain" }, + slo_burn_rate: { label: "SLO burn rate", unit: "burn" }, } satisfies Record /** @@ -147,6 +148,7 @@ export const mapSignalUnit = (unit: SignalUnit): ChartUnit => { case "apdex": case "count": case "plain": + case "burn": return "number" } } diff --git a/packages/backend/src/services/slos/slo-alert-rule.test.ts b/packages/backend/src/services/slos/slo-alert-rule.test.ts index 825a7fafb3..1fb9a9c8d0 100644 --- a/packages/backend/src/services/slos/slo-alert-rule.test.ts +++ b/packages/backend/src/services/slos/slo-alert-rule.test.ts @@ -29,6 +29,19 @@ describe("evaluateSloBurn", () => { ) }) + it("holds the critical tier while its 5m window is empty and still fires the warning", () => { + const evaluation = evaluateSloBurn( + Arr.map(windows({ 30: 8, 60: 20, 360: 7 }), (window) => + window.windowMinutes === 5 ? { ...window, total: 0, good: 0 } : window, + ), + ) + expect(evaluation.status).toBe("breached") + expect(evaluation.levels).toEqual([ + { severity: "warning", status: "breach" }, + { severity: "critical", status: "hold" }, + ]) + }) + it("skips as no data when every short window is empty", () => { const empty = Arr.map(windows({}), (window) => window.windowMinutes === 5 || window.windowMinutes === 30 diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index 2000945b0f..297382782f 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -64,7 +64,7 @@ export const MANAGED_SIGNAL_TYPES = [ ] as const satisfies ReadonlyArray export type ManagedSignalType = (typeof MANAGED_SIGNAL_TYPES)[number] export const isManagedSignalType = (signalType: string): signalType is ManagedSignalType => - signalType === "uptime" || signalType === "slo_burn_rate" + Arr.some(MANAGED_SIGNAL_TYPES, (managed) => managed === signalType) export const AlertGroupByDimension = Schema.String.check(Schema.isMinLength(1), Schema.isTrimmed()).annotate({ identifier: "@maple/AlertGroupByDimension", diff --git a/packages/web-data/src/alerts/overview.test.ts b/packages/web-data/src/alerts/overview.test.ts index a06be184cd..2d8cbd29fe 100644 --- a/packages/web-data/src/alerts/overview.test.ts +++ b/packages/web-data/src/alerts/overview.test.ts @@ -1,5 +1,5 @@ import { AlertIncidentDocument, AlertRuleDocument } from "@maple/domain/http" -import { Schema, Array as Arr } from "effect" +import { Order, Schema, Array as Arr } from "effect" import { describe, expect, it } from "vitest" import type { AlertRuleStateRow } from "../collections/alerts" import { deriveOverview } from "./overview" @@ -13,6 +13,7 @@ const iso = (msAgo: number) => new Date(NOW - msAgo).toISOString() const RULE_A = "00000000-0000-4000-8000-00000000000a" const RULE_B = "00000000-0000-4000-8000-00000000000b" +const RULE_C = "00000000-0000-4000-8000-00000000000c" function makeRule(overrides: Record = {}): AlertRuleDocument { return decodeRuleDoc({ @@ -204,18 +205,27 @@ describe("deriveOverview", () => { expect(overview.timelineRange).toEqual({ min: NOW - DAY_MS, max: NOW }) }) - it("hides uptime monitors' backing rules but keeps their incidents in the inbox", () => { + it("hides monitors' and SLOs' backing rules but keeps their incidents in the inbox", () => { const overview = deriveOverview({ rules: [ makeRule({ id: RULE_A }), makeRule({ id: RULE_B, signalType: "uptime", name: "API health" }), + makeRule({ id: RULE_C, signalType: "slo_burn_rate", name: "SLO burn: Checkout" }), + ], + incidents: [ + makeIncident({ ruleId: RULE_B, ruleName: "API health", signalType: "uptime" }), + makeIncident({ ruleId: RULE_C, ruleName: "SLO burn: Checkout", signalType: "slo_burn_rate" }), ], - incidents: [makeIncident({ ruleId: RULE_B, ruleName: "API health", signalType: "uptime" })], states: [], deliveryEvents: [], now: NOW, }) expect(Arr.map(overview.rules, (rule) => rule.id)).toEqual([RULE_A]) - expect(Arr.map(overview.openIncidents, (incident) => incident.ruleName)).toEqual(["API health"]) + expect( + Arr.sort( + Arr.map(overview.openIncidents, (incident) => incident.ruleName), + Order.String, + ), + ).toEqual(["API health", "SLO burn: Checkout"]) }) }) From a4cf3d1cdeb4cbea3ef1bc395771b93b7fed110e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 03:27:17 +0100 Subject: [PATCH 10/11] test(slos): a held critical tier keeps its page open while the warning tier clears --- .../services/alerts/AlertsService.slo.test.ts | 39 ++++++++++++++++--- .../src/services/slos/slo-alert-rule.test.ts | 14 +++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/services/alerts/AlertsService.slo.test.ts b/packages/backend/src/services/alerts/AlertsService.slo.test.ts index c6a68e8aff..e0ee99d9b9 100644 --- a/packages/backend/src/services/alerts/AlertsService.slo.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.slo.test.ts @@ -32,6 +32,8 @@ interface Traffic { badPerMille: number /** No spans at all: the service is down or ingest stalled. */ silent?: boolean + /** No spans in only the newest minutes; older minutes keep their traffic. */ + silentMinutes?: number } /** @@ -46,11 +48,17 @@ const sloWarehouse = (traffic: Traffic) => const minute = Math.floor(nowMs / 60_000) * 60_000 const rows = options?.context === "sloBurnWindows" && traffic.silent !== true - ? Arr.makeBy(400, (index) => ({ - bucket: formatWarehouseDateTime(minute - index * 60_000), - total: 1000, - good: 1000 - traffic.badPerMille, - })) + ? Arr.map( + Arr.filter( + Arr.makeBy(400, (index) => index), + (index) => index >= (traffic.silentMinutes ?? 0), + ), + (index) => ({ + bucket: formatWarehouseDateTime(minute - index * 60_000), + total: 1000, + good: 1000 - traffic.badPerMille, + }), + ) : [] return yield* compiledQueryOf(compiled).decodeRows(rows).pipe(Effect.orDie) }), @@ -132,6 +140,27 @@ describe("AlertsService SLO burn-rate rules", () => { }).pipe(Effect.provide(layer)) }) + it.effect("a page stays open while its 5m window is empty, even once the warning tier clears", () => { + const traffic: Traffic = { badPerMille: 20 } + const { layer } = harness(traffic) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + assert.strictEqual((yield* alerts.listIncidents(orgId)).incidents[0]?.severity, "critical") + + // The newest minutes go quiet and the older ones turn healthy: warning clears, critical holds. + traffic.badPerMille = 0 + traffic.silentMinutes = 6 + yield* TestClock.adjust(Duration.minutes(1)) + yield* alerts.runSchedulerTick() + const held = (yield* alerts.listIncidents(orgId)).incidents + assert.strictEqual(held[0]?.status, "open") + assert.strictEqual(held[0]?.severity, "critical") + }).pipe(Effect.provide(layer)) + }) + it.effect("an edit brings back a rule whose row is gone", () => { const { layer, testDb } = harness({ badPerMille: 0 }) return Effect.gen(function* () { diff --git a/packages/backend/src/services/slos/slo-alert-rule.test.ts b/packages/backend/src/services/slos/slo-alert-rule.test.ts index 1fb9a9c8d0..ec8339a6c5 100644 --- a/packages/backend/src/services/slos/slo-alert-rule.test.ts +++ b/packages/backend/src/services/slos/slo-alert-rule.test.ts @@ -42,6 +42,20 @@ describe("evaluateSloBurn", () => { ]) }) + it("holds the critical tier while its 5m window is empty even as the warning clears", () => { + const evaluation = evaluateSloBurn( + Arr.map(windows({ 30: 0.5, 60: 20, 360: 0.5 }), (window) => + window.windowMinutes === 5 ? { ...window, total: 0, good: 0 } : window, + ), + ) + // Healthy overall, but the held tier keeps an open critical incident from resolving. + expect(evaluation.status).toBe("healthy") + expect(evaluation.levels).toEqual([ + { severity: "warning", status: "clear" }, + { severity: "critical", status: "hold" }, + ]) + }) + it("skips as no data when every short window is empty", () => { const empty = Arr.map(windows({}), (window) => window.windowMinutes === 5 || window.windowMinutes === 30 From fef1f485e63b89261d5d85808041d8b3230aa17e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 6 Nov 2026 03:33:53 +0100 Subject: [PATCH 11/11] test(slos): each burn tier notifies its own destinations --- .../services/alerts/AlertsService.slo.test.ts | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/services/alerts/AlertsService.slo.test.ts b/packages/backend/src/services/alerts/AlertsService.slo.test.ts index e0ee99d9b9..d5b0a666da 100644 --- a/packages/backend/src/services/alerts/AlertsService.slo.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.slo.test.ts @@ -1,10 +1,11 @@ import { afterEach, assert, describe, it } from "@effect/vitest" -import { Clock, Duration, Effect, Layer, Option, Array as Arr } from "effect" +import { Clock, Duration, Effect, Layer, Option, Order, Array as Arr } from "effect" import { TestClock } from "effect/testing" import { compiledQueryOf } from "@maple/query-engine/execution" import { formatWarehouseDateTime } from "@maple/query-engine" import { AlertDestinationId } from "@maple/domain/primitives" -import { AlertsService } from "./AlertsService" +import { WebhookAlertDestinationConfig } from "@maple/domain/http" +import { AlertsService, type AlertsServiceApi } from "./AlertsService" import { cleanupTestDbs, createTestDb, @@ -92,7 +93,54 @@ const createSlo = Effect.fn(function* (destinationId: AlertDestinationId | undef }) }) +/** A webhook destination of its own, so a test can tell which tier notified whom. */ +const createNamedWebhook = (alerts: AlertsServiceApi, name: string) => + alerts.createDestination( + orgId, + userId, + adminRoles, + new WebhookAlertDestinationConfig({ + type: "webhook", + name, + enabled: true, + url: `https://example.com/${name}`, + signingSecret: "webhook-secret", + }), + ) + +/** Opens an incident at `badPerMille` and returns which destinations it notified. */ +const notifiedBy = (badPerMille: number) => { + const { layer } = harness({ badPerMille }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const pager = yield* createNamedWebhook(alerts, "pager") + const tickets = yield* createNamedWebhook(alerts, "tickets") + const slos = yield* SlosService + yield* slos.create(orgId, { + name: "Checkout availability", + serviceName: "checkout", + environments: ["production"], + sli: { _tag: "availability" }, + targetPercent: 99.9, + alerting: { criticalDestinationIds: [pager.id], warningDestinationIds: [tickets.id] }, + createdBy: userId, + }) + yield* alerts.runSchedulerTick() + const events = (yield* alerts.listDeliveryEvents(orgId)).events + return Arr.sort(Arr.dedupe(Arr.map(events, (event) => event.destinationName)), Order.String) + }).pipe(Effect.provide(layer)) +} + describe("AlertsService SLO burn-rate rules", () => { + it.effect("a slow burn tickets only the warning destination", () => + Effect.map(notifiedBy(8), (notified) => assert.deepStrictEqual(notified, ["tickets"])), + ) + + it.effect("a fast burn pages the critical destination, and the warning one too", () => + Effect.map(notifiedBy(20), (notified) => assert.deepStrictEqual(notified, ["pager", "tickets"])), + ) + it.effect("a fast burn pages, and a recovered short window resolves it", () => { const traffic: Traffic = { badPerMille: 20 } const { layer } = harness(traffic)