feat(slos): multiwindow burn-rate alerts

apps/api/src/routes/v2/alchemy-provider.integration.test.ts

@@ -44,6 +44,7 @@ import { DashboardPersistenceService } from "@maple/backend/services/dashboards/
 import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService"
 import { AlertRuntime, AlertsService } from "@maple/backend/services/alerts/AlertsService"
 import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource"
+import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource"
 import { AlertDestinationsService } from "@maple/backend/services/alerts/AlertDestinationsService"
 import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService"
 import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService"
@@ -186,6 +187,9 @@ const makeHarness = () => {
 				alertReadModelsLive,
 				alertRulesLive,
 				UptimeAlertSource.layer.pipe(Layer.provide(testDb.layer)),
+				Layer.effect(SloAlertSource, SloAlertSource.make).pipe(
+					Layer.provide(Layer.mergeAll(testDb.layer, warehouseLive)),
+				),
 			),
 		),
 	)

apps/api/src/routes/v2/alerts.http.test.ts

@@ -41,6 +41,7 @@ import { DashboardPersistenceService } from "@maple/backend/services/dashboards/
 import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService"
 import { AlertRuntime, AlertsService } from "@maple/backend/services/alerts/AlertsService"
 import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource"
+import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource"
 import { AlertDestinationsService } from "@maple/backend/services/alerts/AlertDestinationsService"
 import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService"
 import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService"
@@ -190,6 +191,9 @@ const makeHarness = (
 				alertReadModelsLive,
 				alertRulesLive,
 				UptimeAlertSource.layer.pipe(Layer.provide(testDb.layer)),
+				Layer.effect(SloAlertSource, SloAlertSource.make).pipe(
+					Layer.provide(Layer.mergeAll(testDb.layer, warehouseLive)),
+				),
 			),
 		),
 	)

apps/api/src/routes/v2/slos.http.test.ts

@@ -178,6 +178,18 @@ describe("v2 slos", () => {
 				window_days: 30,
 			})
 			const id: string = created.body.id
+			// Every SLO gets its burn-rate rule, alerting with no destinations until some are added.
+			expect(created.body.alert_rule_id).toMatch(/^alrt_/)
+			expect(created.body.alerting).toEqual({
+				enabled: true,
+				critical_destination_ids: [],
+				warning_destination_ids: [],
+			})
+			const paused = await harness.request("PATCH", `/v2/slos/${id}`, token, {
+				alerting: { enabled: false },
+			})
+			expect(paused.body.alerting.enabled).toBe(false)
+			expect(paused.body.alert_rule_id).toBe(created.body.alert_rule_id)
 
 			const listed = await harness.request("GET", "/v2/slos", token)
 			expect(listed.body.data.map((slo: { readonly id: string }) => slo.id)).toEqual([id])
@@ -195,6 +207,7 @@ describe("v2 slos", () => {
 				target_percent: 99,
 				window_days: 7,
 				environments: [],
+				alerting: { enabled: false },
 			})
 
 			const retrieved = await harness.request("GET", `/v2/slos/${id}`, token)

apps/api/src/routes/v2/slos.http.ts

@@ -8,7 +8,7 @@ import {
 	sloSliToWire,
 	V2InsufficientPermissions,
 } from "@maple/domain/http/v2"
-import type { V2Slo, V2SloStatus } from "@maple/domain/http/v2"
+import type { V2Slo, V2SloAlertingParams, V2SloStatus } from "@maple/domain/http/v2"
 import { Effect, Array as Arr } from "effect"
 import { auditDiff } from "@/routes/v2/audit-changes"
 import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService"
@@ -28,6 +28,11 @@ export const toV2Slo = (slo: SloRecord): V2Slo => ({
 	target_percent: slo.targetPercent,
 	window_days: slo.windowDays,
 	alert_rule_id: slo.alertRuleId,
+	alerting: {
+		enabled: slo.alerting.enabled,
+		critical_destination_ids: slo.alerting.criticalDestinationIds,
+		warning_destination_ids: slo.alerting.warningDestinationIds,
+	},
 	created_at: isoTimestamp(slo.createdAt),
 	updated_at: isoTimestamp(slo.updatedAt),
 })
@@ -49,9 +54,34 @@ const toV2Status = (sloId: SloRecord["id"], status: SloStatus): V2SloStatus => (
 })
 
 const sloAuditDiff = auditDiff({
-	fields: ["name", "description", "service_name", "environments", "sli", "target_percent", "window_days"],
+	fields: [
+		"name",
+		"description",
+		"service_name",
+		"environments",
+		"sli",
+		"target_percent",
+		"window_days",
+		"alerting",
+	],
 })
 
+/** Wire alerting params to the service's, keeping only the fields the caller sent. */
+const alertingFromWire = (alerting: V2SloAlertingParams | undefined) =>
+	alerting === undefined
+		? undefined
+		: {
+				alerting: {
+					...(alerting.enabled === undefined ? undefined : { enabled: alerting.enabled }),
+					...(alerting.critical_destination_ids === undefined
+						? undefined
+						: { criticalDestinationIds: alerting.critical_destination_ids }),
+					...(alerting.warning_destination_ids === undefined
+						? undefined
+						: { warningDestinationIds: alerting.warning_destination_ids }),
+				},
+			}
+
 const requireManage = Effect.gen(function* () {
 	const tenant = yield* CurrentTenant.Context
 	yield* Policy.check(tenant.roles, "alerts:manage", (denial) =>
@@ -97,6 +127,7 @@ export const HttpV2SlosLive = HttpApiBuilder.group(MapleApiV2, "slos", (handlers
 						...(payload.window_days === undefined
 							? undefined
 							: { windowDays: payload.window_days }),
+						...alertingFromWire(payload.alerting),
 						createdBy: tenant.userId,
 					})
 					yield* recordHttpAudit("slo.created", {
@@ -128,6 +159,7 @@ export const HttpV2SlosLive = HttpApiBuilder.group(MapleApiV2, "slos", (handlers
 						...(payload.window_days === undefined
 							? undefined
 							: { windowDays: payload.window_days }),
+						...alertingFromWire(payload.alerting),
 						updatedBy: tenant.userId,
 					})
 					yield* recordHttpAudit("slo.updated", {

apps/ios/Maple/Components/AlertFormatting.swift

@@ -15,6 +15,8 @@ enum SignalUnit: Codable {
 	case apdex
 	case count
 	case plain
+	/// An SLO burn rate: a multiple of the sustainable rate, shown as `14.4x` like the web.
+	case burnRate
 
 	func format(_ value: Double?) -> String {
 		guard let value, value.isFinite else { return Format.placeholder }
@@ -24,6 +26,7 @@ enum SignalUnit: Codable {
 		case .perMinute: return Format.count(value) + "/min"
 		case .apdex: return String(format: "%.2f", value)
 		case .count, .plain: return Format.count(value)
+		case .burnRate: return String(format: "%.1fx", value)
 		}
 	}
 }
@@ -46,6 +49,8 @@ struct SignalDisplay: Codable {
 		case .rawQuery: self.init(label: "Raw query", unit: .plain)
 		// A monitor's hidden rule: down is 1, up is 0, so the value reads as a plain number.
 		case .uptime: self.init(label: "Uptime check", unit: .plain)
+		// An SLO's hidden rule: the value is how many times faster than sustainable the budget burns.
+		case .sloBurnRate: self.init(label: "SLO burn rate", unit: .burnRate)
 		}
 	}
 

apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json

@@ -6971,7 +6971,8 @@
           "throughput",
           "builder_query",
           "raw_query",
-          "uptime"
+          "uptime",
+          "slo_burn_rate"
         ],
         "examples": [
           "error_rate"

packages/backend/src/services/alerts/AlertDeliveryDispatch.ts

@@ -82,6 +82,9 @@ type SummaryLineContext = Pick<
  */
 export const buildSummaryLine = (context: SummaryLineContext, em: (value: string) => string): string => {
 	if (context.signalType === "uptime") return em(uptimeSummaryLine(context))
+	// An SLO's check names both windows and the tier, which one value and window cannot.
+	if (context.signalType === "slo_burn_rate" && context.detail != null && context.eventType !== "test")
+		return em(context.detail)
 	const signal = formatSignalLabel(context)
 	const observed = formatSignalMetric(context.value, signalDisplayOf(context))
 	const window = formatWindow(context.windowMinutes)

packages/backend/src/services/alerts/AlertRuleModel.ts

@@ -39,6 +39,8 @@ import {
 	type AlertThresholds,
 	type QueryBuilderQueryDraftPayload,
 	type OrgId,
+	isManagedSignalType,
+	type ManagedSignalType,
 } from "@maple/domain/http"
 import * as PG from "@maple-dev/effect-orm/postgres"
 import { AlertRules, type AlertRuleRow } from "@maple/db/tables"
@@ -229,12 +231,17 @@ const planGroupingTokens = (
 	return groupBy
 }
 
-/** Why the generic rule APIs refuse an `uptime` rule. */
-export const UPTIME_RULE_MANAGED_MESSAGE =
-	"uptime rules belong to an uptime monitor; create, edit or delete the monitor instead"
+/** Why the generic rule APIs refuse a rule another resource owns. */
+export const managedRuleMessage = (signalType: ManagedSignalType): string =>
+	signalType === "uptime"
+		? "uptime rules belong to an uptime monitor; create, edit or delete the monitor instead"
+		: "slo_burn_rate rules belong to an SLO; create, edit or delete the SLO instead"
 
-/** The stored plan of every `uptime` rule: no query, the evaluator reads the monitor's state. */
-export const UPTIME_RULE_PLAN = new CompiledAlertQueryPlan({
+/**
+ * The stored plan of every managed rule (`uptime`, `slo_burn_rate`): no query, the evaluator
+ * reads the owning resource instead.
+ */
+export const MANAGED_RULE_PLAN = new CompiledAlertQueryPlan({
 	kind: "spec",
 	query: null,
 	rawSql: null,
@@ -292,9 +299,9 @@ export const uptimeRuleFields = (monitor: {
 		rawQuerySql: null,
 		destinationIdsJson: [...monitor.destinationIds],
 		querySpecJson: null,
-		reducer: UPTIME_RULE_PLAN.reducer,
+		reducer: MANAGED_RULE_PLAN.reducer,
 		sampleCountStrategy: null,
-		noDataBehavior: UPTIME_RULE_PLAN.noDataBehavior,
+		noDataBehavior: MANAGED_RULE_PLAN.noDataBehavior,
 	} as const
 }
 
@@ -539,8 +546,8 @@ export const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(functi
 			sampleCountStrategy: null,
 			noDataBehavior,
 		})
-	} else if (rule.signalType === "uptime") {
-		return yield* makeAlertValidationError(UPTIME_RULE_MANAGED_MESSAGE)
+	} else if (isManagedSignalType(rule.signalType)) {
+		return yield* makeAlertValidationError(managedRuleMessage(rule.signalType))
 	} else {
 		return yield* makeAlertValidationError(`Unsupported signal type: ${rule.signalType}`)
 	}
@@ -574,8 +581,8 @@ const parseCompiledPlan = (
 			component: "compiled_plan",
 			cause,
 		})
-	// An uptime rule reads its monitor's state, not the warehouse: no query to decode.
-	if (row.signalType === "uptime") return Effect.succeed(UPTIME_RULE_PLAN)
+	// A managed rule reads its owner (a monitor's state, an SLO's burn), never a stored query.
+	if (isManagedSignalType(row.signalType)) return Effect.succeed(MANAGED_RULE_PLAN)
 	if (row.signalType === "raw_query") {
 		if (row.rawQuerySql == null) {
 			return Effect.fail(invalid("Stored raw alert is missing its SQL query", "rawQuerySql is null"))
@@ -868,8 +875,8 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => {
 		request: AlertRuleUpsertRequest,
 		options?: { readonly forPreview?: boolean },
 	): Effect.fn.Return<NormalizedRule, AlertValidationError> {
-		if (request.signalType === "uptime") {
-			return yield* invalidRuleError([UPTIME_RULE_MANAGED_MESSAGE])
+		if (isManagedSignalType(request.signalType)) {
+			return yield* invalidRuleError([managedRuleMessage(request.signalType)])
 		}
 		const name = Str.trim(request.name)
 		// A builder query's draft carries its own scope and never reads the

packages/backend/src/services/alerts/AlertRulesService.ts

@@ -15,6 +15,8 @@ import {
 	type AlertRuleUpsertRequest,
 	type OrgId,
 	type UserId,
+	isManagedSignalType,
+	MANAGED_SIGNAL_TYPES,
 } from "@maple/domain/http"
 import * as Orm from "@maple-dev/effect-orm/database"
 import * as PG from "@maple-dev/effect-orm/postgres"
@@ -39,7 +41,7 @@ import {
 	rowToRuleDocument,
 	selectStoredAlertRules,
 	storedRuleCondition,
-	UPTIME_RULE_MANAGED_MESSAGE,
+	managedRuleMessage,
 	type RuleEvaluationState,
 } from "./AlertRuleModel"
 import { tierConditionChanged } from "./alert-thresholds"
@@ -252,11 +254,11 @@ export const makeAlertRulePersistence = (options: {
 						const activeRows = yield* db.run(
 							PG.from(AlertRules)
 								.select("id")
-								// Monitors' backing rules are capped by the monitor allowance instead.
+								// Managed rules (monitors, SLOs) are not counted against the rule cap.
 								.where(($) => [
 									$.orgId.eq(orgId),
 									$.enabled.eq(true),
-									$.signalType.neq("uptime"),
+									$.signalType.notIn(...MANAGED_SIGNAL_TYPES),
 								]),
 						)
 						const alreadyActive =
@@ -364,7 +366,7 @@ export const makeAlertRulePersistence = (options: {
 			db.run(
 				// Monitors' backing rules are managed through the monitor, so the rule list hides them.
 				selectStoredAlertRules()
-					.where(($) => [$.orgId.eq(orgId), $.signalType.neq("uptime")])
+					.where(($) => [$.orgId.eq(orgId), $.signalType.notIn(...MANAGED_SIGNAL_TYPES)])
 					.orderBy(["createdAt", "desc"], ["id", "desc"]),
 			),
 		)
@@ -414,9 +416,9 @@ export const makeAlertRulePersistence = (options: {
 	) {
 		yield* requireAlertsPermission(roles)
 		const row = yield* requireRuleRow(orgId, ruleId)
-		// Hidden from the generic rule APIs: a monitor's rule is deleted with its monitor.
-		if (row.signalType === "uptime") {
-			return yield* new AlertRuleNotFoundError({ message: UPTIME_RULE_MANAGED_MESSAGE, ruleId })
+		// Hidden from the generic rule APIs: a managed rule is deleted with its owner.
+		if (isManagedSignalType(row.signalType)) {
+			return yield* new AlertRuleNotFoundError({ message: managedRuleMessage(row.signalType), ruleId })
 		}
 		const deleted = yield* dbExecute((db) => db.transaction(deleteRuleRows(db, orgId, ruleId)))
 		const txid = readTxid(deleted)