diff --git a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts index 3ecd966a73..caac85a016 100644 --- a/apps/api/src/routes/v2/alchemy-provider.integration.test.ts +++ b/apps/api/src/routes/v2/alchemy-provider.integration.test.ts @@ -44,6 +44,7 @@ import { DashboardPersistenceService } from "@maple/backend/services/dashboards/ import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService" import { AlertRuntime, AlertsService } from "@maple/backend/services/alerts/AlertsService" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" +import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource" import { AlertDestinationsService } from "@maple/backend/services/alerts/AlertDestinationsService" import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService" import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService" @@ -186,6 +187,9 @@ const makeHarness = () => { alertReadModelsLive, alertRulesLive, UptimeAlertSource.layer.pipe(Layer.provide(testDb.layer)), + Layer.effect(SloAlertSource, SloAlertSource.make).pipe( + Layer.provide(Layer.mergeAll(testDb.layer, warehouseLive)), + ), ), ), ) diff --git a/apps/api/src/routes/v2/alerts.http.test.ts b/apps/api/src/routes/v2/alerts.http.test.ts index 0c49db26a0..178cb2729c 100644 --- a/apps/api/src/routes/v2/alerts.http.test.ts +++ b/apps/api/src/routes/v2/alerts.http.test.ts @@ -41,6 +41,7 @@ import { DashboardPersistenceService } from "@maple/backend/services/dashboards/ import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService" import { AlertRuntime, AlertsService } from "@maple/backend/services/alerts/AlertsService" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" +import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource" import { AlertDestinationsService } from "@maple/backend/services/alerts/AlertDestinationsService" import { AlertReadModelsService } from "@maple/backend/services/alerts/AlertReadModelsService" import { AlertRulesService } from "@maple/backend/services/alerts/AlertRulesService" @@ -190,6 +191,9 @@ const makeHarness = ( alertReadModelsLive, alertRulesLive, UptimeAlertSource.layer.pipe(Layer.provide(testDb.layer)), + Layer.effect(SloAlertSource, SloAlertSource.make).pipe( + Layer.provide(Layer.mergeAll(testDb.layer, warehouseLive)), + ), ), ), ) diff --git a/apps/api/src/routes/v2/slos.http.test.ts b/apps/api/src/routes/v2/slos.http.test.ts index afaf014ce7..eeb48a24bb 100644 --- a/apps/api/src/routes/v2/slos.http.test.ts +++ b/apps/api/src/routes/v2/slos.http.test.ts @@ -178,6 +178,18 @@ describe("v2 slos", () => { window_days: 30, }) const id: string = created.body.id + // Every SLO gets its burn-rate rule, alerting with no destinations until some are added. + expect(created.body.alert_rule_id).toMatch(/^alrt_/) + expect(created.body.alerting).toEqual({ + enabled: true, + critical_destination_ids: [], + warning_destination_ids: [], + }) + const paused = await harness.request("PATCH", `/v2/slos/${id}`, token, { + alerting: { enabled: false }, + }) + expect(paused.body.alerting.enabled).toBe(false) + expect(paused.body.alert_rule_id).toBe(created.body.alert_rule_id) const listed = await harness.request("GET", "/v2/slos", token) expect(listed.body.data.map((slo: { readonly id: string }) => slo.id)).toEqual([id]) @@ -195,6 +207,7 @@ describe("v2 slos", () => { target_percent: 99, window_days: 7, environments: [], + alerting: { enabled: false }, }) const retrieved = await harness.request("GET", `/v2/slos/${id}`, token) diff --git a/apps/api/src/routes/v2/slos.http.ts b/apps/api/src/routes/v2/slos.http.ts index 42846ffec9..0995938140 100644 --- a/apps/api/src/routes/v2/slos.http.ts +++ b/apps/api/src/routes/v2/slos.http.ts @@ -8,7 +8,7 @@ import { sloSliToWire, V2InsufficientPermissions, } from "@maple/domain/http/v2" -import type { V2Slo, V2SloStatus } from "@maple/domain/http/v2" +import type { V2Slo, V2SloAlertingParams, V2SloStatus } from "@maple/domain/http/v2" import { Effect, Array as Arr } from "effect" import { auditDiff } from "@/routes/v2/audit-changes" import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService" @@ -28,6 +28,11 @@ export const toV2Slo = (slo: SloRecord): V2Slo => ({ target_percent: slo.targetPercent, window_days: slo.windowDays, alert_rule_id: slo.alertRuleId, + alerting: { + enabled: slo.alerting.enabled, + critical_destination_ids: slo.alerting.criticalDestinationIds, + warning_destination_ids: slo.alerting.warningDestinationIds, + }, created_at: isoTimestamp(slo.createdAt), updated_at: isoTimestamp(slo.updatedAt), }) @@ -49,9 +54,34 @@ const toV2Status = (sloId: SloRecord["id"], status: SloStatus): V2SloStatus => ( }) const sloAuditDiff = auditDiff({ - fields: ["name", "description", "service_name", "environments", "sli", "target_percent", "window_days"], + fields: [ + "name", + "description", + "service_name", + "environments", + "sli", + "target_percent", + "window_days", + "alerting", + ], }) +/** Wire alerting params to the service's, keeping only the fields the caller sent. */ +const alertingFromWire = (alerting: V2SloAlertingParams | undefined) => + alerting === undefined + ? undefined + : { + alerting: { + ...(alerting.enabled === undefined ? undefined : { enabled: alerting.enabled }), + ...(alerting.critical_destination_ids === undefined + ? undefined + : { criticalDestinationIds: alerting.critical_destination_ids }), + ...(alerting.warning_destination_ids === undefined + ? undefined + : { warningDestinationIds: alerting.warning_destination_ids }), + }, + } + const requireManage = Effect.gen(function* () { const tenant = yield* CurrentTenant.Context yield* Policy.check(tenant.roles, "alerts:manage", (denial) => @@ -97,6 +127,7 @@ export const HttpV2SlosLive = HttpApiBuilder.group(MapleApiV2, "slos", (handlers ...(payload.window_days === undefined ? undefined : { windowDays: payload.window_days }), + ...alertingFromWire(payload.alerting), createdBy: tenant.userId, }) yield* recordHttpAudit("slo.created", { @@ -128,6 +159,7 @@ export const HttpV2SlosLive = HttpApiBuilder.group(MapleApiV2, "slos", (handlers ...(payload.window_days === undefined ? undefined : { windowDays: payload.window_days }), + ...alertingFromWire(payload.alerting), updatedBy: tenant.userId, }) yield* recordHttpAudit("slo.updated", { diff --git a/apps/ios/Maple/Components/AlertFormatting.swift b/apps/ios/Maple/Components/AlertFormatting.swift index 63dfe78165..bdbeb3db82 100644 --- a/apps/ios/Maple/Components/AlertFormatting.swift +++ b/apps/ios/Maple/Components/AlertFormatting.swift @@ -15,6 +15,8 @@ enum SignalUnit: Codable { case apdex case count case plain + /// An SLO burn rate: a multiple of the sustainable rate, shown as `14.4x` like the web. + case burnRate func format(_ value: Double?) -> String { guard let value, value.isFinite else { return Format.placeholder } @@ -24,6 +26,7 @@ enum SignalUnit: Codable { case .perMinute: return Format.count(value) + "/min" case .apdex: return String(format: "%.2f", value) case .count, .plain: return Format.count(value) + case .burnRate: return String(format: "%.1fx", value) } } } @@ -46,6 +49,8 @@ struct SignalDisplay: Codable { case .rawQuery: self.init(label: "Raw query", unit: .plain) // A monitor's hidden rule: down is 1, up is 0, so the value reads as a plain number. case .uptime: self.init(label: "Uptime check", unit: .plain) + // An SLO's hidden rule: the value is how many times faster than sustainable the budget burns. + case .sloBurnRate: self.init(label: "SLO burn rate", unit: .burnRate) } } diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index e8c0b0320f..4cc5df05c0 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -6971,7 +6971,8 @@ "throughput", "builder_query", "raw_query", - "uptime" + "uptime", + "slo_burn_rate" ], "examples": [ "error_rate" diff --git a/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts b/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts index c4e0d1c487..908d77b2ba 100644 --- a/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts +++ b/packages/backend/src/services/alerts/AlertDeliveryDispatch.ts @@ -82,6 +82,9 @@ type SummaryLineContext = Pick< */ export const buildSummaryLine = (context: SummaryLineContext, em: (value: string) => string): string => { if (context.signalType === "uptime") return em(uptimeSummaryLine(context)) + // An SLO's check names both windows and the tier, which one value and window cannot. + if (context.signalType === "slo_burn_rate" && context.detail != null && context.eventType !== "test") + return em(context.detail) const signal = formatSignalLabel(context) const observed = formatSignalMetric(context.value, signalDisplayOf(context)) const window = formatWindow(context.windowMinutes) diff --git a/packages/backend/src/services/alerts/AlertRuleModel.ts b/packages/backend/src/services/alerts/AlertRuleModel.ts index 94923a0bc4..cb5d721581 100644 --- a/packages/backend/src/services/alerts/AlertRuleModel.ts +++ b/packages/backend/src/services/alerts/AlertRuleModel.ts @@ -39,6 +39,8 @@ import { type AlertThresholds, type QueryBuilderQueryDraftPayload, type OrgId, + isManagedSignalType, + type ManagedSignalType, } from "@maple/domain/http" import * as PG from "@maple-dev/effect-orm/postgres" import { AlertRules, type AlertRuleRow } from "@maple/db/tables" @@ -229,12 +231,17 @@ const planGroupingTokens = ( return groupBy } -/** Why the generic rule APIs refuse an `uptime` rule. */ -export const UPTIME_RULE_MANAGED_MESSAGE = - "uptime rules belong to an uptime monitor; create, edit or delete the monitor instead" +/** Why the generic rule APIs refuse a rule another resource owns. */ +export const managedRuleMessage = (signalType: ManagedSignalType): string => + signalType === "uptime" + ? "uptime rules belong to an uptime monitor; create, edit or delete the monitor instead" + : "slo_burn_rate rules belong to an SLO; create, edit or delete the SLO instead" -/** The stored plan of every `uptime` rule: no query, the evaluator reads the monitor's state. */ -export const UPTIME_RULE_PLAN = new CompiledAlertQueryPlan({ +/** + * The stored plan of every managed rule (`uptime`, `slo_burn_rate`): no query, the evaluator + * reads the owning resource instead. + */ +export const MANAGED_RULE_PLAN = new CompiledAlertQueryPlan({ kind: "spec", query: null, rawSql: null, @@ -292,9 +299,9 @@ export const uptimeRuleFields = (monitor: { rawQuerySql: null, destinationIdsJson: [...monitor.destinationIds], querySpecJson: null, - reducer: UPTIME_RULE_PLAN.reducer, + reducer: MANAGED_RULE_PLAN.reducer, sampleCountStrategy: null, - noDataBehavior: UPTIME_RULE_PLAN.noDataBehavior, + noDataBehavior: MANAGED_RULE_PLAN.noDataBehavior, } as const } @@ -539,8 +546,8 @@ export const compileRulePlan = Effect.fn("AlertsService.compileRulePlan")(functi sampleCountStrategy: null, noDataBehavior, }) - } else if (rule.signalType === "uptime") { - return yield* makeAlertValidationError(UPTIME_RULE_MANAGED_MESSAGE) + } else if (isManagedSignalType(rule.signalType)) { + return yield* makeAlertValidationError(managedRuleMessage(rule.signalType)) } else { return yield* makeAlertValidationError(`Unsupported signal type: ${rule.signalType}`) } @@ -574,8 +581,8 @@ const parseCompiledPlan = ( component: "compiled_plan", cause, }) - // An uptime rule reads its monitor's state, not the warehouse: no query to decode. - if (row.signalType === "uptime") return Effect.succeed(UPTIME_RULE_PLAN) + // A managed rule reads its owner (a monitor's state, an SLO's burn), never a stored query. + if (isManagedSignalType(row.signalType)) return Effect.succeed(MANAGED_RULE_PLAN) if (row.signalType === "raw_query") { if (row.rawQuerySql == null) { return Effect.fail(invalid("Stored raw alert is missing its SQL query", "rawQuerySql is null")) @@ -868,8 +875,8 @@ export const makeAlertRuleNormalizer = (runtime: AlertRuntimeApi) => { request: AlertRuleUpsertRequest, options?: { readonly forPreview?: boolean }, ): Effect.fn.Return { - if (request.signalType === "uptime") { - return yield* invalidRuleError([UPTIME_RULE_MANAGED_MESSAGE]) + if (isManagedSignalType(request.signalType)) { + return yield* invalidRuleError([managedRuleMessage(request.signalType)]) } const name = Str.trim(request.name) // A builder query's draft carries its own scope and never reads the diff --git a/packages/backend/src/services/alerts/AlertRulesService.ts b/packages/backend/src/services/alerts/AlertRulesService.ts index 66009f1b59..e694d52b93 100644 --- a/packages/backend/src/services/alerts/AlertRulesService.ts +++ b/packages/backend/src/services/alerts/AlertRulesService.ts @@ -15,6 +15,8 @@ import { type AlertRuleUpsertRequest, type OrgId, type UserId, + isManagedSignalType, + MANAGED_SIGNAL_TYPES, } from "@maple/domain/http" import * as Orm from "@maple-dev/effect-orm/database" import * as PG from "@maple-dev/effect-orm/postgres" @@ -39,7 +41,7 @@ import { rowToRuleDocument, selectStoredAlertRules, storedRuleCondition, - UPTIME_RULE_MANAGED_MESSAGE, + managedRuleMessage, type RuleEvaluationState, } from "./AlertRuleModel" import { tierConditionChanged } from "./alert-thresholds" @@ -252,11 +254,11 @@ export const makeAlertRulePersistence = (options: { const activeRows = yield* db.run( PG.from(AlertRules) .select("id") - // Monitors' backing rules are capped by the monitor allowance instead. + // Managed rules (monitors, SLOs) are not counted against the rule cap. .where(($) => [ $.orgId.eq(orgId), $.enabled.eq(true), - $.signalType.neq("uptime"), + $.signalType.notIn(...MANAGED_SIGNAL_TYPES), ]), ) const alreadyActive = @@ -364,7 +366,7 @@ export const makeAlertRulePersistence = (options: { db.run( // Monitors' backing rules are managed through the monitor, so the rule list hides them. selectStoredAlertRules() - .where(($) => [$.orgId.eq(orgId), $.signalType.neq("uptime")]) + .where(($) => [$.orgId.eq(orgId), $.signalType.notIn(...MANAGED_SIGNAL_TYPES)]) .orderBy(["createdAt", "desc"], ["id", "desc"]), ), ) @@ -414,9 +416,9 @@ export const makeAlertRulePersistence = (options: { ) { yield* requireAlertsPermission(roles) const row = yield* requireRuleRow(orgId, ruleId) - // Hidden from the generic rule APIs: a monitor's rule is deleted with its monitor. - if (row.signalType === "uptime") { - return yield* new AlertRuleNotFoundError({ message: UPTIME_RULE_MANAGED_MESSAGE, ruleId }) + // Hidden from the generic rule APIs: a managed rule is deleted with its owner. + if (isManagedSignalType(row.signalType)) { + return yield* new AlertRuleNotFoundError({ message: managedRuleMessage(row.signalType), ruleId }) } const deleted = yield* dbExecute((db) => db.transaction(deleteRuleRows(db, orgId, ruleId))) const txid = readTxid(deleted) diff --git a/packages/backend/src/services/alerts/AlertsService.slo.test.ts b/packages/backend/src/services/alerts/AlertsService.slo.test.ts new file mode 100644 index 0000000000..d5b0a666da --- /dev/null +++ b/packages/backend/src/services/alerts/AlertsService.slo.test.ts @@ -0,0 +1,316 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { Clock, Duration, Effect, Layer, Option, Order, Array as Arr } from "effect" +import { TestClock } from "effect/testing" +import { compiledQueryOf } from "@maple/query-engine/execution" +import { formatWarehouseDateTime } from "@maple/query-engine" +import { AlertDestinationId } from "@maple/domain/primitives" +import { WebhookAlertDestinationConfig } from "@maple/domain/http" +import { AlertsService, type AlertsServiceApi } from "./AlertsService" +import { + cleanupTestDbs, + createTestDb, + executeSql, + queryFirstRow, + type TestDb, +} from "@maple/backend/platform/test-pglite" +import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { MAX_SLOS_PER_ORG, SlosService } from "@maple/backend/services/slos/SlosService" +import { + DEFAULT_CLOCK_EPOCH_MS, + adminRoles, + asOrgId, + asUserId, + createWebhookDestination, + makeLayer, +} from "./alerts-service-test-support" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +/** Bad events per 1000 in every minute the burn read sees; mutated as a test moves on. */ +interface Traffic { + badPerMille: number + /** No spans at all: the service is down or ingest stalled. */ + silent?: boolean + /** No spans in only the newest minutes; older minutes keep their traffic. */ + silentMinutes?: number +} + +/** + * A warehouse answering the SLO burn read with 400 minutes of uniform traffic ending at the + * test clock, so every burn window sees the same bad share. + */ +const sloWarehouse = (traffic: Traffic) => + makeWarehouseServiceStub({ + compiledQuery: (_tenant, compiled, options) => + Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis + const minute = Math.floor(nowMs / 60_000) * 60_000 + const rows = + options?.context === "sloBurnWindows" && traffic.silent !== true + ? Arr.map( + Arr.filter( + Arr.makeBy(400, (index) => index), + (index) => index >= (traffic.silentMinutes ?? 0), + ), + (index) => ({ + bucket: formatWarehouseDateTime(minute - index * 60_000), + total: 1000, + good: 1000 - traffic.badPerMille, + }), + ) + : [] + return yield* compiledQueryOf(compiled).decodeRows(rows).pipe(Effect.orDie) + }), + }) + +const harness = (traffic: Traffic) => { + const testDb = createTestDb(trackedDbs) + const warehouse = sloWarehouse(traffic) + const slos = Layer.effect(SlosService, SlosService.make).pipe( + Layer.provide(Layer.mergeAll(testDb.layer, Layer.succeed(WarehouseQueryService, warehouse))), + ) + return { testDb, layer: Layer.mergeAll(makeLayer(testDb, warehouse), slos) } +} + +const orgId = asOrgId("org_slo_alerts") +const userId = asUserId("user_slo_alerts") + +const createSlo = Effect.fn(function* (destinationId: AlertDestinationId | undefined) { + const slos = yield* SlosService + return yield* slos.create(orgId, { + name: "Checkout availability", + serviceName: "checkout", + environments: ["production"], + sli: { _tag: "availability" }, + targetPercent: 99.9, + alerting: + destinationId === undefined + ? {} + : { criticalDestinationIds: [destinationId], warningDestinationIds: [destinationId] }, + createdBy: userId, + }) +}) + +/** A webhook destination of its own, so a test can tell which tier notified whom. */ +const createNamedWebhook = (alerts: AlertsServiceApi, name: string) => + alerts.createDestination( + orgId, + userId, + adminRoles, + new WebhookAlertDestinationConfig({ + type: "webhook", + name, + enabled: true, + url: `https://example.com/${name}`, + signingSecret: "webhook-secret", + }), + ) + +/** Opens an incident at `badPerMille` and returns which destinations it notified. */ +const notifiedBy = (badPerMille: number) => { + const { layer } = harness({ badPerMille }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const pager = yield* createNamedWebhook(alerts, "pager") + const tickets = yield* createNamedWebhook(alerts, "tickets") + const slos = yield* SlosService + yield* slos.create(orgId, { + name: "Checkout availability", + serviceName: "checkout", + environments: ["production"], + sli: { _tag: "availability" }, + targetPercent: 99.9, + alerting: { criticalDestinationIds: [pager.id], warningDestinationIds: [tickets.id] }, + createdBy: userId, + }) + yield* alerts.runSchedulerTick() + const events = (yield* alerts.listDeliveryEvents(orgId)).events + return Arr.sort(Arr.dedupe(Arr.map(events, (event) => event.destinationName)), Order.String) + }).pipe(Effect.provide(layer)) +} + +describe("AlertsService SLO burn-rate rules", () => { + it.effect("a slow burn tickets only the warning destination", () => + Effect.map(notifiedBy(8), (notified) => assert.deepStrictEqual(notified, ["tickets"])), + ) + + it.effect("a fast burn pages the critical destination, and the warning one too", () => + Effect.map(notifiedBy(20), (notified) => assert.deepStrictEqual(notified, ["pager", "tickets"])), + ) + + it.effect("a fast burn pages, and a recovered short window resolves it", () => { + const traffic: Traffic = { badPerMille: 20 } + const { layer } = harness(traffic) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const slo = yield* createSlo(destination.id) + assert.isNotNull(slo.alertRuleId) + + // 2% bad at 99.9% burns 20x in every window: past both tiers, so critical. + yield* alerts.runSchedulerTick() + const opened = (yield* alerts.listIncidents(orgId)).incidents + assert.lengthOf(opened, 1) + assert.strictEqual(opened[0]?.signalType, "slo_burn_rate") + assert.strictEqual(opened[0]?.severity, "critical") + assert.strictEqual(opened[0]?.ruleId, slo.alertRuleId) + assert.strictEqual(opened[0]?.ruleName, "SLO burn: Checkout availability") + + traffic.badPerMille = 0 + yield* TestClock.adjust(Duration.minutes(1)) + yield* alerts.runSchedulerTick() + const resolved = (yield* alerts.listIncidents(orgId)).incidents + assert.strictEqual(resolved[0]?.status, "resolved") + }).pipe(Effect.provide(layer)) + }) + + it.effect("a page stays open while the service sends no spans at all", () => { + const traffic: Traffic = { badPerMille: 20 } + const { layer } = harness(traffic) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + assert.strictEqual((yield* alerts.listIncidents(orgId)).incidents[0]?.status, "open") + + // An outage zeroes every window: that is no data, not a recovery. + traffic.silent = true + yield* TestClock.adjust(Duration.minutes(1)) + yield* alerts.runSchedulerTick() + const held = (yield* alerts.listIncidents(orgId)).incidents + assert.strictEqual(held[0]?.status, "open") + assert.strictEqual(held[0]?.severity, "critical") + }).pipe(Effect.provide(layer)) + }) + + it.effect("a page stays open while its 5m window is empty, even once the warning tier clears", () => { + const traffic: Traffic = { badPerMille: 20 } + const { layer } = harness(traffic) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + assert.strictEqual((yield* alerts.listIncidents(orgId)).incidents[0]?.severity, "critical") + + // The newest minutes go quiet and the older ones turn healthy: warning clears, critical holds. + traffic.badPerMille = 0 + traffic.silentMinutes = 6 + yield* TestClock.adjust(Duration.minutes(1)) + yield* alerts.runSchedulerTick() + const held = (yield* alerts.listIncidents(orgId)).incidents + assert.strictEqual(held[0]?.status, "open") + assert.strictEqual(held[0]?.severity, "critical") + }).pipe(Effect.provide(layer)) + }) + + it.effect("an edit brings back a rule whose row is gone", () => { + const { layer, testDb } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const slos = yield* SlosService + const slo = yield* createSlo(undefined) + yield* Effect.promise(() => + executeSql(testDb, "delete from alert_rules where id = $1", [slo.alertRuleId]), + ) + const updated = yield* slos.update(orgId, slo.id, { targetPercent: 99.5, updatedBy: userId }) + assert.strictEqual(updated.alertRuleId, slo.alertRuleId) + assert.isTrue(updated.alerting.enabled) + }).pipe(Effect.provide(layer)) + }) + + it.effect("a slow burn opens a warning, not a page", () => { + // 0.8% bad burns 8x: over the 6x ticket factor, under the 14.4x page factor. + const { layer } = harness({ badPerMille: 8 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + const opened = (yield* alerts.listIncidents(orgId)).incidents + assert.lengthOf(opened, 1) + assert.strictEqual(opened[0]?.severity, "warning") + }).pipe(Effect.provide(layer)) + }) + + it.effect("a burn within budget opens nothing", () => { + const { layer } = harness({ badPerMille: 1 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + yield* createSlo(undefined) + yield* alerts.runSchedulerTick() + assert.lengthOf((yield* alerts.listIncidents(orgId)).incidents, 0) + }).pipe(Effect.provide(layer)) + }) + + it.effect("the rule is hidden from the rule APIs and deleted with its SLO", () => { + const { layer, testDb } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const slos = yield* SlosService + const slo = yield* createSlo(undefined) + const ruleId = Option.getOrThrow(Option.fromNullOr(slo.alertRuleId)) + + assert.lengthOf((yield* alerts.listRules(orgId)).rules, 0) + const refused = yield* alerts.deleteRule(orgId, adminRoles, ruleId).pipe(Effect.flip) + assert.strictEqual(refused._tag, "@maple/http/errors/AlertRuleNotFoundError") + + // Switching alerting off disables the rule; the SLO keeps it. + const paused = yield* slos.update(orgId, slo.id, { + alerting: { enabled: false }, + updatedBy: userId, + }) + assert.isFalse(paused.alerting.enabled) + + yield* slos.delete(orgId, slo.id) + const remaining = yield* Effect.promise(() => + queryFirstRow<{ count: number }>( + testDb, + "select count(*)::int as count from alert_rules where id = $1", + [ruleId], + ), + ) + assert.strictEqual(remaining?.count, 0) + }).pipe(Effect.provide(layer)) + }) + + it.effect("caps an org at MAX_SLOS_PER_ORG SLOs", () => { + const { layer } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const slos = yield* SlosService + const create = (index: number) => + slos.create(orgId, { + name: `SLO ${index}`, + serviceName: "checkout", + sli: { _tag: "availability" }, + targetPercent: 99, + createdBy: userId, + }) + yield* Effect.forEach(Arr.range(1, MAX_SLOS_PER_ORG), create, { discard: true }) + const refused = yield* create(MAX_SLOS_PER_ORG + 1).pipe(Effect.flip) + assert.strictEqual(refused._tag, "@maple/http/errors/SloValidationError") + assert.lengthOf(yield* slos.list(orgId), MAX_SLOS_PER_ORG) + }).pipe(Effect.provide(layer)) + }) + + it.effect("refuses an unknown destination and keeps the SLO unsaved", () => { + const { layer } = harness({ badPerMille: 0 }) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const slos = yield* SlosService + const refused = yield* createSlo( + AlertDestinationId.make("00000000-0000-4000-8000-000000000000"), + ).pipe(Effect.flip) + assert.strictEqual(refused._tag, "@maple/http/errors/SloValidationError") + assert.lengthOf(yield* slos.list(orgId), 0) + }).pipe(Effect.provide(layer)) + }) +}) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 132b1b55e2..f9e9093137 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -72,6 +72,7 @@ import { RoleName, WarehouseUpstreamError, type UserId, + isManagedSignalType, } from "@maple/domain/http" import * as PG from "@maple-dev/effect-orm/postgres" import * as Orm from "@maple-dev/effect-orm/database" @@ -163,6 +164,8 @@ import { makeAlertDestinationDelivery, parseAlertDestinationEncryptionKey } from import { AlertReadModelsService, type AlertReadModelsServiceApi } from "./AlertReadModelsService" import { AlertRulesService, makeAlertRulePersistence, type AlertRulesServiceApi } from "./AlertRulesService" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" +import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource" +import { evaluateSloBurn } from "@maple/backend/services/slos/slo-alert-rule" import { openForAlertIncident, resolveForAlertIncident, @@ -180,7 +183,7 @@ import { selectStoredAlertRules, type NormalizedRule, type StoredAlertRuleRow, - UPTIME_RULE_MANAGED_MESSAGE, + managedRuleMessage, } from "./AlertRuleModel" import { mapSignalUnit, resolveSignalDisplay } from "./alert-signal-display" import { summarizeCause } from "@maple/backend/platform/describe-cause" @@ -621,6 +624,7 @@ export class AlertsService extends Context.Service + Effect.fail(makePersistenceError(error)), + ), + ) + return Option.match(observation, { + onNone: () => [], + onSome: (windows) => [ + { evaluation: evaluateSloBurn(windows), groupKey: UNGROUPED_GROUP_KEY }, + ], + }) + } const startMs = endMs - rule.windowMinutes * 60_000 const plan = rule.compiledPlan const source = yield* planEvaluateSource(plan, rule) @@ -1298,7 +1319,7 @@ export class AlertsService extends Context.Service round(metric, 3)), Match.when("rpm", () => `${grouped(metric)} rpm`), Match.whenOr("count", "plain", () => grouped(metric)), + Match.when("burn", () => `${round(metric, 1)}x`), Match.exhaustive, ), }) @@ -190,9 +191,11 @@ const uptimeObserved = (context: Pick): str export const formatObservedSummary = (context: ObservedContext): string => context.signalType === "uptime" ? uptimeObserved(context) - : context.value === null - ? `No data (threshold ${formatThresholdSummary(context)})` - : `${formatSignalMetric(context.value, signalDisplayOf(context))} ${formatThresholdSummary(context)}` + : context.signalType === "slo_burn_rate" && context.detail != null + ? context.detail + : context.value === null + ? `No data (threshold ${formatThresholdSummary(context)})` + : `${formatSignalMetric(context.value, signalDisplayOf(context))} ${formatThresholdSummary(context)}` /** * A notification's subject line without its emoji: `checkout-api is down` for an uptime rule, diff --git a/packages/backend/src/services/alerts/alert-persistence.ts b/packages/backend/src/services/alerts/alert-persistence.ts index 6916a783c8..7af309539f 100644 --- a/packages/backend/src/services/alerts/alert-persistence.ts +++ b/packages/backend/src/services/alerts/alert-persistence.ts @@ -1,4 +1,6 @@ -import { AlertPersistenceError } from "@maple/domain/http" +import * as Orm from "@maple-dev/effect-orm/database" +import type { MapleDb } from "@maple/db/client" +import { AlertPersistenceError, type OrgId } from "@maple/domain/http" import { makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" /** Shared `unknown -> AlertPersistenceError` mapper for every alerts service. */ @@ -6,3 +8,10 @@ export const makePersistenceError = makePersistenceErrorMapper( AlertPersistenceError, "Alert persistence failed", ) + +/** + * The per-org lock every alert rule write takes inside its transaction, so a name check or a + * destination check and the write that follows are atomic. + */ +export const lockOrgAlertRules = (db: MapleDb, orgId: OrgId) => + db.execute(Orm.sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) diff --git a/packages/backend/src/services/alerts/alert-signal-display.ts b/packages/backend/src/services/alerts/alert-signal-display.ts index 4a9587ccc6..c20b58a016 100644 --- a/packages/backend/src/services/alerts/alert-signal-display.ts +++ b/packages/backend/src/services/alerts/alert-signal-display.ts @@ -19,7 +19,8 @@ import { AGGREGATIONS_BY_SOURCE } from "@maple/query-engine/query-builder" import type { ChartUnit } from "@maple/widgets/chart/static-chart" import { Array as Arr, String as Str } from "effect" -export type SignalUnit = "ratio" | "ms" | "rpm" | "apdex" | "count" | "plain" +/** `burn`: an SLO burn rate, a multiple of the sustainable rate shown as `14.4x`. */ +export type SignalUnit = "ratio" | "ms" | "rpm" | "apdex" | "count" | "plain" | "burn" export interface SignalDisplay { /** Human-readable name of the measured quantity, e.g. `sum(db.query.duration)`. */ @@ -45,6 +46,7 @@ const PRESET_SIGNALS: Record = { apdex: { label: "Apdex", unit: "apdex" }, throughput: { label: "Throughput", unit: "rpm" }, uptime: { label: "Uptime check", unit: "plain" }, + slo_burn_rate: { label: "SLO burn rate", unit: "burn" }, } satisfies Record /** @@ -146,6 +148,7 @@ export const mapSignalUnit = (unit: SignalUnit): ChartUnit => { case "apdex": case "count": case "plain": + case "burn": return "number" } } diff --git a/packages/backend/src/services/alerts/alerts-service-test-support.ts b/packages/backend/src/services/alerts/alerts-service-test-support.ts index 4bee096407..6102371ff1 100644 --- a/packages/backend/src/services/alerts/alerts-service-test-support.ts +++ b/packages/backend/src/services/alerts/alerts-service-test-support.ts @@ -34,6 +34,7 @@ import { AlertRuntime, type AlertRuntimeApi, AlertsService, type AlertsServiceAp import { AlertDestinationsService } from "./AlertDestinationsService" import { AlertReadModelsService } from "./AlertReadModelsService" import { AlertRulesService } from "./AlertRulesService" +import { SloAlertSource } from "@maple/backend/services/slos/SloAlertSource" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" import { BucketCacheService } from "@maple/query-engine/caching" import { EdgeCacheService } from "@maple/cache" @@ -327,6 +328,11 @@ export const makeLayer = ( Layer.provide(alertReadModelsLive), Layer.provide(alertRulesLive), Layer.provide(UptimeAlertSource.layer.pipe(Layer.provide(databaseLive))), + Layer.provide( + Layer.effect(SloAlertSource, SloAlertSource.make).pipe( + Layer.provide(Layer.mergeAll(databaseLive, warehouseLive)), + ), + ), Layer.provide(chatSessions === undefined ? Layer.empty : Layer.succeed(ChatSessions, chatSessions)), ) return Layer.mergeAll(alertDestinationsLive, alertReadModelsLive, alertRulesLive, alertsLive).pipe( diff --git a/packages/backend/src/services/slos/SloAlertSource.ts b/packages/backend/src/services/slos/SloAlertSource.ts new file mode 100644 index 0000000000..eb731d6a02 --- /dev/null +++ b/packages/backend/src/services/slos/SloAlertSource.ts @@ -0,0 +1,79 @@ +import * as PG from "@maple-dev/effect-orm/postgres" +import { Slos } from "@maple/db/tables" +import { SloPersistenceError, type WarehouseReadError } from "@maple/domain/http" +import type { AlertRuleId, OrgId } from "@maple/domain/primitives" +import type { SloBurnWindow } from "@maple/domain/slo" +import { Context, Effect, Layer, Option } from "effect" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { makeSloPersistenceError } from "./slo-persistence" +import { systemTenant } from "@maple/backend/services/alerts/system-tenant" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { readBurnWindows } from "./slo-reads" + +export interface SloAlertSourceApi { + /** The burn windows ending at `endMs` of the SLO that owns `ruleId`; `None` when none does. */ + readonly observe: ( + orgId: OrgId, + ruleId: AlertRuleId, + endMs: number, + ) => Effect.Effect>, SloPersistenceError | WarehouseReadError> +} + +/** + * The alert evaluator's view of SLOs: its own service, with only the database and the + * warehouse, so the alerting worker takes on nothing else of the SLO API. + */ +export class SloAlertSource extends Context.Service()( + "@maple/backend/services/slos/SloAlertSource", + { + make: Effect.gen(function* () { + const database = yield* Database + const dbExecute = makeDbExecute(database, "SloAlertSource", makeSloPersistenceError) + const warehouse = yield* WarehouseQueryService + + const observe = Effect.fn("SloAlertSource.observe")(function* ( + orgId: OrgId, + ruleId: AlertRuleId, + endMs: number, + ) { + const rows = yield* dbExecute((db) => + db.run( + PG.from(Slos) + .select( + "id", + "orgId", + "serviceName", + "environmentsJson", + "sliJson", + "targetPercent", + "windowDays", + ) + .where(($) => [$.orgId.eq(orgId), $.alertRuleId.eq(ruleId)]) + .limit(1), + ), + ) + const slo = rows[0] + if (slo === undefined) return Option.none>() + yield* Effect.annotateCurrentSpan({ "maple.slo.id": slo.id }) + const windows = yield* readBurnWindows( + systemTenant(orgId), + { + orgId, + serviceName: slo.serviceName, + environments: slo.environmentsJson, + sli: slo.sliJson, + targetPercent: slo.targetPercent, + windowDays: slo.windowDays, + }, + endMs, + ).pipe(Effect.provideService(WarehouseQueryService, warehouse)) + return Option.some(windows) + }) + + return { observe } satisfies SloAlertSourceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(WarehouseQueryService.layer)) +} diff --git a/packages/backend/src/services/slos/SlosService.ts b/packages/backend/src/services/slos/SlosService.ts index 0afa69cf0e..33643bb8f2 100644 --- a/packages/backend/src/services/slos/SlosService.ts +++ b/packages/backend/src/services/slos/SlosService.ts @@ -1,28 +1,41 @@ import { randomUUID } from "node:crypto" import * as PG from "@maple-dev/effect-orm/postgres" -import { Slos, type SloRow } from "@maple/db/tables" +import { AlertDestinations, AlertRuleClaims, AlertRules, Slos } from "@maple/db/tables" import { SloNotFoundError, SloPersistenceError, SloValidationError, + AlertThresholds, type WarehouseReadError, } from "@maple/domain/http" -import { type AlertRuleId, type OrgId, SloId, SloWindowDays, type UserId } from "@maple/domain/primitives" +import { + type AlertDestinationId, + AlertRuleId, + type OrgId, + SloId, + SloWindowDays, + type UserId, +} from "@maple/domain/primitives" import { budgetRemaining, sliRatio, SloTargetPercent, type SloBurnWindow, type SloHistoryPoint, + sloRuleName, type SloSli, } from "@maple/domain/slo" -import { Array as Arr, Clock, Context, Effect, Layer, Schema, String as Str } from "effect" +import { Array as Arr, Clock, Context, Effect, Layer, Option, Schema, String as Str } from "effect" import type { MapleDb } from "@maple/db/client" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { makeSloPersistenceError } from "./slo-persistence" import type { TenantContext } from "@maple/backend/services/auth/AuthService" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { deleteRuleRows } from "@maple/backend/services/alerts/AlertRulesService" +import { lockOrgAlertRules } from "@maple/backend/services/alerts/alert-persistence" import { readBudgetHistory, readBurnWindows, readStatuses, readWindowTotals } from "./slo-reads" +import { sloRuleFields, type SloRuleInput } from "./slo-alert-rule" export interface SloRecord { readonly id: SloId @@ -35,10 +48,20 @@ export interface SloRecord { readonly targetPercent: number readonly windowDays: SloWindowDays readonly alertRuleId: AlertRuleId | null + readonly alerting: SloAlerting readonly createdAt: number readonly updatedAt: number } +/** The SLO's burn-rate alert: on or off, and who each tier notifies. */ +export interface SloAlerting { + readonly enabled: boolean + /** Notified when the fast burn (critical, page) fires. */ + readonly criticalDestinationIds: ReadonlyArray + /** Notified when the slow burn (warning, ticket) fires; a critical incident notifies them too. */ + readonly warningDestinationIds: ReadonlyArray +} + export interface CreateSloInput { readonly name: string readonly description?: string | null @@ -47,6 +70,7 @@ export interface CreateSloInput { readonly sli: SloSli readonly targetPercent: number readonly windowDays?: SloWindowDays + readonly alerting?: Partial readonly createdBy: UserId } @@ -58,6 +82,7 @@ export interface UpdateSloInput { readonly sli?: SloSli readonly targetPercent?: number readonly windowDays?: SloWindowDays + readonly alerting?: Partial readonly updatedBy: UserId } @@ -113,11 +138,8 @@ export interface SlosServiceApi { export const DEFAULT_SLO_WINDOW_DAYS: SloWindowDays = 30 -const toPersistenceError = (error: unknown) => - new SloPersistenceError({ - message: error instanceof Error ? error.message : "SLO persistence failed", - cause: error, - }) +/** Each SLO's rule reads the warehouse every minute; the cap bounds that per org. */ +export const MAX_SLOS_PER_ORG = 50 const invalid = (message: string) => new SloValidationError({ message }) const notFound = (sloId: SloId) => new SloNotFoundError({ sloId, message: "SLO not found" }) @@ -140,7 +162,58 @@ const validateWindow = (value: number) => const normalizeEnvironments = (environments: ReadonlyArray) => Arr.dedupe(Arr.filter(Arr.map(environments, Str.trim), Str.isNonEmpty)) -const toRecord = (row: Omit): SloRecord => ({ +/** Every SLO with its burn-rate rule, which carries the alerting settings. */ +const selectSlos = () => + PG.from(Slos, "s") + .leftJoin(AlertRules, "r", (slo, rule) => slo.alertRuleId.eq(rule.id).and(slo.orgId.eq(rule.orgId))) + .select(($) => ({ + id: $.id, + orgId: $.orgId, + name: $.name, + description: $.description, + serviceName: $.serviceName, + environmentsJson: $.environmentsJson, + sliJson: $.sliJson, + targetPercent: $.targetPercent, + windowDays: $.windowDays, + alertRuleId: $.alertRuleId, + ruleEnabled: $.r.enabled, + ruleThresholdsJson: $.r.thresholdsJson, + createdAt: $.createdAt, + updatedAt: $.updatedAt, + })) + +interface SloSelectRow { + readonly id: SloId + readonly orgId: OrgId + readonly name: string + readonly description: string | null + readonly serviceName: string + readonly environmentsJson: ReadonlyArray + readonly sliJson: SloSli + readonly targetPercent: number + readonly windowDays: SloWindowDays + readonly alertRuleId: AlertRuleId | null + readonly ruleEnabled: boolean | null + /** Untyped jsonb: read through `decodeThresholds`. */ + readonly ruleThresholdsJson: unknown + readonly createdAt: number + readonly updatedAt: number +} + +const decodeThresholds = Schema.decodeUnknownOption(AlertThresholds) + +/** A tier's destinations; none when the rule is missing (an SLO saved before burn alerts). */ +const tierDestinations = (row: SloSelectRow, severity: "critical" | "warning") => + decodeThresholds(row.ruleThresholdsJson).pipe( + Option.flatMap((tiers) => Arr.findFirst(tiers, (tier) => tier.severity === severity)), + Option.match({ + onNone: (): ReadonlyArray => [], + onSome: (tier) => tier.destinationIds, + }), + ) + +const toRecord = (row: SloSelectRow): SloRecord => ({ id: row.id, orgId: row.orgId, name: row.name, @@ -151,25 +224,49 @@ const toRecord = (row: Omit): SloRecord => ({ targetPercent: row.targetPercent, windowDays: row.windowDays, alertRuleId: row.alertRuleId, + alerting: { + enabled: row.ruleEnabled ?? false, + criticalDestinationIds: tierDestinations(row, "critical"), + warningDestinationIds: tierDestinations(row, "warning"), + }, createdAt: row.createdAt, updatedAt: row.updatedAt, }) -const selectSlos = () => - PG.from(Slos).select( - "id", - "orgId", - "name", - "description", - "serviceName", - "environmentsJson", - "sliJson", - "targetPercent", - "windowDays", - "alertRuleId", - "createdAt", - "updatedAt", +/** Checks a burn-rate rule write: its destinations exist and its name is free among alert rules. */ +const claimRuleWrite = Effect.fnUntraced(function* ( + db: MapleDb, + orgId: OrgId, + ruleId: AlertRuleId, + rule: SloRuleInput, +) { + const destinationIds = Arr.dedupe([...rule.criticalDestinationIds, ...rule.warningDestinationIds]) + if (destinationIds.length > 0) { + const found = yield* db.run( + PG.from(AlertDestinations) + .select("id") + .where(($) => [$.orgId.eq(orgId), $.id.in_(...destinationIds)]), + ) + const known = new Set(Arr.map(found, (row) => row.id)) + const missing = Arr.findFirst(destinationIds, (id) => !known.has(id)) + if (Option.isSome(missing)) return yield* invalid(`Unknown alert destination ${missing.value}`) + } + const ruleName = sloRuleName(rule.name) + const sameName = yield* db.run( + PG.from(AlertRules) + .select("id") + .where(($) => [$.orgId.eq(orgId), $.name.eq(ruleName), $.id.neq(ruleId)]) + .limit(1), ) + if (sameName.length > 0) { + return yield* invalid( + `An alert rule named '${ruleName}' already exists; rename it or pick another SLO name`, + ) + } +}) + +const sameIds = (a: ReadonlyArray, b: ReadonlyArray) => + a.length === b.length && Arr.every(a, (id, index) => id === b[index]) /** Names are unique per org: the list, incidents and notifications identify an SLO by it. */ const ensureNameFree = Effect.fnUntraced(function* (db: MapleDb, orgId: OrgId, name: string, sloId: SloId) { @@ -187,7 +284,7 @@ export class SlosService extends Context.Service()( { make: Effect.gen(function* () { const database = yield* Database - const dbExecute = makeDbExecute(database, "SlosService", toPersistenceError) + const dbExecute = makeDbExecute(database, "SlosService", makeSloPersistenceError) const warehouse = yield* WarehouseQueryService const withWarehouse = Effect.provideService(WarehouseQueryService, warehouse) const nowMinute = Effect.map(Clock.currentTimeMillis, (now) => Math.floor(now / 60_000) * 60_000) @@ -229,15 +326,48 @@ export class SlosService extends Context.Service()( yield* Effect.annotateCurrentSpan({ orgId, "maple.slo.sli": input.sli._tag }) const name = yield* requireText("name", input.name) const serviceName = yield* requireText("service_name", input.serviceName) - const sli = input.sli const targetPercent = yield* validateTarget(input.targetPercent) const windowDays = yield* validateWindow(input.windowDays ?? DEFAULT_SLO_WINDOW_DAYS) + const environments = normalizeEnvironments(input.environments ?? []) const now = yield* Clock.currentTimeMillis const id = SloId.make(randomUUID()) + const ruleId = AlertRuleId.make(randomUUID()) + const rule: SloRuleInput = { + name, + serviceName, + environments, + enabled: input.alerting?.enabled ?? true, + criticalDestinationIds: Arr.dedupe(input.alerting?.criticalDestinationIds ?? []), + warningDestinationIds: Arr.dedupe(input.alerting?.warningDestinationIds ?? []), + } yield* dbExecute((db) => db.transaction( Effect.gen(function* () { + yield* lockOrgAlertRules(db, orgId) + const [existing] = yield* db.run( + PG.from(Slos) + .select(() => ({ count: PG.count() })) + .where(($) => [$.orgId.eq(orgId)]), + ) + if (Number(existing?.count ?? 0) >= MAX_SLOS_PER_ORG) { + return yield* invalid( + `An organization can have at most ${MAX_SLOS_PER_ORG} SLOs`, + ) + } yield* ensureNameFree(db, orgId, name, id) + yield* claimRuleWrite(db, orgId, ruleId, rule) + yield* db.run( + PG.insertInto(AlertRules).values({ + id: ruleId, + orgId, + ...sloRuleFields(rule), + lastScheduledAt: null, + createdAt: now, + updatedAt: now, + createdBy: input.createdBy, + updatedBy: input.createdBy, + }), + ) yield* db.run( PG.insertInto(Slos).values({ id, @@ -245,11 +375,11 @@ export class SlosService extends Context.Service()( name, description: input.description ?? null, serviceName, - environmentsJson: normalizeEnvironments(input.environments ?? []), - sliJson: sli, + environmentsJson: environments, + sliJson: input.sli, targetPercent, windowDays, - alertRuleId: null, + alertRuleId: ruleId, createdBy: input.createdBy, updatedBy: input.createdBy, createdAt: now, @@ -277,54 +407,133 @@ export class SlosService extends Context.Service()( input.serviceName === undefined ? undefined : yield* requireText("service_name", input.serviceName) - const sli = input.sli const targetPercent = input.targetPercent === undefined ? undefined : yield* validateTarget(input.targetPercent) const windowDays = input.windowDays === undefined ? undefined : yield* validateWindow(input.windowDays) + const environments = + input.environments === undefined ? undefined : normalizeEnvironments(input.environments) const now = yield* Clock.currentTimeMillis - const updated = yield* dbExecute((db) => + yield* dbExecute((db) => db.transaction( Effect.gen(function* () { + // Lock, then re-read: the rule is rebuilt from a row no other write changes + // before commit, and a concurrent delete either finished first or waits. + yield* lockOrgAlertRules(db, orgId) + const lockedRows = yield* db.run( + selectSlos() + .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) + .limit(1), + ) + const lockedRow = lockedRows[0] + if (lockedRow === undefined) return yield* notFound(sloId) + const locked = toRecord(lockedRow) + // No rule yet (saved before burn alerts), or its row is gone: write one. + const ruleMissing = locked.alertRuleId === null || lockedRow.ruleEnabled === null if (name !== undefined) yield* ensureNameFree(db, orgId, name, sloId) - return yield* db.run( + const rule: SloRuleInput = { + name: name ?? locked.name, + serviceName: serviceName ?? locked.serviceName, + environments: environments ?? locked.environments, + // A missing rule comes back switched on. + enabled: input.alerting?.enabled ?? (ruleMissing || locked.alerting.enabled), + criticalDestinationIds: + input.alerting?.criticalDestinationIds === undefined + ? locked.alerting.criticalDestinationIds + : Arr.dedupe(input.alerting.criticalDestinationIds), + warningDestinationIds: + input.alerting?.warningDestinationIds === undefined + ? locked.alerting.warningDestinationIds + : Arr.dedupe(input.alerting.warningDestinationIds), + } + // The rule is written only when it changes: alert_rules is synced to browsers. + const ruleChanged = + ruleMissing || + rule.name !== locked.name || + rule.serviceName !== locked.serviceName || + !sameIds(rule.environments, locked.environments) || + rule.enabled !== locked.alerting.enabled || + !sameIds( + rule.criticalDestinationIds, + locked.alerting.criticalDestinationIds, + ) || + !sameIds(rule.warningDestinationIds, locked.alerting.warningDestinationIds) + const ruleId = locked.alertRuleId ?? AlertRuleId.make(randomUUID()) + if (ruleChanged) { + yield* claimRuleWrite(db, orgId, ruleId, rule) + if (ruleMissing) { + yield* db.run( + PG.insertInto(AlertRules).values({ + id: ruleId, + orgId, + ...sloRuleFields(rule), + lastScheduledAt: null, + createdAt: now, + updatedAt: now, + createdBy: input.updatedBy, + updatedBy: input.updatedBy, + }), + ) + } else { + // Ends a tick's claim on the old config, as a rule edit does. + yield* db.run( + PG.deleteFrom(AlertRuleClaims).where(($) => [$.ruleId.eq(ruleId)]), + ) + yield* db.run( + PG.update(AlertRules) + .set({ + ...sloRuleFields(rule), + updatedAt: now, + updatedBy: input.updatedBy, + }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(ruleId)]), + ) + } + } + yield* db.run( PG.update(Slos) .set({ + ...(locked.alertRuleId === null + ? { alertRuleId: ruleId } + : undefined), ...(name === undefined ? undefined : { name }), ...(input.description === undefined ? undefined : { description: input.description }), ...(serviceName === undefined ? undefined : { serviceName }), - ...(input.environments === undefined + ...(environments === undefined ? undefined - : { - environmentsJson: normalizeEnvironments( - input.environments, - ), - }), - ...(sli === undefined ? undefined : { sliJson: sli }), + : { environmentsJson: environments }), + ...(input.sli === undefined ? undefined : { sliJson: input.sli }), ...(targetPercent === undefined ? undefined : { targetPercent }), ...(windowDays === undefined ? undefined : { windowDays }), updatedBy: input.updatedBy, updatedAt: now, }) - .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) - .returning("id"), + .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]), ) }), ), ) - if (updated.length === 0) return yield* notFound(sloId) return yield* selectOne(orgId, sloId) }) const remove = Effect.fn("SlosService.delete")(function* (orgId: OrgId, sloId: SloId) { yield* Effect.annotateCurrentSpan({ orgId, "maple.slo.id": sloId }) + // The burn-rate rule goes with the SLO, through the rule delete's own cascade. const removed = yield* dbExecute((db) => - db.run( - PG.deleteFrom(Slos) - .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) - .returning("id"), + db.transaction( + Effect.gen(function* () { + yield* lockOrgAlertRules(db, orgId) + const rows = yield* db.run( + PG.deleteFrom(Slos) + .where(($) => [$.orgId.eq(orgId), $.id.eq(sloId)]) + .returning("id", "alertRuleId"), + ) + const ruleId = rows[0]?.alertRuleId ?? null + if (ruleId !== null) yield* deleteRuleRows(db, orgId, ruleId) + return rows + }), ), ) if (removed.length === 0) return yield* notFound(sloId) diff --git a/packages/backend/src/services/slos/slo-alert-rule.test.ts b/packages/backend/src/services/slos/slo-alert-rule.test.ts new file mode 100644 index 0000000000..ec8339a6c5 --- /dev/null +++ b/packages/backend/src/services/slos/slo-alert-rule.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest" +import { Array as Arr } from "effect" +import { SLO_BURN_WINDOWS_MINUTES, type SloBurnWindow } from "@maple/domain/slo" +import { evaluateSloBurn } from "./slo-alert-rule" + +/** Every burn window at the given rates, keyed by window length; 1000 events each. */ +const windows = (rates: Record): ReadonlyArray => + Arr.map(SLO_BURN_WINDOWS_MINUTES, (windowMinutes) => ({ + windowMinutes, + total: 1000, + good: 1000, + burnRate: rates[windowMinutes] ?? 0, + })) + +describe("evaluateSloBurn", () => { + it("names the firing tier's windows", () => { + const evaluation = evaluateSloBurn(windows({ 5: 8, 30: 8, 60: 7, 360: 7 })) + expect(evaluation.status).toBe("breached") + expect(evaluation.reason).toBe( + "SLO burning the error budget at 7x over 6h and 8x over 30m (warning at 6x)", + ) + }) + + it("names every tier on recovery, so a resolved warning reports its own windows", () => { + const evaluation = evaluateSloBurn(windows({ 5: 0.2, 30: 0.3, 60: 0.4, 360: 0.5 })) + expect(evaluation.status).toBe("healthy") + expect(evaluation.reason).toBe( + "SLO within budget: 0.4x over 1h and 0.2x over 5m (critical at 14.4x); 0.5x over 6h and 0.3x over 30m (warning at 6x)", + ) + }) + + it("holds the critical tier while its 5m window is empty and still fires the warning", () => { + const evaluation = evaluateSloBurn( + Arr.map(windows({ 30: 8, 60: 20, 360: 7 }), (window) => + window.windowMinutes === 5 ? { ...window, total: 0, good: 0 } : window, + ), + ) + expect(evaluation.status).toBe("breached") + expect(evaluation.levels).toEqual([ + { severity: "warning", status: "breach" }, + { severity: "critical", status: "hold" }, + ]) + }) + + it("holds the critical tier while its 5m window is empty even as the warning clears", () => { + const evaluation = evaluateSloBurn( + Arr.map(windows({ 30: 0.5, 60: 20, 360: 0.5 }), (window) => + window.windowMinutes === 5 ? { ...window, total: 0, good: 0 } : window, + ), + ) + // Healthy overall, but the held tier keeps an open critical incident from resolving. + expect(evaluation.status).toBe("healthy") + expect(evaluation.levels).toEqual([ + { severity: "warning", status: "clear" }, + { severity: "critical", status: "hold" }, + ]) + }) + + it("skips as no data when every short window is empty", () => { + const empty = Arr.map(windows({}), (window) => + window.windowMinutes === 5 || window.windowMinutes === 30 + ? { ...window, total: 0, good: 0 } + : window, + ) + expect(evaluateSloBurn(empty)).toMatchObject({ status: "skipped", skipReason: "no_data", levels: [] }) + }) +}) diff --git a/packages/backend/src/services/slos/slo-alert-rule.ts b/packages/backend/src/services/slos/slo-alert-rule.ts new file mode 100644 index 0000000000..5cf620eab2 --- /dev/null +++ b/packages/backend/src/services/slos/slo-alert-rule.ts @@ -0,0 +1,151 @@ +// The hidden `slo_burn_rate` alert rule behind each SLO, and what one evaluation of it observes. +import type { AlertEvaluation } from "@maple/alerting-core" +import type { AlertDestinationId, AlertThresholds } from "@maple/domain/http" +import { + SLO_BURN_TIERS, + SLO_BURN_LOOKBACK_MINUTES, + burnWindowOf, + sloRuleName, + tierState, + windowLabel, + type SloBurnTier, + type SloBurnWindow, +} from "@maple/domain/slo" +import { Array as Arr, Option } from "effect" +import { MANAGED_RULE_PLAN } from "@maple/backend/services/alerts/AlertRuleModel" + +const [CRITICAL_TIER, WARNING_TIER] = SLO_BURN_TIERS + +export interface SloRuleInput { + readonly name: string + readonly serviceName: string + readonly environments: ReadonlyArray + readonly enabled: boolean + readonly criticalDestinationIds: ReadonlyArray + readonly warningDestinationIds: ReadonlyArray +} + +/** + * The `alert_rules` columns of an SLO's burn-rate rule: a warning tier (ticket) and a critical + * tier (page) at their burn factors, checked every minute. One breached check opens and one + * healthy check resolves: the short windows already keep it from flapping. + */ +export const sloRuleFields = (slo: SloRuleInput) => { + const thresholds: AlertThresholds = [ + { + severity: "warning", + value: WARNING_TIER.factor, + upper: null, + recoveryValue: null, + destinationIds: [...slo.warningDestinationIds], + }, + { + severity: "critical", + value: CRITICAL_TIER.factor, + upper: null, + recoveryValue: null, + destinationIds: [...slo.criticalDestinationIds], + }, + ] + return { + name: sloRuleName(slo.name), + notes: null, + notificationTemplateJson: null, + enabled: slo.enabled, + severity: "critical", + serviceNamesJson: [slo.serviceName], + excludeServiceNamesJson: null, + environmentsJson: slo.environments.length === 0 ? null : [...slo.environments], + tagsJson: null, + groupBy: null, + signalType: "slo_burn_rate", + comparator: "gte", + threshold: WARNING_TIER.factor, + thresholdUpper: null, + thresholdsJson: thresholds, + windowMinutes: SLO_BURN_LOOKBACK_MINUTES, + evaluationIntervalMinutes: 1, + slowedIntervalMinutes: null, + slowedAt: null, + minimumSampleCount: 0, + consecutiveBreachesRequired: 1, + breachesOutOf: null, + consecutiveHealthyRequired: 1, + renotifyIntervalMinutes: 60, + apdexThresholdMs: null, + queryBuilderDraftJson: null, + rawQuerySql: null, + destinationIdsJson: Arr.dedupe([...slo.criticalDestinationIds, ...slo.warningDestinationIds]), + querySpecJson: null, + reducer: MANAGED_RULE_PLAN.reducer, + sampleCountStrategy: null, + noDataBehavior: MANAGED_RULE_PLAN.noDataBehavior, + } as const +} + +const formatRate = (rate: number) => `${rate >= 100 ? Math.round(rate) : Math.round(rate * 10) / 10}x` + +const describeTier = (tier: SloBurnTier, windows: ReadonlyArray) => { + const rate = (minutes: number) => + Option.match(burnWindowOf(windows, minutes), { onNone: () => 0, onSome: (window) => window.burnRate }) + const long = rate(tier.longWindowMinutes) + const short = rate(tier.shortWindowMinutes) + const rates = `${formatRate(long)} over ${windowLabel(tier.longWindowMinutes)} and ${formatRate(short)} over ${windowLabel(tier.shortWindowMinutes)}` + return { + long, + text: `burning the error budget at ${rates} (${tier.severity} at ${tier.factor}x)`, + rates: `${rates} (${tier.severity} at ${tier.factor}x)`, + } +} + +/** + * One check of an SLO's burn-rate rule. Each tier is judged on its own pair of windows, so the + * levels are set here rather than by comparing one value to every tier. A tier whose short + * window saw no events holds its state; with every tier held the check is skipped as no data. + * The value is the long-window burn of the highest tier that fires, else the 1h burn. + */ +export const evaluateSloBurn = (windows: ReadonlyArray): AlertEvaluation => { + const tiers = [WARNING_TIER, CRITICAL_TIER] + const levels = Arr.map(tiers, (tier) => ({ + severity: tier.severity, + status: tierState(tier, windows), + })) + const hour = Option.getOrUndefined(burnWindowOf(windows, CRITICAL_TIER.longWindowMinutes)) + const base = { + sampleCount: Math.round(hour?.total ?? 0), + threshold: WARNING_TIER.factor, + thresholdUpper: null, + comparator: "gte" as const, + derivedFromNoData: false, + } + if (Arr.every(levels, (level) => level.status === "hold")) { + return { + ...base, + status: "skipped", + value: null, + reason: "No events in the SLO's short burn windows", + skipReason: "no_data", + levels: [], + } + } + const firing = Arr.findFirst([CRITICAL_TIER, WARNING_TIER], (tier) => + Arr.some(levels, (level) => level.severity === tier.severity && level.status === "breach"), + ) + const headline = describeTier( + Option.getOrElse(firing, () => CRITICAL_TIER), + windows, + ) + return { + ...base, + status: Option.isSome(firing) ? "breached" : "healthy", + levels, + value: headline.long, + // A recovery names every tier, so a resolved warning reports the windows it fired on. + reason: Option.isSome(firing) + ? `SLO ${headline.text}` + : `SLO within budget: ${Arr.join( + Arr.map([CRITICAL_TIER, WARNING_TIER], (tier) => describeTier(tier, windows).rates), + "; ", + )}`, + } +} diff --git a/packages/backend/src/services/slos/slo-persistence.ts b/packages/backend/src/services/slos/slo-persistence.ts new file mode 100644 index 0000000000..f2e62ec5bb --- /dev/null +++ b/packages/backend/src/services/slos/slo-persistence.ts @@ -0,0 +1,8 @@ +import { SloPersistenceError } from "@maple/domain/http" +import { makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" + +/** Shared `unknown -> SloPersistenceError` mapper for the SLO services. */ +export const makeSloPersistenceError = makePersistenceErrorMapper( + SloPersistenceError, + "SLO persistence failed", +) diff --git a/packages/backend/src/services/slos/slo-reads.ts b/packages/backend/src/services/slos/slo-reads.ts index 4e0dfda470..f37a20338d 100644 --- a/packages/backend/src/services/slos/slo-reads.ts +++ b/packages/backend/src/services/slos/slo-reads.ts @@ -97,7 +97,7 @@ export const readBudgetHistory = Effect.fn("SloReads.budgetHistory")(function* ( tenant, CH.compile(CH.sloBucketsQuery(queryOpts(slo), "hour"), { orgId: slo.orgId, - startTime: formatWarehouseDateTime(Math.ceil(endMs / HOUR_MS) * HOUR_MS - 2 * windowMs), + startTime: formatWarehouseDateTime(Math.ceil(endMs / HOUR_MS) * HOUR_MS - 2 * windowMs + HOUR_MS), endTime: formatWarehouseDateTime(endMs), bucketSeconds: 3600, }), diff --git a/packages/domain/src/http/alerts.ts b/packages/domain/src/http/alerts.ts index d349a5a966..297382782f 100644 --- a/packages/domain/src/http/alerts.ts +++ b/packages/domain/src/http/alerts.ts @@ -49,12 +49,23 @@ export const AlertSignalType = Schema.Literals([ "raw_query", /** The hidden rule behind an uptime monitor; managed only through the monitor. */ "uptime", + /** The hidden burn-rate rule behind an SLO; managed only through the SLO. */ + "slo_burn_rate", ]).annotate({ identifier: "@maple/AlertSignalType", title: "Alert Signal Type", }) export type AlertSignalType = Schema.Schema.Type +/** Signal types whose rule another resource owns: hidden from and refused by the generic rule APIs. */ +export const MANAGED_SIGNAL_TYPES = [ + "uptime", + "slo_burn_rate", +] as const satisfies ReadonlyArray +export type ManagedSignalType = (typeof MANAGED_SIGNAL_TYPES)[number] +export const isManagedSignalType = (signalType: string): signalType is ManagedSignalType => + Arr.some(MANAGED_SIGNAL_TYPES, (managed) => managed === signalType) + export const AlertGroupByDimension = Schema.String.check(Schema.isMinLength(1), Schema.isTrimmed()).annotate({ identifier: "@maple/AlertGroupByDimension", title: "Alert Group By Dimension", diff --git a/packages/domain/src/http/v2/slos.ts b/packages/domain/src/http/v2/slos.ts index b9fc38e38a..6d358371f8 100644 --- a/packages/domain/src/http/v2/slos.ts +++ b/packages/domain/src/http/v2/slos.ts @@ -9,7 +9,7 @@ import { V2InsufficientPermissions, V2ParameterInvalid } from "./errors" import { V2WarehouseReadErrors } from "./query-errors" import { publicErrors } from "./public-error" import { PublicId, PublicIdPrefixes } from "./public-id" -import { AlertRulePublicId } from "./resource-ids" +import { AlertDestinationPublicId, AlertRulePublicId } from "./resource-ids" /** `slo_…` public ID ⇄ internal `SloId` (raw UUID). */ export const SloPublicId = PublicId(PublicIdPrefixes.slo, SloId) @@ -69,6 +69,36 @@ const windowField = SloWindowDays.annotate({ examples: [30], }) +const destinationIdsField = (description: string) => + Schema.Array(AlertDestinationPublicId).annotate({ description }) + +export const V2SloAlerting = Schema.Struct({ + enabled: Schema.Boolean.annotate({ + description: "Whether the burn-rate alert runs. Incidents open on the SLO's alert rule.", + }), + critical_destination_ids: destinationIdsField( + "Notified when the budget burns fast: 14.4x the sustainable rate over both 1 hour and 5 minutes (page).", + ), + warning_destination_ids: destinationIdsField( + "Notified when the budget burns slowly: 6x over both 6 hours and 30 minutes (ticket). A critical incident notifies these too.", + ), +}).annotate({ + identifier: "SloAlerting", + title: "SLO burn-rate alerting", + description: "Multiwindow, multi-burn-rate alerting on the SLO's error budget.", +}) + +export const V2SloAlertingParams = Schema.Struct({ + enabled: Schema.optionalKey(V2SloAlerting.fields.enabled), + critical_destination_ids: Schema.optionalKey(V2SloAlerting.fields.critical_destination_ids), + warning_destination_ids: Schema.optionalKey(V2SloAlerting.fields.warning_destination_ids), +}).annotate({ + identifier: "SloAlertingParams", + title: "SLO burn-rate alerting parameters", + description: "Omitted fields keep their value; a new SLO alerts with no destinations until you add some.", +}) +export type V2SloAlertingParams = Schema.Schema.Type + const sloExample = { id: "slo_YofPTrK9782DWwcnXhpcCw", object: "slo", @@ -79,7 +109,12 @@ const sloExample = { sli: { type: "availability" }, target_percent: 99.9, window_days: 30, - alert_rule_id: null, + alert_rule_id: "alrt_YofPTrK9782DWwcnXhpcCw", + alerting: { + enabled: true, + critical_destination_ids: ["dest_oybbpTBhtSFGShMjjLiCrh"], + warning_destination_ids: [], + }, created_at: "2026-07-01T12:00:00.000Z", updated_at: "2026-07-15T09:12:00.000Z", } as const @@ -100,6 +135,7 @@ export const V2Slo = Schema.Struct({ alert_rule_id: Schema.NullOr(AlertRulePublicId).annotate({ description: "The burn-rate alert rule backing this SLO: its incidents carry this rule id.", }), + alerting: V2SloAlerting, created_at: Timestamp.annotate({ description: "When the SLO was created." }), updated_at: Timestamp.annotate({ description: "When the SLO was last updated." }), }).annotate({ @@ -119,6 +155,7 @@ export const V2SloCreateParams = Schema.Struct({ sli: V2SloSli, target_percent: targetField, window_days: Schema.optionalKey(windowField.annotate({ description: "Defaults to 30." })), + alerting: Schema.optionalKey(V2SloAlertingParams), }).annotate({ identifier: "SloCreateParams", title: "SLO create parameters", @@ -143,6 +180,7 @@ export const V2SloUpdateParams = Schema.Struct({ sli: Schema.optionalKey(V2SloSli), target_percent: Schema.optionalKey(targetField), window_days: Schema.optionalKey(windowField), + alerting: Schema.optionalKey(V2SloAlertingParams), }).annotate({ identifier: "SloUpdateParams", title: "SLO update parameters", diff --git a/packages/domain/src/slo.ts b/packages/domain/src/slo.ts index b88344fcda..c44829aa77 100644 --- a/packages/domain/src/slo.ts +++ b/packages/domain/src/slo.ts @@ -1,4 +1,4 @@ -import { Array as Arr, Option, Schema } from "effect" +import { Array as Arr, Option, Schema, String as Str } from "effect" import { SloLatencyThresholdMs } from "./primitives" // Service level objectives: what counts as a good event (the SLI), the target share of good @@ -68,14 +68,45 @@ export interface SloBurnWindow extends SloCounts { readonly burnRate: number } +/** The window of the given length, if read. */ +export const burnWindowOf = ( + windows: ReadonlyArray, + minutes: number, +): Option.Option => Arr.findFirst(windows, (window) => window.windowMinutes === minutes) + +const burnOf = (windows: ReadonlyArray, minutes: number): number => + Option.match(burnWindowOf(windows, minutes), { onNone: () => 0, onSome: (window) => window.burnRate }) + /** Whether a tier fires on the given window burns. */ -export const tierBreached = (tier: SloBurnTier, windows: ReadonlyArray): boolean => { - const rateOf = (minutes: number) => - Arr.findFirst(windows, (window) => window.windowMinutes === minutes).pipe( - Option.match({ onNone: () => 0, onSome: (window) => window.burnRate }), - ) - return rateOf(tier.longWindowMinutes) >= tier.factor && rateOf(tier.shortWindowMinutes) >= tier.factor -} +export const tierBreached = (tier: SloBurnTier, windows: ReadonlyArray): boolean => + burnOf(windows, tier.longWindowMinutes) >= tier.factor && + burnOf(windows, tier.shortWindowMinutes) >= tier.factor + +/** + * Whether a tier can be judged: its short window saw events. An empty short window is an + * outage or an ingest gap as often as a quiet service, so it neither fires nor resolves. + */ +export const tierObserved = (tier: SloBurnTier, windows: ReadonlyArray): boolean => + Option.match(burnWindowOf(windows, tier.shortWindowMinutes), { + onNone: () => false, + onSome: (window) => window.total > 0, + }) + +/** How a tier reads now: firing, holding its state (no events in its short window), or clear. */ +export type SloTierState = "breach" | "hold" | "clear" + +export const tierState = (tier: SloBurnTier, windows: ReadonlyArray): SloTierState => + !tierObserved(tier, windows) ? "hold" : tierBreached(tier, windows) ? "breach" : "clear" + +/** `5m`, `30m`, `1h`, `6h`. */ +export const windowLabel = (minutes: number): string => (minutes >= 60 ? `${minutes / 60}h` : `${minutes}m`) + +/** The hidden burn-rate rule's name: it titles incidents and notifications. */ +export const SLO_RULE_NAME_PREFIX = "SLO burn: " +export const sloRuleName = (sloName: string): string => `${SLO_RULE_NAME_PREFIX}${sloName}` +/** The SLO name inside a burn-rate rule's name, or `null` for any other rule name. */ +export const sloNameFromRuleName = (ruleName: string): string | null => + Str.startsWith(SLO_RULE_NAME_PREFIX)(ruleName) ? Str.slice(SLO_RULE_NAME_PREFIX.length)(ruleName) : null /** A bucket of events starting at `startMs`. */ export interface SloBucket extends SloCounts { @@ -114,7 +145,7 @@ export interface SloHistoryPoint { } /** - * The budget chart from buckets covering two SLO windows: one point per bucket of the last + * The budget chart from buckets covering two SLO windows less one bucket: one point per bucket of the last * window, each reading the SLO window that ends with it. Buckets are `bucketMs` wide and * aligned to it; the last point is `endMs`, inside the newest (partial) bucket. */ @@ -126,7 +157,8 @@ export const budgetHistory = (input: { readonly targetPercent: number }): ReadonlyArray => { const slotsPerWindow = Math.round(input.windowMs / input.bucketMs) - const slotCount = 2 * slotsPerWindow + // The oldest point's window starts one bucket after the first slot's start. + const slotCount = 2 * slotsPerWindow - 1 const alignedEnd = Math.ceil(input.endMs / input.bucketMs) * input.bucketMs const origin = alignedEnd - slotCount * input.bucketMs const slots = Arr.makeBy(slotCount, () => ({ total: 0, good: 0 })) @@ -147,7 +179,7 @@ export const budgetHistory = (input: { return { total: end.total - start.total, good: end.good - start.good } } return Arr.makeBy(slotsPerWindow, (index) => { - const slot = slotsPerWindow + index + const slot = slotsPerWindow - 1 + index const last = between(slot, slot + 1) return { atMs: Math.min(origin + (slot + 1) * input.bucketMs, input.endMs), diff --git a/packages/web-data/src/alerts/form-utils.ts b/packages/web-data/src/alerts/form-utils.ts index f87216b077..4f5063039c 100644 --- a/packages/web-data/src/alerts/form-utils.ts +++ b/packages/web-data/src/alerts/form-utils.ts @@ -130,6 +130,7 @@ export const signalLabels: Record = { builder_query: "Query builder", raw_query: "Raw SQL", uptime: "Uptime check", + slo_burn_rate: "SLO burn rate", } satisfies Record export const RAW_QUERY_REDUCER_LABELS: Record = { @@ -185,6 +186,9 @@ export function formatSignalValue(signalType: AlertSignalType, value: number | n // An uptime rule observes 1 while its monitor is down, else 0. case "uptime": return value >= 1 ? "Down" : "Up" + // An SLO rule observes how many times faster than sustainable the budget burns. + case "slo_burn_rate": + return `${value.toFixed(1)}x` } } diff --git a/packages/web-data/src/alerts/overview.test.ts b/packages/web-data/src/alerts/overview.test.ts index a06be184cd..2d8cbd29fe 100644 --- a/packages/web-data/src/alerts/overview.test.ts +++ b/packages/web-data/src/alerts/overview.test.ts @@ -1,5 +1,5 @@ import { AlertIncidentDocument, AlertRuleDocument } from "@maple/domain/http" -import { Schema, Array as Arr } from "effect" +import { Order, Schema, Array as Arr } from "effect" import { describe, expect, it } from "vitest" import type { AlertRuleStateRow } from "../collections/alerts" import { deriveOverview } from "./overview" @@ -13,6 +13,7 @@ const iso = (msAgo: number) => new Date(NOW - msAgo).toISOString() const RULE_A = "00000000-0000-4000-8000-00000000000a" const RULE_B = "00000000-0000-4000-8000-00000000000b" +const RULE_C = "00000000-0000-4000-8000-00000000000c" function makeRule(overrides: Record = {}): AlertRuleDocument { return decodeRuleDoc({ @@ -204,18 +205,27 @@ describe("deriveOverview", () => { expect(overview.timelineRange).toEqual({ min: NOW - DAY_MS, max: NOW }) }) - it("hides uptime monitors' backing rules but keeps their incidents in the inbox", () => { + it("hides monitors' and SLOs' backing rules but keeps their incidents in the inbox", () => { const overview = deriveOverview({ rules: [ makeRule({ id: RULE_A }), makeRule({ id: RULE_B, signalType: "uptime", name: "API health" }), + makeRule({ id: RULE_C, signalType: "slo_burn_rate", name: "SLO burn: Checkout" }), + ], + incidents: [ + makeIncident({ ruleId: RULE_B, ruleName: "API health", signalType: "uptime" }), + makeIncident({ ruleId: RULE_C, ruleName: "SLO burn: Checkout", signalType: "slo_burn_rate" }), ], - incidents: [makeIncident({ ruleId: RULE_B, ruleName: "API health", signalType: "uptime" })], states: [], deliveryEvents: [], now: NOW, }) expect(Arr.map(overview.rules, (rule) => rule.id)).toEqual([RULE_A]) - expect(Arr.map(overview.openIncidents, (incident) => incident.ruleName)).toEqual(["API health"]) + expect( + Arr.sort( + Arr.map(overview.openIncidents, (incident) => incident.ruleName), + Order.String, + ), + ).toEqual(["API health", "SLO burn: Checkout"]) }) }) diff --git a/packages/web-data/src/alerts/overview.ts b/packages/web-data/src/alerts/overview.ts index d436d15247..1eed696d2d 100644 --- a/packages/web-data/src/alerts/overview.ts +++ b/packages/web-data/src/alerts/overview.ts @@ -1,4 +1,9 @@ -import type { AlertDeliveryEventDocument, AlertIncidentDocument, AlertRuleDocument } from "@maple/domain/http" +import { + isManagedSignalType, + type AlertDeliveryEventDocument, + type AlertIncidentDocument, + type AlertRuleDocument, +} from "@maple/domain/http" import type { AlertRuleStateRow } from "../collections/alerts" import { deriveRuleStatus, type DerivedRuleStatus, needsAttention } from "./rule-status" @@ -69,8 +74,8 @@ export const deriveOverview = (inputs: OverviewInputs): AlertsOverviewReady => { const { now } = inputs // ISO timestamps order lexicographically; newest first, as the server lists do. const byIsoDesc = (a: string, b: string) => (a < b ? 1 : a > b ? -1 : 0) - // Uptime monitors own their backing rules: the rule list hides them, their incidents stay. - const rules = Arr.filter(inputs.rules, (rule) => rule.signalType !== "uptime").sort((a, b) => + // Monitors and SLOs own their backing rules: the rule list hides them, their incidents stay. + const rules = Arr.filter(inputs.rules, (rule) => !isManagedSignalType(rule.signalType)).sort((a, b) => byIsoDesc(a.updatedAt, b.updatedAt), ) const incidentCutoff = now - INCIDENT_WINDOW_MS