feat(slos): SLI, error budget and burn rates

.github/workflows/ci.yml

@@ -469,7 +469,7 @@ jobs:
               run: bun run --cwd lib/effect-clickhouse-http test
 
             # Migration replay and search indexes, raw-vs-rollup parity for web
-            # analytics and the service map (a divergence renders as a plausible wrong number), the
+            # analytics, the service map and SLOs (a divergence renders as a plausible wrong number), the
             # cascaded trace facets rollup (one that never fires reads as "no data"), and the users
             # directory's group view.
             - name: Replay migrations and verify rollups
@@ -482,6 +482,7 @@ jobs:
                   src/services/warehouse/ci-runs.clickhouse.e2e.test.ts
                   src/services/warehouse/service-api-materialization.clickhouse.e2e.test.ts
                   src/services/warehouse/service-map-parity.clickhouse.e2e.test.ts
+                  src/services/warehouse/slo-parity.clickhouse.e2e.test.ts
                   src/services/users/users-groups.clickhouse.e2e.test.ts
 
             # The fingerprint the error views compute, against the TS mirror every

apps/api/scripts/bench-suites/slos.ts

@@ -0,0 +1,87 @@
+/**
+ * SLO reads (SlosService) against a populated snapshot, plus the flat scan the rollup splice
+ * replaces. Set BENCH_ORG and BENCH_SERVICE to a busy service; windows end at BENCH_END
+ * (`YYYY-MM-DD HH:MM:SS`, default now) so the splice edges are partial minutes and hours.
+ */
+import { Effect, Array as Arr } from "effect"
+import * as CH from "@maple/query-engine/ch"
+import * as E from "@maple-dev/effect-orm/expr"
+import * as Bench from "@maple-dev/effect-orm/benchmark"
+import { OrgId } from "@maple/domain"
+import type { SloSli } from "@maple/domain/slo"
+import { formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine/datetime"
+
+const ORG = OrgId.make(process.env.BENCH_ORG ?? "org_sql_catalog")
+const SERVICE = process.env.BENCH_SERVICE ?? "api"
+const DAY_MS = 86_400_000
+const HOUR_MS = 3_600_000
+
+const END_MS = process.env.BENCH_END ? parseWarehouseDateTime(process.env.BENCH_END) : Date.now()
+const window = (spanMs: number) => ({
+	orgId: ORG,
+	startTime: formatWarehouseDateTime(END_MS - spanMs),
+	endTime: formatWarehouseDateTime(END_MS),
+})
+
+const AVAILABILITY: SloSli = { _tag: "availability" }
+const LATENCY_500: SloSli = { _tag: "latency", thresholdMs: 500 }
+const opts = (sli: SloSli) => ({ serviceName: SERVICE, environments: [], sli })
+
+const totals = (id: string, sli: SloSli) =>
+	Bench.query({
+		id,
+		inputs: {
+			...window(30 * DAY_MS),
+			sli: sli._tag === "latency" ? `latency-${sli.thresholdMs}` : sli._tag,
+		},
+		compile: ({ orgId, startTime, endTime }) =>
+			CH.compile(CH.sloTotalsQuery(opts(sli)), { orgId, startTime, endTime }),
+		results: "unordered",
+	})
+
+const buckets = (id: string, grain: "hour" | "minute", bucketSeconds: number, spanMs: number) =>
+	Bench.query({
+		id,
+		inputs: { ...window(spanMs), bucketSeconds },
+		compile: (inputs) => CH.compile(CH.sloBucketsQuery(opts(AVAILABILITY), grain), inputs),
+		results: "ordered",
+	})
+
+/** What the splice replaces: one scan of every entry-point span in the window. */
+const flatScan = Bench.query({
+	id: "baseline/30d/flat-scan-availability",
+	inputs: window(30 * DAY_MS),
+	compile: (inputs) =>
+		CH.compile(
+			CH.from(CH.tables.ServiceOverviewSpans)
+				.select(($) => ({
+					total: E.sum($.SampleRate),
+					good: E.sumIf($.SampleRate, $.StatusCode.neq("Error")),
+				}))
+				.where(($) => [
+					$.OrgId.eq(CH.tables.orgIdParam),
+					$.ServiceName.eq(SERVICE),
+					$.Timestamp.gte(CH.tables.utcSecondsParam("startTime")),
+					$.Timestamp.lte(CH.tables.utcSecondsParam("endTime")),
+				])
+				.format("JSON"),
+			inputs,
+		),
+	results: "unordered",
+})
+
+export default Bench.defineSuite({
+	name: "slos",
+	dataset: process.env.BENCH_DATASET ?? "replace-with-your-snapshot-revision",
+	cases: Arr.map(
+		[
+			totals("totals/30d/availability", AVAILABILITY),
+			totals("totals/30d/latency-500ms", LATENCY_500),
+			buckets("burn/6h/minute-60s", "minute", 60, 6 * HOUR_MS),
+			buckets("history/60d/hour-3600s", "hour", 3600, 60 * DAY_MS),
+			flatScan,
+		],
+		// `defineSuite` takes one error type; a compile failure is a broken suite.
+		(sample) => Effect.orDie(sample),
+	),
+})

apps/api/src/routes/v2/slos.http.test.ts

@@ -14,6 +14,13 @@ import { DashboardPersistenceService } from "@maple/backend/services/dashboards/
 import { SharedDashboardService } from "@maple/backend/services/dashboards/SharedDashboardService"
 import { LiveActivitiesService } from "@maple/backend/services/push/LiveActivitiesService"
 import { MobileDevicesService } from "@maple/backend/services/push/MobileDevicesService"
+import {
+	WarehouseQueryService,
+	type WarehouseQueryServiceApi,
+} from "@maple/backend/services/warehouse/WarehouseQueryService"
+import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support"
+import { compiledQueryOf } from "@maple/query-engine/execution"
+import { formatWarehouseDateTime } from "@maple/query-engine"
 import { V2TransportErrorBoundaryLive } from "./error-envelope"
 import {
 	AlertsServiceStubLayer,
@@ -50,7 +57,18 @@ const testConfig = () =>
 		}),
 	)
 
-const makeHarness = () => {
+/** A warehouse whose SLO reads answer from fixed rows, decoded like real ones. */
+const sloWarehouse = (
+	rowsByContext: Record<string, ReadonlyArray<Record<string, unknown>>>,
+): WarehouseQueryServiceApi =>
+	makeWarehouseServiceStub({
+		compiledQuery: (_tenant, compiled, options) =>
+			compiledQueryOf(compiled)
+				.decodeRows(rowsByContext[options?.context ?? ""] ?? [])
+				.pipe(Effect.orDie),
+	})
+
+const makeHarness = (warehouse: WarehouseQueryServiceApi = makeWarehouseServiceStub()) => {
 	const testDb = createTestDb(createdDbs)
 	const envLive = Env.layer.pipe(Layer.provide(testConfig()))
 	const servicesLive = Layer.mergeAll(
@@ -64,6 +82,8 @@ const makeHarness = () => {
 
 	const routes = HttpApiBuilder.layer(MapleApiV2).pipe(
 		Layer.provide(AllV2GroupLayersLive),
+		// Ahead of the stub bundles, which carry an inert warehouse of their own.
+		Layer.provide(Layer.succeed(WarehouseQueryService, warehouse)),
 		Layer.provide(V2TransportErrorBoundaryLive),
 		Layer.provide(AlertsServiceStubLayer),
 		Layer.provide(ConfigResourceServiceStubsLayer),
@@ -116,10 +136,13 @@ const makeHarness = () => {
 	}
 }
 
-const withHarness = (body: (harness: ReturnType<typeof makeHarness>) => Promise<void>) =>
+const withHarness = (
+	body: (harness: ReturnType<typeof makeHarness>) => Promise<void>,
+	warehouse?: WarehouseQueryServiceApi,
+) =>
 	Effect.runPromise(
 		Effect.acquireUseRelease(
-			Effect.sync(makeHarness),
+			Effect.sync(() => makeHarness(warehouse)),
 			(harness) => Effect.promise(() => body(harness)),
 			(harness) => Effect.promise(() => harness.dispose()),
 		),
@@ -250,4 +273,105 @@ describe("v2 slos", () => {
 			const { secret: writer } = await harness.bootstrapKey(undefined, ["slos:write"])
 			expect((await harness.request("PATCH", path, writer, { target_percent: 99 })).status).toBe(200)
 		}))
+
+	it("reports the SLI, budget left and burn windows from the warehouse", () => {
+		const nowMinute = Math.floor(Date.now() / 60_000) * 60_000
+		// 2% of every minute is bad: 20x the 0.1% budget in every window.
+		const minutes = Arr.makeBy(400, (index) => ({
+			bucket: formatWarehouseDateTime(nowMinute - (index - 1) * 60_000),
+			total: 100,
+			good: 98,
+		}))
+		const warehouse = sloWarehouse({
+			sloWindowTotals: [{ total: 10_000, good: 9_995 }],
+			sloBurnWindows: minutes,
+		})
+		return withHarness(async (harness) => {
+			const { secret: token } = await harness.bootstrapKey()
+			const created = await harness.request("POST", "/v2/slos", token, checkoutSlo)
+			const status = await harness.request("GET", `/v2/slos/${created.body.id}/status`, token)
+			expect(status.status).toBe(200)
+			expect(status.body).toMatchObject({ object: "slo.status", slo_id: created.body.id })
+			expect(status.body.total_events).toBe(10_000)
+			expect(status.body.sli).toBeCloseTo(0.9995)
+			expect(status.body.budget_remaining).toBeCloseTo(0.5)
+			expect(
+				Arr.map(status.body.burn_windows, (window: { window_minutes: number; burn_rate: number }) => [
+					window.window_minutes,
+					Math.round(window.burn_rate),
+				]),
+			).toEqual([
+				[5, 20],
+				[30, 20],
+				[60, 20],
+				[360, 20],
+			])
+
+			// An SLO that is gone, or another org's, is a 404 on both reads.
+			const gone = "slo_YofPTrK9782DWwcnXhpcCw"
+			const missing = await Promise.all([
+				harness.request("GET", `/v2/slos/${gone}/status`, token),
+				harness.request("GET", `/v2/slos/${gone}/history`, token),
+			])
+			expect(Arr.map(missing, (response) => [response.status, response.body.error.code])).toEqual([
+				[404, "slo_not_found"],
+				[404, "slo_not_found"],
+			])
+
+			const history = await harness.request("GET", `/v2/slos/${created.body.id}/history`, token)
+			expect(history.status).toBe(200)
+			// 30 days of hourly points, untouched since the stub has no hourly rows.
+			expect(history.body.points).toHaveLength(720)
+			expect(history.body.points[0].budget_remaining).toBe(1)
+		}, warehouse)
+	})
+
+	it("reads every SLO's status in one batch, counting only each SLO's service and environments", () => {
+		const burn = (total: number, good: number) => ({
+			total5: total,
+			good5: good,
+			total30: total * 6,
+			good30: good * 6,
+			total60: total * 12,
+			good60: good * 12,
+			total360: total * 72,
+			good360: good * 72,
+		})
+		const warehouse = sloWarehouse({
+			sloStatusTotals: [
+				{ serviceName: "checkout", environment: "production", total: 10_000, good: 9_995 },
+				// Outside the checkout SLO's environments: counting it would spend the whole budget.
+				{ serviceName: "checkout", environment: "staging", total: 500, good: 0 },
+				{ serviceName: "search", environment: "production", total: 2_000, good: 2_000 },
+			],
+			sloStatusBurn: [
+				{ serviceName: "checkout", environment: "production", ...burn(100, 98) },
+				{ serviceName: "checkout", environment: "staging", ...burn(100, 0) },
+				{ serviceName: "search", environment: "production", ...burn(100, 100) },
+			],
+		})
+		return withHarness(async (harness) => {
+			const { secret: token } = await harness.bootstrapKey()
+			const checkout = await harness.request("POST", "/v2/slos", token, checkoutSlo)
+			const search = await harness.request("POST", "/v2/slos", token, {
+				name: "Search latency",
+				service_name: "search",
+				sli: { type: "latency", threshold_ms: 500 },
+				target_percent: 99,
+			})
+			const listed = await harness.request("GET", "/v2/slos/statuses", token)
+			expect(listed.status).toBe(200)
+			expect(listed.body.object).toBe("slo.status_list")
+			const byId = new Map(
+				Arr.map(listed.body.data, (status: { slo_id: string }) => [status.slo_id, status] as const),
+			)
+			const checkoutStatus = byId.get(checkout.body.id)
+			expect(checkoutStatus.total_events).toBe(10_000)
+			expect(checkoutStatus.budget_remaining).toBeCloseTo(0.5)
+			expect(Math.round(checkoutStatus.burn_windows[0].burn_rate)).toBe(20)
+			const searchStatus = byId.get(search.body.id)
+			expect(searchStatus.sli).toBe(1)
+			expect(searchStatus.burn_windows[3].burn_rate).toBe(0)
+		}, warehouse)
+	})
 })

apps/api/src/routes/v2/slos.http.ts

@@ -8,12 +8,12 @@ import {
 	sloSliToWire,
 	V2InsufficientPermissions,
 } from "@maple/domain/http/v2"
-import type { V2Slo } from "@maple/domain/http/v2"
+import type { V2Slo, V2SloStatus } from "@maple/domain/http/v2"
 import { Effect, Array as Arr } from "effect"
 import { auditDiff } from "@/routes/v2/audit-changes"
 import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService"
 import * as Policy from "@maple/domain/policy"
-import { type SloRecord, SlosService } from "@maple/backend/services/slos/SlosService"
+import { type SloRecord, type SloStatus, SlosService } from "@maple/backend/services/slos/SlosService"
 
 // An SLO owns a burn-rate alert rule, so writes need the same permission as alert rules.
 
@@ -32,6 +32,22 @@ export const toV2Slo = (slo: SloRecord): V2Slo => ({
 	updated_at: isoTimestamp(slo.updatedAt),
 })
 
+const toV2Status = (sloId: SloRecord["id"], status: SloStatus): V2SloStatus => ({
+	object: "slo.status",
+	slo_id: sloId,
+	as_of: isoTimestamp(status.asOfMs),
+	total_events: status.total,
+	good_events: status.good,
+	sli: status.sli,
+	budget_remaining: status.budgetRemaining,
+	burn_windows: Arr.map(status.burnWindows, (window) => ({
+		window_minutes: window.windowMinutes,
+		burn_rate: window.burnRate,
+		total_events: window.total,
+		good_events: window.good,
+	})),
+})
+
 const sloAuditDiff = auditDiff({
 	fields: ["name", "description", "service_name", "environments", "sli", "target_percent", "window_days"],
 })
@@ -122,6 +138,41 @@ export const HttpV2SlosLive = HttpApiBuilder.group(MapleApiV2, "slos", (handlers
 					return toV2Slo(updated)
 				}),
 			)
+			.handle("status", ({ params }) =>
+				Effect.gen(function* () {
+					const tenant = yield* CurrentTenant.Context
+					const status = yield* service.status(tenant, params.id)
+					return toV2Status(params.id, status)
+				}),
+			)
+			.handle("statuses", () =>
+				Effect.gen(function* () {
+					const tenant = yield* CurrentTenant.Context
+					const rows = yield* service.statuses(tenant)
+					return {
+						object: "slo.status_list" as const,
+						data: Arr.map(rows, (row) => toV2Status(row.slo.id, row.status)),
+					}
+				}),
+			)
+			.handle("history", ({ params }) =>
+				Effect.gen(function* () {
+					const tenant = yield* CurrentTenant.Context
+					const history = yield* service.history(tenant, params.id)
+					return {
+						object: "slo.history" as const,
+						slo_id: params.id,
+						as_of: isoTimestamp(history.asOfMs),
+						bucket_seconds: history.bucketSeconds,
+						points: Arr.map(history.points, (point) => ({
+							at: isoTimestamp(point.atMs),
+							sli: point.sli,
+							budget_remaining: point.budgetRemaining,
+							burn_rate: point.burnRate,
+						})),
+					}
+				}),
+			)
 			.handle("delete", ({ params }) =>
 				Effect.gen(function* () {
 					const tenant = yield* requireManage

apps/api/src/routes/v2/v2-test-support.ts

@@ -194,8 +194,8 @@ const v2GroupLayersExceptOnboardingChecklist = (
 		HttpV2AuditLogLive.pipe(Layer.provide(Layer.merge(auditLog, OrgMembersServiceStubLayer))),
 		HttpV2ScrapeTargetsLive,
 		HttpV2UptimeMonitorsLive.pipe(Layer.provide(uptimeMonitors)),
-		// Real service: it needs only the Database every harness already provides.
-		HttpV2SlosLive.pipe(Layer.provide(SlosService.layer)),
+		// Real service over whichever `WarehouseQueryService` stub the test provides.
+		HttpV2SlosLive.pipe(Layer.provide(Layer.effect(SlosService, SlosService.make))),
 		// Real service: it needs only the Database every harness already provides.
 		Layer.mergeAll(HttpV2StatusPagesLive, HttpV2StatusPageIncidentsLive).pipe(
 			Layer.provide(