diff --git a/backend/src/__tests__/endpointSlaRegistry.test.ts b/backend/src/__tests__/endpointSlaRegistry.test.ts index b97df420..7161341a 100644 --- a/backend/src/__tests__/endpointSlaRegistry.test.ts +++ b/backend/src/__tests__/endpointSlaRegistry.test.ts @@ -20,4 +20,12 @@ describe('endpointSlaRegistry', () => { const summary = getEndpointSla('/api/v1/vault/summary'); expect(deposit?.p95BudgetMs).toBeGreaterThan(summary!.p95BudgetMs); }); + + it('includes p99BudgetMs annotations for all endpoints which are greater than p95BudgetMs', () => { + const registry = listEndpointSlaRegistry(); + for (const entry of registry) { + expect(entry.p99BudgetMs).toBeDefined(); + expect(entry.p99BudgetMs).toBeGreaterThan(entry.p95BudgetMs); + } + }); }); diff --git a/backend/src/endpointSlaRegistry.ts b/backend/src/endpointSlaRegistry.ts index a0912d86..095465d2 100644 --- a/backend/src/endpointSlaRegistry.ts +++ b/backend/src/endpointSlaRegistry.ts @@ -16,6 +16,8 @@ export interface EndpointSlaAnnotation { type: EndpointType; /** P95 latency budget in milliseconds. */ p95BudgetMs: number; + /** P99 latency budget in milliseconds. */ + p99BudgetMs: number; /** Documented availability SLO (e.g. 0.999). */ availabilityTarget: AvailabilityTarget; tier: SlaTier; @@ -34,6 +36,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 50, + p99BudgetMs: 150, availabilityTarget: 0.999, tier: 'critical', ownerTeam: 'platform', @@ -44,6 +47,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 100, + p99BudgetMs: 250, availabilityTarget: 0.999, tier: 'critical', ownerTeam: 'platform', @@ -54,6 +58,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 200, + p99BudgetMs: 500, availabilityTarget: 0.99, tier: 'standard', ownerTeam: 'platform', @@ -64,6 +69,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 200, + p99BudgetMs: 500, availabilityTarget: 0.995, tier: 'critical', ownerTeam: 'backend', @@ -74,6 +80,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 200, + p99BudgetMs: 500, availabilityTarget: 0.995, tier: 'standard', ownerTeam: 'backend', @@ -84,6 +91,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 200, + p99BudgetMs: 500, availabilityTarget: 0.995, tier: 'standard', ownerTeam: 'backend', @@ -94,6 +102,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 200, + p99BudgetMs: 500, availabilityTarget: 0.995, tier: 'standard', ownerTeam: 'backend', @@ -104,6 +113,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['POST'], type: EndpointType.WRITE, p95BudgetMs: 500, + p99BudgetMs: 1200, availabilityTarget: 0.99, tier: 'critical', ownerTeam: 'backend', @@ -114,6 +124,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['POST'], type: EndpointType.WRITE, p95BudgetMs: 500, + p99BudgetMs: 1200, availabilityTarget: 0.99, tier: 'critical', ownerTeam: 'backend', @@ -124,6 +135,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['POST'], type: EndpointType.WRITE, p95BudgetMs: 500, + p99BudgetMs: 1200, availabilityTarget: 0.99, tier: 'standard', ownerTeam: 'platform', @@ -134,6 +146,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [ methods: ['GET'], type: EndpointType.READ, p95BudgetMs: 200, + p99BudgetMs: 500, availabilityTarget: 0.99, tier: 'standard', ownerTeam: 'platform', diff --git a/docs/api/README.md b/docs/api/README.md index 0666c1e1..7c2da398 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -129,6 +129,10 @@ windows, sunset policy, and client migration guide. API endpoints are rate limited. See [RATE_LIMITING.md](./RATE_LIMITING.md) for details. +### SLA and SLO Targets + +API reliability, availability/uptime, and latency targets are formally documented and validated. See [SLA_SLO.md](./SLA_SLO.md) for the exact targets and monitoring procedures. + ### Error Handling All errors follow a consistent format: diff --git a/docs/api/SLA_SLO.md b/docs/api/SLA_SLO.md new file mode 100644 index 00000000..75288f78 --- /dev/null +++ b/docs/api/SLA_SLO.md @@ -0,0 +1,65 @@ +# API SLA/SLO Targets + +This document defines the formal Service Level Agreements (SLA) and Service Level Objectives (SLO) for the `YieldVault-RWA` backend API, establishing target uptime/availability, read/write latencies (at P95 and P99 percentiles), error budgets, and alerting thresholds. + +--- + +## 1. Purpose & Scope + +To ensure a high-quality user experience and predictable performance for integrated applications, the YieldVault backend API is governed by strict reliability and speed targets. These objectives guide on-call engineers, developers, and operators in system maintenance, deployment gates, and incident responses. + +All API endpoints listed in the [Endpoint SLA Registry](file:///c:/Users/BUMBLECODE/Documents/Projects/YieldVault-RWA/backend/src/endpointSlaRegistry.ts) are subject to these targets. + +--- + +## 2. Service Level Objectives (SLO) & Service Level Indicators (SLI) + +### 2.1 Uptime / Availability SLO + +API uptime is evaluated as the proportion of successful HTTP requests (excluding client-side 4xx errors) over a rolling 30-day window. + +- **Uptime Target**: **99.9%** availability (Tier 2 baseline). +- **Max Permitted Downtime**: 43.8 minutes per month. +- **Evaluation Mechanism**: Inbound status health checks (`GET /health` and `GET /ready`) measured via external uptime monitors. + +### 2.2 Latency SLOs + +Latency targets are monitored using a rolling 5-minute window for all API requests. We measure latency at both the **95th percentile (P95)** and the **99th percentile (P99)** to bound worst-case tail performance. + +| Traffic Category | Endpoints Included | P95 Target | P99 Target | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | ------------ | ------------- | +| **Read Requests (GET)** | `/api/v1/vault/summary`, `/api/v1/vault/metrics`, `/api/v1/vault/apy`, `/api/v1/vault/:id`, `/health`, `/ready`, `/metrics` | **< 200 ms** | **< 500 ms** | +| **Write Requests (POST/PUT/DELETE)** | `/api/v1/vault/deposit`, `/api/v1/vault/withdraw`, `/admin/cache/invalidate`, `/admin/api-keys/register` | **< 500 ms** | **< 1200 ms** | + +--- + +## 3. Alerting and Error Budget Policy + +### 3.1 Monthly Error Budget + +The monthly error budget represents the allowed rate of service failure. For a 99.9% availability target, the error budget is **0.1%** of all API requests. + +### 3.2 Burn Rate Alert Triggers + +If a sudden spike in errors or latency consumes the error budget too quickly, alerts are dispatched to maintainers: + +- **Fast Burn (P0 / Critical)**: **2%** of the monthly budget consumed in **1 hour**. + - _Response_: High-priority pager notification via PagerDuty. Immediate deployment freeze. +- **Slow Burn (P1 / High)**: **5%** of the monthly budget consumed in **6 hours**. + - _Response_: Standard alert via Slack/PagerDuty. Ticket scheduled in current sprint. + +For details on alert configurations, refer to the [Latency Monitoring Guide](file:///c:/Users/BUMBLECODE/Documents/Projects/YieldVault-RWA/backend/LATENCY_MONITORING.md). + +--- + +## 4. Automated Verification + +The reliability targets specified here are stored as machine-readable configuration in [`docs/nfr-baselines.json`](file:///c:/Users/BUMBLECODE/Documents/Projects/YieldVault-RWA/docs/nfr-baselines.json) and are programmatically validated. + +Run the NFR baselines validator to verify compliance: + +```bash +npm run validate:nfr-baselines +``` + +This validation runs automatically as part of the continuous integration (CI) pipeline to guarantee documentation and configuration stay in sync. diff --git a/docs/nfr-baselines.json b/docs/nfr-baselines.json index cf10c0e2..ea4ac5ac 100644 --- a/docs/nfr-baselines.json +++ b/docs/nfr-baselines.json @@ -23,6 +23,8 @@ "availability_percent": 99.9, "latency_p95_read_ms": 200, "latency_p95_write_ms": 500, + "latency_p99_read_ms": 500, + "latency_p99_write_ms": 1200, "error_rate_max_percent": 0.1 }, "rto_minutes": 60, diff --git a/scripts/validate-nfr-baselines.test.ts b/scripts/validate-nfr-baselines.test.ts index f381a134..4acacef5 100644 --- a/scripts/validate-nfr-baselines.test.ts +++ b/scripts/validate-nfr-baselines.test.ts @@ -4,6 +4,7 @@ import { resolve } from 'node:path'; import { validateNFRJsonSpec, validateNFRDocContent, + validateApiSlaSloDocContent, validateObservabilityAlignment, runFullNFRValidation, } from './validate-nfr-baselines'; @@ -30,11 +31,21 @@ describe('NFR Baselines Validator Unit Tests', () => { expect(validateNFRJsonSpec('{ bad json').valid).toBe(false); }); - it('rejects NFR spec with out-of-bound availability or RTO values', () => { + it('rejects NFR spec with out-of-bound availability or latency values', () => { const json = JSON.stringify({ tiers: [ { tier: 't1', name: 'T1', slo: { availability_percent: 50.0 }, rto_minutes: 9999, rpo_minutes: 0 }, - { tier: 't2', name: 'T2', slo: { availability_percent: 99.9 }, rto_minutes: 60, rpo_minutes: 15 }, + { + tier: 't2', + name: 'T2', + slo: { + availability_percent: 99.9, + latency_p95_read_ms: 1200, + latency_p99_write_ms: -10, + }, + rto_minutes: 60, + rpo_minutes: 15, + }, { tier: 't3', name: 'T3', slo: { availability_percent: 99.9 }, rto_minutes: 15, rpo_minutes: 0 }, ], error_budget_policy: { fast_burn: {}, slow_burn: {} }, @@ -43,6 +54,8 @@ describe('NFR Baselines Validator Unit Tests', () => { expect(res.valid).toBe(false); expect(res.errors.some((e) => e.includes('invalid availability SLO'))).toBe(true); expect(res.errors.some((e) => e.includes('invalid RTO'))).toBe(true); + expect(res.errors.some((e) => e.includes('invalid latency_p95_read_ms'))).toBe(true); + expect(res.errors.some((e) => e.includes('invalid latency_p99_write_ms'))).toBe(true); }); }); @@ -59,6 +72,23 @@ describe('NFR Baselines Validator Unit Tests', () => { }); }); + describe('validateApiSlaSloDocContent', () => { + it('validates repository docs/api/SLA_SLO.md file', () => { + const docPath = resolve(__dirname, '../docs/api/SLA_SLO.md'); + expect(existsSync(docPath)).toBe(true); + const markdown = readFileSync(docPath, 'utf8'); + expect(validateApiSlaSloDocContent(markdown).valid).toBe(true); + }); + + it('rejects empty markdown content', () => { + expect(validateApiSlaSloDocContent('').valid).toBe(false); + }); + + it('rejects markdown content missing required headings', () => { + expect(validateApiSlaSloDocContent('# API SLA/SLO Targets').valid).toBe(false); + }); + }); + describe('validateObservabilityAlignment', () => { it('passes when observability docs align with NFR latency targets', () => { const res = validateObservabilityAlignment('200 ms and 500 ms', '200 ms and 500 ms'); diff --git a/scripts/validate-nfr-baselines.ts b/scripts/validate-nfr-baselines.ts index 95eb9054..87e9cd23 100644 --- a/scripts/validate-nfr-baselines.ts +++ b/scripts/validate-nfr-baselines.ts @@ -14,6 +14,8 @@ export interface NFRTierConfig { availability_percent: number; latency_p95_read_ms?: number; latency_p95_write_ms?: number; + latency_p99_read_ms?: number; + latency_p99_write_ms?: number; }; rto_minutes: number; rpo_minutes: number; @@ -54,6 +56,33 @@ export function validateNFRJsonSpec(jsonContent: string): ValidationResult { if (typeof t.rpo_minutes !== 'number' || t.rpo_minutes < 0 || t.rpo_minutes > 60) { errors.push(`Tier "${t.tier}" has invalid RPO: ${t.rpo_minutes} minutes (must be between 0 and 60 mins).`); } + + if (t.slo) { + if ('latency_p95_read_ms' in t.slo) { + const val = t.slo.latency_p95_read_ms; + if (typeof val !== 'number' || val <= 0 || val > 1000) { + errors.push(`Tier "${t.tier}" has invalid latency_p95_read_ms SLO: ${val} ms (must be a positive number up to 1000).`); + } + } + if ('latency_p95_write_ms' in t.slo) { + const val = t.slo.latency_p95_write_ms; + if (typeof val !== 'number' || val <= 0 || val > 3000) { + errors.push(`Tier "${t.tier}" has invalid latency_p95_write_ms SLO: ${val} ms (must be a positive number up to 3000).`); + } + } + if ('latency_p99_read_ms' in t.slo) { + const val = t.slo.latency_p99_read_ms; + if (typeof val !== 'number' || val <= 0 || val > 2000) { + errors.push(`Tier "${t.tier}" has invalid latency_p99_read_ms SLO: ${val} ms (must be a positive number up to 2000).`); + } + } + if ('latency_p99_write_ms' in t.slo) { + const val = t.slo.latency_p99_write_ms; + if (typeof val !== 'number' || val <= 0 || val > 5000) { + errors.push(`Tier "${t.tier}" has invalid latency_p99_write_ms SLO: ${val} ms (must be a positive number up to 5000).`); + } + } + } } if (!spec.error_budget_policy || !spec.error_budget_policy.fast_burn || !spec.error_budget_policy.slow_burn) { @@ -96,6 +125,37 @@ export function validateNFRDocContent(markdownContent: string): ValidationResult return { valid: errors.length === 0, errors, warnings }; } +/** + * Validates section completeness in docs/api/SLA_SLO.md. + */ +export function validateApiSlaSloDocContent(markdownContent: string): ValidationResult { + const errors: string[] = []; + const warnings: string[] = []; + + if (!markdownContent || markdownContent.trim() === '') { + errors.push('SLA_SLO.md content cannot be empty.'); + return { valid: false, errors, warnings }; + } + + const requiredHeadings = [ + 'API SLA/SLO Targets', + '1. Purpose & Scope', + '2. Service Level Objectives (SLO) & Service Level Indicators (SLI)', + '2.1 Uptime / Availability SLO', + '2.2 Latency SLOs', + '3. Alerting and Error Budget Policy', + '4. Automated Verification', + ]; + + for (const heading of requiredHeadings) { + if (!markdownContent.includes(heading)) { + errors.push(`SLA_SLO.md is missing required section: "${heading}"`); + } + } + + return { valid: errors.length === 0, errors, warnings }; +} + /** * Validates alignment between NFR baselines JSON and monitoring observability docs. */ @@ -150,6 +210,48 @@ export function runFullNFRValidation(rootDir: string = process.cwd()): Validatio } } + const apiSlaSloPath = resolve(rootDir, 'docs/api/SLA_SLO.md'); + if (!existsSync(apiSlaSloPath)) { + allErrors.push('docs/api/SLA_SLO.md file does not exist.'); + } else { + const apiSlaSloContent = readFileSync(apiSlaSloPath, 'utf8'); + const resApiDoc = validateApiSlaSloDocContent(apiSlaSloContent); + allErrors.push(...resApiDoc.errors); + allWarnings.push(...resApiDoc.warnings); + + if (jsonContent) { + try { + const spec = JSON.parse(jsonContent); + const tier2 = spec.tiers?.find((t: any) => t.tier === 'tier2_backend_api'); + if (tier2 && tier2.slo) { + const availability = tier2.slo.availability_percent; + const readP95 = tier2.slo.latency_p95_read_ms; + const writeP95 = tier2.slo.latency_p95_write_ms; + const readP99 = tier2.slo.latency_p99_read_ms; + const writeP99 = tier2.slo.latency_p99_write_ms; + + if (availability !== undefined && !apiSlaSloContent.includes(`${availability}%`)) { + allErrors.push(`docs/api/SLA_SLO.md does not document the correct availability target of ${availability}%.`); + } + if (readP95 !== undefined && !apiSlaSloContent.includes(`${readP95} ms`)) { + allErrors.push(`docs/api/SLA_SLO.md does not document the correct read P95 latency target of ${readP95} ms.`); + } + if (writeP95 !== undefined && !apiSlaSloContent.includes(`${writeP95} ms`)) { + allErrors.push(`docs/api/SLA_SLO.md does not document the correct write P95 latency target of ${writeP95} ms.`); + } + if (readP99 !== undefined && !apiSlaSloContent.includes(`${readP99} ms`)) { + allErrors.push(`docs/api/SLA_SLO.md does not document the correct read P99 latency target of ${readP99} ms.`); + } + if (writeP99 !== undefined && !apiSlaSloContent.includes(`${writeP99} ms`)) { + allErrors.push(`docs/api/SLA_SLO.md does not document the correct write P99 latency target of ${writeP99} ms.`); + } + } + } catch { + // Handled by validateNFRJsonSpec + } + } + } + return { valid: allErrors.length === 0, errors: allErrors, warnings: allWarnings }; }