Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions backend/src/__tests__/endpointSlaRegistry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,12 @@ describe('endpointSlaRegistry', () => {
const summary = getEndpointSla('/api/v1/vault/summary');
expect(deposit?.p95BudgetMs).toBeGreaterThan(summary!.p95BudgetMs);
});

it('includes p99BudgetMs annotations for all endpoints which are greater than p95BudgetMs', () => {
const registry = listEndpointSlaRegistry();
for (const entry of registry) {
expect(entry.p99BudgetMs).toBeDefined();
expect(entry.p99BudgetMs).toBeGreaterThan(entry.p95BudgetMs);
}
});
});
13 changes: 13 additions & 0 deletions backend/src/endpointSlaRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ export interface EndpointSlaAnnotation {
type: EndpointType;
/** P95 latency budget in milliseconds. */
p95BudgetMs: number;
/** P99 latency budget in milliseconds. */
p99BudgetMs: number;
/** Documented availability SLO (e.g. 0.999). */
availabilityTarget: AvailabilityTarget;
tier: SlaTier;
Expand All @@ -34,6 +36,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 50,
p99BudgetMs: 150,
availabilityTarget: 0.999,
tier: 'critical',
ownerTeam: 'platform',
Expand All @@ -44,6 +47,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 100,
p99BudgetMs: 250,
availabilityTarget: 0.999,
tier: 'critical',
ownerTeam: 'platform',
Expand All @@ -54,6 +58,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 200,
p99BudgetMs: 500,
availabilityTarget: 0.99,
tier: 'standard',
ownerTeam: 'platform',
Expand All @@ -64,6 +69,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 200,
p99BudgetMs: 500,
availabilityTarget: 0.995,
tier: 'critical',
ownerTeam: 'backend',
Expand All @@ -74,6 +80,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 200,
p99BudgetMs: 500,
availabilityTarget: 0.995,
tier: 'standard',
ownerTeam: 'backend',
Expand All @@ -84,6 +91,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 200,
p99BudgetMs: 500,
availabilityTarget: 0.995,
tier: 'standard',
ownerTeam: 'backend',
Expand All @@ -94,6 +102,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 200,
p99BudgetMs: 500,
availabilityTarget: 0.995,
tier: 'standard',
ownerTeam: 'backend',
Expand All @@ -104,6 +113,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['POST'],
type: EndpointType.WRITE,
p95BudgetMs: 500,
p99BudgetMs: 1200,
availabilityTarget: 0.99,
tier: 'critical',
ownerTeam: 'backend',
Expand All @@ -114,6 +124,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['POST'],
type: EndpointType.WRITE,
p95BudgetMs: 500,
p99BudgetMs: 1200,
availabilityTarget: 0.99,
tier: 'critical',
ownerTeam: 'backend',
Expand All @@ -124,6 +135,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['POST'],
type: EndpointType.WRITE,
p95BudgetMs: 500,
p99BudgetMs: 1200,
availabilityTarget: 0.99,
tier: 'standard',
ownerTeam: 'platform',
Expand All @@ -134,6 +146,7 @@ export const ENDPOINT_SLA_REGISTRY: readonly EndpointSlaAnnotation[] = [
methods: ['GET'],
type: EndpointType.READ,
p95BudgetMs: 200,
p99BudgetMs: 500,
availabilityTarget: 0.99,
tier: 'standard',
ownerTeam: 'platform',
Expand Down
4 changes: 4 additions & 0 deletions docs/api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,10 @@ windows, sunset policy, and client migration guide.

API endpoints are rate limited. See [RATE_LIMITING.md](./RATE_LIMITING.md) for details.

### SLA and SLO Targets

API reliability, availability/uptime, and latency targets are formally documented and validated. See [SLA_SLO.md](./SLA_SLO.md) for the exact targets and monitoring procedures.

### Error Handling

All errors follow a consistent format:
Expand Down
65 changes: 65 additions & 0 deletions docs/api/SLA_SLO.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# API SLA/SLO Targets

This document defines the formal Service Level Agreements (SLA) and Service Level Objectives (SLO) for the `YieldVault-RWA` backend API, establishing target uptime/availability, read/write latencies (at P95 and P99 percentiles), error budgets, and alerting thresholds.

---

## 1. Purpose & Scope

To ensure a high-quality user experience and predictable performance for integrated applications, the YieldVault backend API is governed by strict reliability and speed targets. These objectives guide on-call engineers, developers, and operators in system maintenance, deployment gates, and incident responses.

All API endpoints listed in the [Endpoint SLA Registry](file:///c:/Users/BUMBLECODE/Documents/Projects/YieldVault-RWA/backend/src/endpointSlaRegistry.ts) are subject to these targets.

---

## 2. Service Level Objectives (SLO) & Service Level Indicators (SLI)

### 2.1 Uptime / Availability SLO

API uptime is evaluated as the proportion of successful HTTP requests (excluding client-side 4xx errors) over a rolling 30-day window.

- **Uptime Target**: **99.9%** availability (Tier 2 baseline).
- **Max Permitted Downtime**: 43.8 minutes per month.
- **Evaluation Mechanism**: Inbound status health checks (`GET /health` and `GET /ready`) measured via external uptime monitors.

### 2.2 Latency SLOs

Latency targets are monitored using a rolling 5-minute window for all API requests. We measure latency at both the **95th percentile (P95)** and the **99th percentile (P99)** to bound worst-case tail performance.

| Traffic Category | Endpoints Included | P95 Target | P99 Target |
| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | ------------ | ------------- |
| **Read Requests (GET)** | `/api/v1/vault/summary`, `/api/v1/vault/metrics`, `/api/v1/vault/apy`, `/api/v1/vault/:id`, `/health`, `/ready`, `/metrics` | **< 200 ms** | **< 500 ms** |
| **Write Requests (POST/PUT/DELETE)** | `/api/v1/vault/deposit`, `/api/v1/vault/withdraw`, `/admin/cache/invalidate`, `/admin/api-keys/register` | **< 500 ms** | **< 1200 ms** |

---

## 3. Alerting and Error Budget Policy

### 3.1 Monthly Error Budget

The monthly error budget represents the allowed rate of service failure. For a 99.9% availability target, the error budget is **0.1%** of all API requests.

### 3.2 Burn Rate Alert Triggers

If a sudden spike in errors or latency consumes the error budget too quickly, alerts are dispatched to maintainers:

- **Fast Burn (P0 / Critical)**: **2%** of the monthly budget consumed in **1 hour**.
- _Response_: High-priority pager notification via PagerDuty. Immediate deployment freeze.
- **Slow Burn (P1 / High)**: **5%** of the monthly budget consumed in **6 hours**.
- _Response_: Standard alert via Slack/PagerDuty. Ticket scheduled in current sprint.

For details on alert configurations, refer to the [Latency Monitoring Guide](file:///c:/Users/BUMBLECODE/Documents/Projects/YieldVault-RWA/backend/LATENCY_MONITORING.md).

---

## 4. Automated Verification

The reliability targets specified here are stored as machine-readable configuration in [`docs/nfr-baselines.json`](file:///c:/Users/BUMBLECODE/Documents/Projects/YieldVault-RWA/docs/nfr-baselines.json) and are programmatically validated.

Run the NFR baselines validator to verify compliance:

```bash
npm run validate:nfr-baselines
```

This validation runs automatically as part of the continuous integration (CI) pipeline to guarantee documentation and configuration stay in sync.
2 changes: 2 additions & 0 deletions docs/nfr-baselines.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
"availability_percent": 99.9,
"latency_p95_read_ms": 200,
"latency_p95_write_ms": 500,
"latency_p99_read_ms": 500,
"latency_p99_write_ms": 1200,
"error_rate_max_percent": 0.1
},
"rto_minutes": 60,
Expand Down
34 changes: 32 additions & 2 deletions scripts/validate-nfr-baselines.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { resolve } from 'node:path';
import {
validateNFRJsonSpec,
validateNFRDocContent,
validateApiSlaSloDocContent,
validateObservabilityAlignment,
runFullNFRValidation,
} from './validate-nfr-baselines';
Expand All @@ -30,11 +31,21 @@ describe('NFR Baselines Validator Unit Tests', () => {
expect(validateNFRJsonSpec('{ bad json').valid).toBe(false);
});

it('rejects NFR spec with out-of-bound availability or RTO values', () => {
it('rejects NFR spec with out-of-bound availability or latency values', () => {
const json = JSON.stringify({
tiers: [
{ tier: 't1', name: 'T1', slo: { availability_percent: 50.0 }, rto_minutes: 9999, rpo_minutes: 0 },
{ tier: 't2', name: 'T2', slo: { availability_percent: 99.9 }, rto_minutes: 60, rpo_minutes: 15 },
{
tier: 't2',
name: 'T2',
slo: {
availability_percent: 99.9,
latency_p95_read_ms: 1200,
latency_p99_write_ms: -10,
},
rto_minutes: 60,
rpo_minutes: 15,
},
{ tier: 't3', name: 'T3', slo: { availability_percent: 99.9 }, rto_minutes: 15, rpo_minutes: 0 },
],
error_budget_policy: { fast_burn: {}, slow_burn: {} },
Expand All @@ -43,6 +54,8 @@ describe('NFR Baselines Validator Unit Tests', () => {
expect(res.valid).toBe(false);
expect(res.errors.some((e) => e.includes('invalid availability SLO'))).toBe(true);
expect(res.errors.some((e) => e.includes('invalid RTO'))).toBe(true);
expect(res.errors.some((e) => e.includes('invalid latency_p95_read_ms'))).toBe(true);
expect(res.errors.some((e) => e.includes('invalid latency_p99_write_ms'))).toBe(true);
});
});

Expand All @@ -59,6 +72,23 @@ describe('NFR Baselines Validator Unit Tests', () => {
});
});

describe('validateApiSlaSloDocContent', () => {
it('validates repository docs/api/SLA_SLO.md file', () => {
const docPath = resolve(__dirname, '../docs/api/SLA_SLO.md');
expect(existsSync(docPath)).toBe(true);
const markdown = readFileSync(docPath, 'utf8');
expect(validateApiSlaSloDocContent(markdown).valid).toBe(true);
});

it('rejects empty markdown content', () => {
expect(validateApiSlaSloDocContent('').valid).toBe(false);
});

it('rejects markdown content missing required headings', () => {
expect(validateApiSlaSloDocContent('# API SLA/SLO Targets').valid).toBe(false);
});
});

describe('validateObservabilityAlignment', () => {
it('passes when observability docs align with NFR latency targets', () => {
const res = validateObservabilityAlignment('200 ms and 500 ms', '200 ms and 500 ms');
Expand Down
102 changes: 102 additions & 0 deletions scripts/validate-nfr-baselines.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ export interface NFRTierConfig {
availability_percent: number;
latency_p95_read_ms?: number;
latency_p95_write_ms?: number;
latency_p99_read_ms?: number;
latency_p99_write_ms?: number;
};
rto_minutes: number;
rpo_minutes: number;
Expand Down Expand Up @@ -54,6 +56,33 @@ export function validateNFRJsonSpec(jsonContent: string): ValidationResult {
if (typeof t.rpo_minutes !== 'number' || t.rpo_minutes < 0 || t.rpo_minutes > 60) {
errors.push(`Tier "${t.tier}" has invalid RPO: ${t.rpo_minutes} minutes (must be between 0 and 60 mins).`);
}

if (t.slo) {
if ('latency_p95_read_ms' in t.slo) {
const val = t.slo.latency_p95_read_ms;
if (typeof val !== 'number' || val <= 0 || val > 1000) {
errors.push(`Tier "${t.tier}" has invalid latency_p95_read_ms SLO: ${val} ms (must be a positive number up to 1000).`);
}
}
if ('latency_p95_write_ms' in t.slo) {
const val = t.slo.latency_p95_write_ms;
if (typeof val !== 'number' || val <= 0 || val > 3000) {
errors.push(`Tier "${t.tier}" has invalid latency_p95_write_ms SLO: ${val} ms (must be a positive number up to 3000).`);
}
}
if ('latency_p99_read_ms' in t.slo) {
const val = t.slo.latency_p99_read_ms;
if (typeof val !== 'number' || val <= 0 || val > 2000) {
errors.push(`Tier "${t.tier}" has invalid latency_p99_read_ms SLO: ${val} ms (must be a positive number up to 2000).`);
}
}
if ('latency_p99_write_ms' in t.slo) {
const val = t.slo.latency_p99_write_ms;
if (typeof val !== 'number' || val <= 0 || val > 5000) {
errors.push(`Tier "${t.tier}" has invalid latency_p99_write_ms SLO: ${val} ms (must be a positive number up to 5000).`);
}
}
}
}

if (!spec.error_budget_policy || !spec.error_budget_policy.fast_burn || !spec.error_budget_policy.slow_burn) {
Expand Down Expand Up @@ -96,6 +125,37 @@ export function validateNFRDocContent(markdownContent: string): ValidationResult
return { valid: errors.length === 0, errors, warnings };
}

/**
* Validates section completeness in docs/api/SLA_SLO.md.
*/
export function validateApiSlaSloDocContent(markdownContent: string): ValidationResult {
const errors: string[] = [];
const warnings: string[] = [];

if (!markdownContent || markdownContent.trim() === '') {
errors.push('SLA_SLO.md content cannot be empty.');
return { valid: false, errors, warnings };
}

const requiredHeadings = [
'API SLA/SLO Targets',
'1. Purpose & Scope',
'2. Service Level Objectives (SLO) & Service Level Indicators (SLI)',
'2.1 Uptime / Availability SLO',
'2.2 Latency SLOs',
'3. Alerting and Error Budget Policy',
'4. Automated Verification',
];

for (const heading of requiredHeadings) {
if (!markdownContent.includes(heading)) {
errors.push(`SLA_SLO.md is missing required section: "${heading}"`);
}
}

return { valid: errors.length === 0, errors, warnings };
}

/**
* Validates alignment between NFR baselines JSON and monitoring observability docs.
*/
Expand Down Expand Up @@ -150,6 +210,48 @@ export function runFullNFRValidation(rootDir: string = process.cwd()): Validatio
}
}

const apiSlaSloPath = resolve(rootDir, 'docs/api/SLA_SLO.md');
if (!existsSync(apiSlaSloPath)) {
allErrors.push('docs/api/SLA_SLO.md file does not exist.');
} else {
const apiSlaSloContent = readFileSync(apiSlaSloPath, 'utf8');
const resApiDoc = validateApiSlaSloDocContent(apiSlaSloContent);
allErrors.push(...resApiDoc.errors);
allWarnings.push(...resApiDoc.warnings);

if (jsonContent) {
try {
const spec = JSON.parse(jsonContent);
const tier2 = spec.tiers?.find((t: any) => t.tier === 'tier2_backend_api');
if (tier2 && tier2.slo) {
const availability = tier2.slo.availability_percent;
const readP95 = tier2.slo.latency_p95_read_ms;
const writeP95 = tier2.slo.latency_p95_write_ms;
const readP99 = tier2.slo.latency_p99_read_ms;
const writeP99 = tier2.slo.latency_p99_write_ms;

if (availability !== undefined && !apiSlaSloContent.includes(`${availability}%`)) {
allErrors.push(`docs/api/SLA_SLO.md does not document the correct availability target of ${availability}%.`);
}
if (readP95 !== undefined && !apiSlaSloContent.includes(`${readP95} ms`)) {
allErrors.push(`docs/api/SLA_SLO.md does not document the correct read P95 latency target of ${readP95} ms.`);
}
if (writeP95 !== undefined && !apiSlaSloContent.includes(`${writeP95} ms`)) {
allErrors.push(`docs/api/SLA_SLO.md does not document the correct write P95 latency target of ${writeP95} ms.`);
}
if (readP99 !== undefined && !apiSlaSloContent.includes(`${readP99} ms`)) {
allErrors.push(`docs/api/SLA_SLO.md does not document the correct read P99 latency target of ${readP99} ms.`);
}
if (writeP99 !== undefined && !apiSlaSloContent.includes(`${writeP99} ms`)) {
allErrors.push(`docs/api/SLA_SLO.md does not document the correct write P99 latency target of ${writeP99} ms.`);
}
}
} catch {
// Handled by validateNFRJsonSpec
}
}
}

return { valid: allErrors.length === 0, errors: allErrors, warnings: allWarnings };
}

Expand Down
Loading