diff --git a/.env.example b/.env.example index 955379a..fe88adc 100644 --- a/.env.example +++ b/.env.example @@ -31,7 +31,11 @@ INVENTORY_SERVICE_BASE_URL=http://127.0.0.1:5105 # make run sets this; with it unset, callers use the fixed URLs above (as in unit tests and Azure). # REGISTRY_URL=http://127.0.0.1:8500 -# --- Demo switches (patterns 4, 10, 12) --- +# --- Event bus (patterns 7 and 8) --- +# make run starts redis-server on 6390 and sets this. Unset: events wait in the order outbox. +# REDIS_URL=redis://127.0.0.1:6390/0 + +# --- Demo switches (patterns 4, 7, 10, 12) --- # make run sets CHAOS_ENABLED=1 so the pages can stop a replica and break payment. Keep it off anywhere public. # CHAOS_ENABLED=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2339c22..99da61e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,7 @@ jobs: requirements: services/inventory/requirements.txt tests: services/inventory/tests shared: flask + contracts: true steps: - uses: actions/checkout@v4 @@ -101,6 +102,50 @@ jobs: - name: Type-check and build run: npm run build + containers: + # Pattern 9. Builds every image exactly as Azure will run it (linux/amd64), starts it, and checks + # it answers /health and does not run as root. Nothing is pushed anywhere. + name: Container image (${{ matrix.image }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { image: user, dir: services/user, port: 5101, shared: true } + - { image: catalog, dir: services/catalog, port: 5102, shared: true } + - { image: order, dir: services/order, port: 5103, shared: true } + - { image: payment, dir: services/payment, port: 5104, shared: true } + - { image: inventory, dir: services/inventory, port: 5105, shared: true } + - { image: gateway, dir: gateway, port: 8100, shared: true } + - { image: registry, dir: registry, port: 8500, shared: false } + steps: + - uses: actions/checkout@v4 + + - name: Stage shared package (shop_common) + if: matrix.shared + run: cp -r shared "${{ matrix.dir }}/_shared" + + - name: Build for linux/amd64 + run: > + docker build --platform linux/amd64 --build-arg GIT_SHA=${{ github.sha }} + -t ms12p-${{ matrix.image }}:${{ github.sha }} ${{ matrix.dir }} + + - name: Run it, check health and that it is not root + run: | + docker run -d --name svc -p ${{ matrix.port }}:${{ matrix.port }} -e RUN_MODE=local \ + ms12p-${{ matrix.image }}:${{ github.sha }} + for i in $(seq 1 30); do curl -sf http://127.0.0.1:${{ matrix.port }}/health && break; sleep 1; done + curl -sf http://127.0.0.1:${{ matrix.port }}/health + uid=$(docker exec svc id -u) + echo "runs as uid $uid" + test "$uid" != "0" + size=$(docker image inspect ms12p-${{ matrix.image }}:${{ github.sha }} --format '{{.Size}}') + echo "| ${{ matrix.image }} | $((size / 1024 / 1024)) MB | uid $uid | linux/amd64 |" >> "$GITHUB_STEP_SUMMARY" + + - name: Container logs (on failure) + if: failure() + run: docker logs svc || true + bicep: name: Bicep (build) runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index 1957bc2..9d7d8f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ history, monolith dropped). The full agreed plan is in [`docs/PLAN.md`](docs/PLA - **Local + Azure.** `make provision` / `make teardown` / `make verify`, $25/mo budget alert. Nothing is deployed to Azure before milestone M3, and only with the owner's go-ahead. - **Phased delivery**: M0 bootstrap, then M1 (patterns 1, 2, 5, 6), M2 (10, 12, 4, 11), - M3 (7, 8, 9, 3 + Azure). One PR per milestone. + M3a (8, 7, 9, local) and M3b (3 + Azure, owner go-ahead). One PR per milestone. - Screenshots live in `docs/screenshots/` and are referenced by relative path. Never use GitHub `user-attachments` links (they went dead in the parent project). @@ -27,7 +27,7 @@ history, monolith dropped). The full agreed plan is in [`docs/PLAN.md`](docs/PLA - Name prefix: `ms12p` (parent uses `flaskms`). Teardown deletes the whole resource group, so a shared name would delete the other project. -## Current state (M2: patterns 1, 2, 4, 5, 6, 10, 11, 12 live) +## Current state (M3a: every pattern except 3 is live) - Folders: `services/{user,catalog,order,payment,inventory}/` (Flask, one DB each), `gateway/` (FastAPI), `registry/` (FastAPI), `shared/` (`shop_common`), `contracts/`, `frontend/`, `infra/`, `scripts/`, `tests/`. @@ -35,7 +35,19 @@ history, monolith dropped). The full agreed plan is in [`docs/PLAN.md`](docs/PLA `REPLICA_PORTS` (order also on 5113) and `CONTRACT_SUITES`. `setup.py`, `run_local.py`, `provision.py` and the smoke tests all read it. Add a service there. - Ports: user 5101, catalog 5102, order 5103 + 5113, payment 5104, inventory 5105, gateway 8100, - registry 8500, frontend 5273. + registry 8500, Redis 6390, frontend 5273. +- Saga (pattern 8): `services/order/application/saga.py`, orchestrated: price, reserve_stock, + charge_payment, confirm; compensation releases the hold (or emits StockReleaseRequested). Steps in + `saga_step`; holds keyed by saga id in inventory's `reservation`. `{"simulate_decline": true}` on + checkout makes payment answer DECLINED (200, not a breaker failure). +- Events (pattern 7): `shop_common/events.py` (Redis Stream `shop.events`, consumer groups, at least + once). Order writes `outbox_event` in the same transaction; `outbox.py` relays every 1 s; inventory's + `consumer.py` commits the hold on OrderConfirmed and dedupes via `consumed_event`. `POST /debug/bus` + (chaos) disconnects a replica's relay. Without `REDIS_URL` events just wait in the outbox. +- Background threads (heartbeat, relay, consumer) start only where `is_serving_process(app)` is true: + Flask's debug reloader runs a watcher plus a worker, and both build the app. +- Containers (pattern 9): multi-stage Dockerfiles, uid 10001, `.dockerignore` per service, `GIT_SHA` + build arg, `/debug/runtime`. CI job "Container image" builds, runs and checks all 7 images. - **`shop_common`** (`shared/`): every service calls `shop_common.flask_support.install(app, name, db)` inside its app context. It adds `/health`, `/health/db`, `/debug/spans`, `/metrics`, `POST /debug/crash` (only with `CHAOS_ENABLED=1`), the `X-Instance-Id` header, tracing, JSON logs, @@ -52,17 +64,23 @@ history, monolith dropped). The full agreed plan is in [`docs/PLAN.md`](docs/PLA - Circuit breaker (pattern 4): `services/order/application/order_api/api/clients.py`, pybreaker (3 failures, 15 s) + tenacity (1 retry, 1 s timeout), only 5xx/timeouts count. Checkout prices from catalog, charges payment, keeps the cart on 503. Payment has a `healthy|failing|slow` switch. -- Contracts (pattern 11): order's tests write `contracts/pacts/order-{payment,catalog}.json`; payment - and catalog verify them. CI fails if the regenerated files differ. Pact (pact-python 4.2) is +- Contracts (pattern 11): order's tests write `contracts/pacts/order-{payment,catalog,inventory}.json`; + those 3 services verify them. CI fails if the regenerated files differ. Pact (pact-python 4.2) is test-only, installed from `contracts/requirements.txt`, never in images. -- `make run` sets `REGISTRY_URL`, `CHAOS_ENABLED=1`, `INSTANCE_ID`, and restarts a stopped service - after 20 s. Registry TTL 10 s, heartbeat 2 s. Azure uses Container Apps DNS (no registry yet). +- `make run` sets `REGISTRY_URL`, `REDIS_URL`, `CHAOS_ENABLED=1`, `INSTANCE_ID`, and restarts a stopped + service after 20 s. Inventory seeds stock for all 24 catalog products; `POST /api/inventory//restock` + (chaos only) keeps smoke runs from running out. Registry TTL 10 s, heartbeat 2 s. Azure uses Container Apps DNS (no registry yet). - Gotchas solved: FastAPI tracing middleware must be added AFTER the request-ID middleware; Flask- - SQLAlchemy's engine is instrumented per engine; Pact's `Verifier` needs `host="127.0.0.1"`; DB writes - inside a Pact state handler deadlock on SQLite (seed before verifying instead). + SQLAlchemy's engine is instrumented per engine; Pact's `Verifier` needs `host="127.0.0.1"`; Pact state + handlers run on another thread, so end the test's DB transaction first or SQLite deadlocks; two + replicas racing `create_all()` (use `shop_common.db_setup.create_tables`). - Smoke tests create users and products in the local SQLite DBs; delete `services/*/instance/` for a clean catalog (this also deletes the local signing key, which logs everyone out). +## How to build (owner's preference) +Follow the Matt Pocock `implement` skill (read it from the mattpocock-skills plugin folder): TDD at +agreed seams, red test first; then `mattpocock-skills:code-review`; then commit. + ## Stack Python 3.12 (Flask 3.1 services, FastAPI gateway), Node 18+ (React 18, TypeScript 5, Vite 5), SQLite locally, Azure MySQL Flexible Server (one server, one DB per service) in Azure. diff --git a/README.md b/README.md index 62dd4cc..4fce399 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ A small Contoso e-commerce system that shows **12 microservices patterns working Each pattern gets its own page with an explanation, when *not* to use it, and a live demo you can click. -> **Status: M2, resilience and independence.** 8 of 12 patterns are live: M1's Database per -> Service, Bounded Context, API Gateway and Observability, plus M2's Circuit Breaker, Stateless -> Services, Contract Testing and Service Discovery. The foundation is copied from +> **Status: M3a, async and packaging.** 11 of 12 patterns are live. M3a adds the Saga, +> Event-Driven Architecture and Containerization to M1 (patterns 1, 2, 5, 6) and M2 (4, 10, 11, 12). +> The foundation is copied from > [Azure-Ecommerce-Monolith-Microservices-Migration](https://github.com/MAOFILHO/Azure-Ecommerce-Monolith-Microservices-Migration). -> The last 4 arrive in milestone M3. See [docs/PLAN.md](docs/PLAN.md). +> CI-CD and the Azure run arrive in M3b. See [docs/PLAN.md](docs/PLAN.md). ## The 12 patterns @@ -16,13 +16,13 @@ can click. |---|---|---|---| | 1 | Database per Service | Gives each service its own database | **Live (M1)** | | 2 | Bounded Context | Splits one large domain into clear business domains | **Live (M1)** | -| 3 | CI-CD Pipeline | Builds, tests and deploys every service automatically | M3 | +| 3 | CI-CD Pipeline | Builds, tests and deploys every service automatically | M3b | | 4 | Circuit Breaker | Stops calls to a failing service to prevent more failures | **Live (M2)** | | 5 | API Gateway | Routes client requests to the right service | **Live (M1)** | | 6 | Observability | Collects logs, metrics and traces from every service | **Live (M1)** | -| 7 | Event-Driven Architecture | Services communicate by publishing and consuming events | M3 | -| 8 | Saga Pattern | Coordinates distributed transactions and rolls back failures safely | M3 | -| 9 | Containerization | Packages each service with all its dependencies | M3 | +| 7 | Event-Driven Architecture | Services communicate by publishing and consuming events | **Live (M3a)** | +| 8 | Saga Pattern | Coordinates distributed transactions and rolls back failures safely | **Live (M3a)** | +| 9 | Containerization | Packages each service with all its dependencies | **Live (M3a)** | | 10 | Stateless Services | Stores session data outside the service | **Live (M2)** | | 11 | Contract Testing | Checks that APIs stay compatible across services | **Live (M2)** | | 12 | Service Discovery | Finds service instances automatically at runtime | **Live (M2)** | @@ -32,11 +32,12 @@ can click. ``` Browser → API gateway ──┬──► user → user_db (signs login tokens) (checks tokens, ├──► catalog → catalog_db - request IDs, ├──► order ×2 → order_db ──► catalog (prices) - tracing, metrics, │ └─► payment, behind a circuit breaker + request IDs, ├──► order ×2 → order_db checkout saga: price → hold stock → + tracing, metrics, │ charge (circuit breaker) → confirm aggregation, ├──► payment → payment_db failover) └──► inventory → inventory_db +order_db outbox ──relay──► Redis Stream "shop.events" ──► inventory (sells the held stock) Every service heartbeats into the registry (:8500); callers look each other up there. ``` @@ -47,11 +48,13 @@ Every service heartbeats into the registry (:8500); callers look each other up t - **`shop_common`** (`shared/`): OpenTelemetry tracing, JSON logs with trace IDs, Prometheus metrics, the `/health/db`, `/debug/spans`, `/metrics` endpoints, token checks and the discovery client. - **Service registry** (`registry/`): about 90 lines of FastAPI. Local only; Azure uses Container Apps DNS. -- **Contracts** (`contracts/pacts/`): the Pact files the order service needs payment and catalog to honour. +- **Contracts** (`contracts/pacts/`): the Pact files the order service needs payment, catalog and inventory to honour. +- **Redis** (Homebrew `redis-server`, in memory, port 6390): the event bus. `make run` starts and stops it. +- **Dockerfiles**: one per service plus gateway and registry, multi-stage, non-root, built only in + the cloud or CI for `linux/amd64`. - **React + TypeScript** frontend (`frontend/`) with a page per pattern. -Still to come (M3): Redis Streams events with an outbox, checkout saga, hardened containers, -deploy pipeline, and a real Azure run. +Still to come (M3b): path-filtered CI with a manual Deploy workflow (OIDC), and a real Azure run. ## The live demos @@ -105,11 +108,30 @@ The live registry. Stop an instance: its heartbeat stops, the registry expires i Service Discovery demo: registry table and an expired then re-registered order replica +### 7. Event-Driven Architecture +Disconnect the bus and buy a mug: OrderConfirmed waits in the order service's outbox, and the mug +stays held in inventory. Reconnect: the event goes out and inventory turns the hold into a sale. + +Event-Driven demo: bus disconnected, one OrderConfirmed waiting in the outbox, one mug held + +### 8. Saga +Buy 2 mugs with "make the bank decline the card" ticked. The saga prices the cart, holds the stock, +the charge fails, and it releases the stock. Stock ends exactly where it started; the cart is kept. + +Saga demo: a declined card, with the stock hold released by the compensation step + +### 9. Containerization +How each process runs (uid, container or not, platform, build). Locally they are plain processes; +CI builds and runs every image for linux/amd64 and fails any that runs as root. + +Containerization demo: every process, its uid and platform, and the Dockerfile shape + ## Run locally (no Docker) ```bash +brew install redis # once: the event bus (patterns 7 and 8) make setup # one venv per service + gateway, shop_common, frontend packages -make run # frontend http://127.0.0.1:5273, gateway http://127.0.0.1:8100, registry :8500 +make run # frontend :5273, gateway :8100, registry :8500, Redis :6390 make test # unit tests, each suite in its own venv make contracts # Pact: order writes its contracts, payment and catalog verify them make smoke # end-to-end checks against the running stack @@ -122,5 +144,5 @@ collector. Without it, traces stay in each process's memory and the Observabilit Azure Container Apps (one app per service plus the gateway, all scale-to-zero) + MySQL Flexible Server, provisioned with `make provision` and removed with `make teardown`. About $0.21 for an -8-hour session. See [cost_estimate.md](cost_estimate.md). Nothing from M1 or M2 has been deployed +8-hour session. See [cost_estimate.md](cost_estimate.md). Nothing from M1, M2 or M3a has been deployed yet. In Azure, discovery uses Container Apps DNS (no registry), and the demo switches stay off. diff --git a/contracts/pacts/order-inventory.json b/contracts/pacts/order-inventory.json new file mode 100644 index 0000000..149f17d --- /dev/null +++ b/contracts/pacts/order-inventory.json @@ -0,0 +1,199 @@ +{ + "consumer": { + "name": "order" + }, + "interactions": [ + { + "description": "a release of a stock hold", + "pending": false, + "providerStates": [ + { + "name": "a stock hold exists", + "params": { + "reservation_id": "3f0c1d2e-0000-4000-8000-000000000001" + } + } + ], + "request": { + "method": "POST", + "path": "/api/inventory/reservations/3f0c1d2e-0000-4000-8000-000000000001/release" + }, + "response": { + "body": { + "content": { + "result": { + "reservation_id": "3f0c1d2e-0000-4000-8000-000000000001", + "status": "RELEASED" + } + }, + "contentType": "application/json", + "encoded": false + }, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "status": 200 + }, + "type": "Synchronous/HTTP" + }, + { + "description": "a stock hold for a checkout", + "pending": false, + "providerStates": [ + { + "name": "product 1 has stock" + } + ], + "request": { + "body": { + "content": { + "items": [ + { + "product_id": 1, + "quantity": 2 + } + ], + "order_id": 42, + "reservation_id": "3f0c1d2e-0000-4000-8000-000000000001" + }, + "contentType": "application/json", + "encoded": false + }, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "method": "POST", + "path": "/api/inventory/reservations" + }, + "response": { + "body": { + "content": { + "result": { + "reservation_id": "3f0c1d2e-0000-4000-8000-000000000001", + "status": "RESERVED" + } + }, + "contentType": "application/json", + "encoded": false + }, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "matchingRules": { + "body": { + "$.result.reservation_id": { + "combine": "AND", + "matchers": [ + { + "match": "type" + } + ] + } + } + }, + "status": 200 + }, + "type": "Synchronous/HTTP" + }, + { + "description": "a stock hold that is more than inventory has", + "pending": false, + "providerStates": [ + { + "name": "product 1 has stock" + } + ], + "request": { + "body": { + "content": { + "items": [ + { + "product_id": 1, + "quantity": 100000 + } + ], + "order_id": 43, + "reservation_id": "3f0c1d2e-0000-4000-8000-000000000002" + }, + "contentType": "application/json", + "encoded": false + }, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "method": "POST", + "path": "/api/inventory/reservations" + }, + "response": { + "body": { + "content": { + "short": [ + { + "available": 25, + "product_id": 1 + } + ] + }, + "contentType": "application/json", + "encoded": false + }, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "matchingRules": { + "body": { + "$.short": { + "combine": "AND", + "matchers": [ + { + "match": "type", + "min": 1 + } + ] + }, + "$.short[*].available": { + "combine": "AND", + "matchers": [ + { + "match": "integer" + } + ] + }, + "$.short[*].product_id": { + "combine": "AND", + "matchers": [ + { + "match": "integer" + } + ] + } + } + }, + "status": 409 + }, + "type": "Synchronous/HTTP" + } + ], + "metadata": { + "pactRust": { + "ffi": "0.5.12", + "models": "1.3.16" + }, + "pactSpecification": { + "version": "4.0" + } + }, + "provider": { + "name": "inventory" + } +} \ No newline at end of file diff --git a/contracts/pacts/order-payment.json b/contracts/pacts/order-payment.json index 7eb390d..64303f3 100644 --- a/contracts/pacts/order-payment.json +++ b/contracts/pacts/order-payment.json @@ -98,6 +98,84 @@ }, "type": "Synchronous/HTTP" }, + { + "description": "a charge the bank declines", + "pending": false, + "providerStates": [ + { + "name": "payment is healthy" + } + ], + "request": { + "body": { + "content": { + "amount_cents": 899, + "order_id": 43, + "simulate": "decline" + }, + "contentType": "application/json", + "encoded": false + }, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "matchingRules": { + "body": { + "$.amount_cents": { + "combine": "AND", + "matchers": [ + { + "match": "integer" + } + ] + }, + "$.order_id": { + "combine": "AND", + "matchers": [ + { + "match": "integer" + } + ] + } + } + }, + "method": "POST", + "path": "/api/payment/charge" + }, + "response": { + "body": { + "content": { + "result": { + "id": 2, + "status": "DECLINED" + } + }, + "contentType": "application/json", + "encoded": false + }, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "matchingRules": { + "body": { + "$.result.id": { + "combine": "AND", + "matchers": [ + { + "match": "integer" + } + ] + } + } + }, + "status": 200 + }, + "type": "Synchronous/HTTP" + }, { "description": "a health check through the circuit breaker", "pending": false, diff --git a/docs/PLAN.md b/docs/PLAN.md index 1fe585b..bb6c93a 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -129,3 +129,42 @@ Each milestone ends with: tests green, a click-through of its 4 pattern pages, s - Manual click-through of all 12 pattern pages in the browser pane, with a trace visible in `otel-desktop-viewer` for one order. - CI: a commit touching only `services/order/` runs only that matrix leg; the documented contract-break branch goes red. - M3 only: `make provision`, repeat the smoke flow against Azure, `make teardown`, `make verify` returns PASS. + +## M3 review (2026-11-15, after M1 and M2 shipped) + +What changed since this plan was written: checkout already prices from catalog and charges payment +behind the circuit breaker (M2), checkout does not reserve stock yet, the registry is local only, +the login signing key has no Azure home yet, and Redis is not installed locally yet. + +Decisions: +1. **Saga style: orchestrated.** The order service calls inventory (reserve), then payment (charge), + then confirms. If payment fails or its breaker is open, it releases the stock and cancels the + order. Each step also publishes an event through the outbox for pattern 7. Chosen over a fully + event-driven saga: checkout answers in about 1 s and reuses the M2 breaker. +2. **Event consumer: inventory finishes the sale.** The saga only reserves stock. Inventory reads + `OrderConfirmed` from Redis and moves the units from reserved to sold (on hand goes down). + "Stop the bus" leaves stock reserved with events waiting in the outbox, then it settles when Redis + is back. Chosen over a new notifications service: no 6th service, and the effect is visible stock. +3. **Pipeline reach: GitHub deploys to Azure.** A manual "Deploy" workflow builds the changed images + and updates the Container Apps, logging in with OIDC (no Azure secret stored in GitHub). One Azure + identity for this repo, with access to `rg-microservices-12-patterns` only; `make provision` + re-grants it after each teardown. One-time setup done together with the owner. + Settled without a decision: Redis in Azure is one more Container App (1 replica while provisioned, + about $0.05 per 8-hour session; Azure Cache for Redis starts near $16/mo). `make provision` creates + the JWT signing key and stores it as a Container Apps secret. +4. **Delivery: M3 splits into 2 PRs.** M3a = patterns 8, 7, 9 (saga, outbox events, inventory + consumer, hardened containers), all local, about 2 days. M3b = pattern 3 (path-filtered CI, OIDC + Deploy workflow, Azure identity setup) plus the one real Azure run, about 1 day, started only on the + owner's go-ahead after M3a merges. + +M3a code review (Matt Pocock code-review skill, 2 axes) — what changed after it: +- Fixed: one failing event no longer blocks the ones behind it (retry, then dead-letter stream after + 5 tries); one checkout per order at a time (`checkout_lock`); stock rows locked in product order; + a duplicate hold insert returns the existing hold; a payment timeout no longer claims "nothing was + charged"; restock input validated; the decline switch needs `CHAOS_ENABLED=1`. +- Spec gaps closed: every saga step publishes an event (StockReserved, PaymentCharged, + OrderConfirmed, CheckoutFailed); a timed-out stock hold is released through the bus. +- Wording: the "stop the bus" demo disconnects the order service's relay from Redis. It does not stop + Redis itself; a real Redis outage behaves the same for the publisher (events wait in the outbox). +- Moved to M3b: SHA image tags and the GIT_SHA build arg on the `az acr build` path (provision), image + size on the Containerization page (it is in the CI run summary today). diff --git a/docs/screenshots/p07-event-driven.jpg b/docs/screenshots/p07-event-driven.jpg new file mode 100644 index 0000000..e42e867 Binary files /dev/null and b/docs/screenshots/p07-event-driven.jpg differ diff --git a/docs/screenshots/p08-saga.jpg b/docs/screenshots/p08-saga.jpg new file mode 100644 index 0000000..fe9ad8b Binary files /dev/null and b/docs/screenshots/p08-saga.jpg differ diff --git a/docs/screenshots/p09-containerization.jpg b/docs/screenshots/p09-containerization.jpg new file mode 100644 index 0000000..e5c7c17 Binary files /dev/null and b/docs/screenshots/p09-containerization.jpg differ diff --git a/frontend/src/api/gatewayClient.ts b/frontend/src/api/gatewayClient.ts index af4175c..c0dc709 100644 --- a/frontend/src/api/gatewayClient.ts +++ b/frontend/src/api/gatewayClient.ts @@ -236,3 +236,77 @@ export const resilienceApi = { body: JSON.stringify({ mode }), }), }; + +// --- M3a demo endpoints (patterns 8, 7, 9) ------------------------------------------------------- + +export interface SagaStep { + step: string; + status: "done" | "failed" | "compensated" | string; + detail: string | null; + at: string | null; +} + +export interface SagaRun { + saga_id: string; + order_id: number; + outcome: "completed" | "compensated" | "failed"; + steps: SagaStep[]; +} + +export interface OutboxRow { + event_id: string; + type: string; + payload: Record; + created_at: string | null; + published_at: string | null; + attempts: number; + last_error: string | null; +} + +export interface StockRow { + product_id: number; + on_hand: number; + reserved: number; + available: number; +} + +export interface EventsView { + outbox: { pending: number; results: OutboxRow[]; bus_connected: boolean } | null; + consumed: { event_id: string; type: string; reservation_id: string | null; outcome: string; handled_at: string | null }[] | null; + stock: StockRow[] | null; + reservations: { reservation_id: string; order_id: number; status: string; items: { product_id: number; quantity: number }[] }[] | null; + bus_connected: boolean[] | null; + chaos_enabled: boolean; +} + +export interface RuntimeInfo { + service: string; + instance: string; + uid?: number | null; + root?: boolean; + in_container?: boolean; + os?: string; + arch?: string; + python?: string; + git_sha?: string; + unreachable?: boolean; +} + +export const asyncApi = { + checkout: (token: string, simulateDecline: boolean) => + getAny<{ message?: string; saga_id?: string; failed_step?: string }>("/api/shop/order/checkout", { + method: "POST", + headers: { ...authHeaders(token), "Content-Type": "application/json" }, + body: JSON.stringify({ simulate_decline: simulateDecline }), + }), + sagas: (token: string) => request<{ results: SagaRun[] }>("/api/shop/order/sagas", { headers: authHeaders(token) }), + addToCart: (token: string, productId: number, qty: number) => shopApi.addToCart(token, productId, qty), + events: () => request("/api/patterns/events"), + setBus: (connected: boolean) => + getAny<{ bus_connected?: boolean; message?: string }>("/api/patterns/events/bus", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ connected }), + }), + runtime: () => request<{ processes: RuntimeInfo[] }>("/api/patterns/runtime"), +}; diff --git a/frontend/src/pages/Home.tsx b/frontend/src/pages/Home.tsx index cbed26d..fa8bff0 100644 --- a/frontend/src/pages/Home.tsx +++ b/frontend/src/pages/Home.tsx @@ -18,7 +18,7 @@ export default function Home() {

Patterns

-

The 12 patterns. 8 are live now, from Database per Service to Service Discovery; the last 4 arrive in M3.

+

The 12 patterns. 11 are live now; CI-CD arrives in M3b.

diff --git a/frontend/src/pages/PatternPage.tsx b/frontend/src/pages/PatternPage.tsx index 162bdd2..7874e03 100644 --- a/frontend/src/pages/PatternPage.tsx +++ b/frontend/src/pages/PatternPage.tsx @@ -3,9 +3,12 @@ import { Link, useParams } from "react-router-dom"; import ApiGatewayDemo from "../patterns/demos/ApiGatewayDemo"; import BoundedContextDemo from "../patterns/demos/BoundedContextDemo"; import CircuitBreakerDemo from "../patterns/demos/CircuitBreakerDemo"; +import ContainerizationDemo from "../patterns/demos/ContainerizationDemo"; import ContractTestingDemo from "../patterns/demos/ContractTestingDemo"; +import EventDrivenDemo from "../patterns/demos/EventDrivenDemo"; import DatabasePerServiceDemo from "../patterns/demos/DatabasePerServiceDemo"; import ObservabilityDemo from "../patterns/demos/ObservabilityDemo"; +import SagaDemo from "../patterns/demos/SagaDemo"; import ServiceDiscoveryDemo from "../patterns/demos/ServiceDiscoveryDemo"; import StatelessDemo from "../patterns/demos/StatelessDemo"; import { PATTERNS } from "../patterns/content"; @@ -19,6 +22,9 @@ const DEMOS: Record ReactNode> = { "stateless-services": () => , "contract-testing": () => , "service-discovery": () => , + "event-driven-architecture": () => , + saga: () => , + containerization: () => , }; export default function PatternPage() { diff --git a/frontend/src/pages/Patterns.tsx b/frontend/src/pages/Patterns.tsx index fce17c7..5ca1ef6 100644 --- a/frontend/src/pages/Patterns.tsx +++ b/frontend/src/pages/Patterns.tsx @@ -9,7 +9,7 @@ export default function Patterns() {

The 12 patterns

Each pattern has a page that explains it, says when not to use it, and runs a live demo against the real - services. {liveCount} of 12 are live; the rest arrive in milestone M3. + services. {liveCount} of 12 are live; the last one, CI-CD, arrives in milestone M3b.

diff --git a/frontend/src/patterns/content.ts b/frontend/src/patterns/content.ts index ee341a3..1e073ab 100644 --- a/frontend/src/patterns/content.ts +++ b/frontend/src/patterns/content.ts @@ -123,9 +123,74 @@ export const PATTERNS: PatternInfo[] = [ ], source: { label: "OpenTelemetry — Python zero-code instrumentation", url: "https://opentelemetry.io/docs/zero-code/python/" }, }, - { n: 7, slug: "event-driven-architecture", name: "Event-Driven Architecture", what: "Services communicate by publishing and consuming events.", milestone: "M3", live: false }, - { n: 8, slug: "saga", name: "Saga Pattern", what: "Coordinates distributed transactions and rolls back failures safely.", milestone: "M3", live: false }, - { n: 9, slug: "containerization", name: "Containerization", what: "Packages each service with all its dependencies.", milestone: "M3", live: false }, + { + n: 7, + slug: "event-driven-architecture", + name: "Event-Driven Architecture", + what: "Services communicate by publishing and consuming events.", + milestone: "M3", + live: true, + problem: + "If the order service had to call inventory, shipping and email directly to finish every sale, it would wait on all of them and fail whenever any one is down. Every new listener means changing the order service.", + inThisProject: [ + "The order service saves OrderConfirmed in an outbox table, in the same database transaction as the order. The event exists if and only if the order does.", + "A relay publishes outbox events to one Redis Stream, shop.events, once a second. If Redis is down, they wait in the table.", + "Inventory reads the stream in its own consumer group and turns the stock hold into a sale. It acknowledges an event only after handling it.", + "Delivery is at least once, so inventory records every event id it handled and skips duplicates.", + ], + whenNotToUse: [ + "When the caller needs the answer now, such as \"is this in stock?\", call the service directly. An event cannot answer a question.", + "Events make the system eventually consistent: for a moment, stock shows as held, not sold. The UI and the business must accept that.", + "Tracing a flow through events is harder than through calls. Carry IDs in every event, as these do.", + ], + source: { label: "microservices.io — Transactional outbox", url: "https://microservices.io/patterns/data/transactional-outbox.html" }, + }, + { + n: 8, + slug: "saga", + name: "Saga Pattern", + what: "Coordinates distributed transactions and rolls back failures safely.", + milestone: "M3", + live: true, + problem: + "Checkout touches 3 databases: stock, payment and orders. No transaction spans them. If the card is declined after stock was held, something must put the stock back, or it is lost for good.", + inThisProject: [ + "The order service orchestrates the saga: price (catalog), hold stock (inventory), charge (payment), confirm.", + "If a step fails, it undoes the earlier ones in reverse order. Only the stock hold changed another service, so the undo is releasing it.", + "If the release call itself fails, a StockReleaseRequested event goes through the outbox, so inventory releases it later.", + "Every step is logged in saga_step. Holds are keyed by the saga id, so a retried request never holds stock twice.", + "A declined card is a normal answer, not a failure, so it does not trip the M2 circuit breaker. The cart is kept for another try.", + ], + whenNotToUse: [ + "If the data can live in one database, use one local transaction. A saga is the price of splitting data, not a goal.", + "Every step needs an undo, and the undo must be safe to run twice. Steps that cannot be undone, such as sending an email, go last.", + "Between steps, other users can see the half-done state (held stock). Design for it.", + ], + source: { label: "microservices.io — Saga", url: "https://microservices.io/patterns/data/saga.html" }, + }, + { + n: 9, + slug: "containerization", + name: "Containerization", + what: "Packages each service with all its dependencies.", + milestone: "M3", + live: true, + problem: + "\"It works on my machine\": each server needs the right Python, libraries and settings. The owner's Mac is arm64 and Azure runs amd64, so even the processor differs.", + inThisProject: [ + "One image per service, plus the gateway and the registry: 7 Dockerfiles.", + "Multi-stage builds: dependencies are installed in a first stage, and only the virtualenv and source are copied into the final image.", + "Every image runs as uid 10001, never root, and carries the git commit it was built from.", + "Images are built only in the cloud (az acr build) or in GitHub Actions, always for linux/amd64. Never on the Mac.", + "CI builds every image on every pull request, starts it, checks /health, and fails if it runs as root.", + ], + whenNotToUse: [ + "A container is not a security boundary on its own. Non-root, small images and patched base images still matter.", + "Do not put secrets in images. Here, keys and passwords come from Container Apps secrets at run time.", + "For a single small app, a managed platform that builds from source can be simpler than owning Dockerfiles.", + ], + source: { label: "Docker docs — Multi-stage builds", url: "https://docs.docker.com/build/building/multi-stage/" }, + }, { n: 10, slug: "stateless-services", diff --git a/frontend/src/patterns/demos/ContainerizationDemo.tsx b/frontend/src/patterns/demos/ContainerizationDemo.tsx new file mode 100644 index 0000000..0aa7276 --- /dev/null +++ b/frontend/src/patterns/demos/ContainerizationDemo.tsx @@ -0,0 +1,112 @@ +import { useEffect, useState } from "react"; +import { RuntimeInfo, asyncApi } from "../../api/gatewayClient"; + +const DOCKERFILE = `FROM python:3.12-slim AS build # stage 1: install dependencies +RUN python -m venv /opt/venv +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY _shared ./_shared +RUN pip install --no-cache-dir "./_shared[flask]" + +FROM python:3.12-slim # stage 2: only what runs +ARG GIT_SHA=unknown +ENV RUN_MODE=azure GIT_SHA=$GIT_SHA ... +RUN useradd --uid 10001 --no-create-home app +COPY --from=build /opt/venv /opt/venv # no pip cache, no build files +COPY --chown=10001:10001 . . +USER 10001 # never root +CMD ["gunicorn", "--workers", "1", "--threads", "4", ...]`; + +export default function ContainerizationDemo() { + const [processes, setProcesses] = useState([]); + const [error, setError] = useState(null); + + useEffect(() => { + asyncApi + .runtime() + .then((r) => setProcesses(r.processes)) + .catch((err) => setError(String(err))); + }, []); + + const anyContainer = processes.some((p) => p.in_container); + + return ( +
+
+

How each process is running right now

+

+ {anyContainer + ? "These processes run in containers built from the Dockerfiles below." + : "Locally, make run starts plain processes, no containers (this project never uses Docker on the Mac). In Azure and in CI the same code runs in the images below, as uid 10001."} +

+ {error &&

{error}

} +
+ + + + + + + + + + + + + {processes.map((p) => ( + + + {p.unreachable ? ( + + ) : ( + <> + + + + + + + )} + + ))} + +
ProcessContainerRuns asPlatformPythonBuild
+ + {p.instance} + + unreachable + {p.in_container ? "yes" : "no"} + {p.root ? "root" : `uid ${p.uid}`} + + {p.os?.toLowerCase()}/{p.arch} + {p.python} + {p.git_sha === "local" ? "local" : p.git_sha?.slice(0, 7)} +
+
+
+ +
+
+

Every service's Dockerfile, in short

+
{DOCKERFILE}
+
+
+

What CI checks on every pull request

+
    +
  1. + Builds all 7 images (5 services, gateway, registry) for linux/amd64, the platform Azure + Container Apps runs. The owner's Mac is arm64, so images are never built on it. +
  2. +
  3. Starts each image and waits for /health to answer.
  4. +
  5. Fails if the process inside runs as root (uid 0).
  6. +
  7. Writes each image's size to the run summary.
  8. +
+

+ A .dockerignore in each service keeps local virtualenvs, databases and keys out of the build. + The image is labelled with the git commit it was built from, shown in the Build column in Azure. +

+
+
+
+ ); +} diff --git a/frontend/src/patterns/demos/EventDrivenDemo.tsx b/frontend/src/patterns/demos/EventDrivenDemo.tsx new file mode 100644 index 0000000..93fdb79 --- /dev/null +++ b/frontend/src/patterns/demos/EventDrivenDemo.tsx @@ -0,0 +1,169 @@ +import { useEffect, useState } from "react"; +import { EventsView, asyncApi, savedToken } from "../../api/gatewayClient"; + +function time(iso: string | null): string { + return iso ? new Date(iso.endsWith("Z") || iso.includes("+") ? iso : `${iso}Z`).toLocaleTimeString() : "–"; +} + +export default function EventDrivenDemo() { + const token = savedToken(); + const [view, setView] = useState(null); + const [error, setError] = useState(null); + const [message, setMessage] = useState(null); + const [busy, setBusy] = useState(false); + + function load() { + asyncApi + .events() + .then((v) => { + setView(v); + setError(null); + }) + .catch((err) => setError(String(err))); + } + + useEffect(() => { + load(); + const timer = setInterval(load, 1500); + return () => clearInterval(timer); + }, []); + + async function setBus(connected: boolean) { + const { status, body } = await asyncApi.setBus(connected); + setMessage(status === 200 ? (connected ? "Bus reconnected. Watch the waiting events go out." : "Bus disconnected. Buy something and watch the event wait.") : body.message ?? `HTTP ${status}`); + load(); + } + + async function buyOne() { + if (!token) return; + setBusy(true); + try { + await asyncApi.addToCart(token, 1, 1); + const { status, body } = await asyncApi.checkout(token, false); + setMessage(status === 200 ? "Bought 1 mug. OrderConfirmed is in the outbox." : body.message ?? `HTTP ${status}`); + } finally { + setBusy(false); + load(); + } + } + + const connected = view?.bus_connected?.every(Boolean) ?? true; + const mug = view?.stock?.find((s) => s.product_id === 1); + + return ( +
+ {error &&

{error}

} +
+
+

The bus

+

+ {connected ? "connected" : "disconnected"}{" "} + + {view?.outbox ? `${view.outbox.pending} event(s) waiting in the outbox` : "order service unreachable"} + +

+
+ + + {token && ( + + )} +
+ {!view?.chaos_enabled &&

Demo switches are off. Run the stack with make run to use them.

} + {message &&

{message}

} +
+
+

Mug stock in inventory

+ {mug && ( + + + + + + + + + + + +
On hand{mug.on_hand}
Held, waiting for OrderConfirmed{mug.reserved}
+ )} +

+ While the bus is down, a sale stays "held". Inventory sells it as soon as the event arrives. +

+
+
+ +
+
+

Order's outbox (publisher)

+

+ Saved in the same database transaction as the order. A relay publishes it to Redis, once a second. +

+
+ + + + + + + + + + + {view?.outbox?.results.slice(0, 8).map((e) => ( + + + + + + + ))} + +
EventSavedStatusTries
+ {e.type} + {time(e.created_at)} + + {e.published_at ? `published ${time(e.published_at)}` : "waiting"} + + {e.attempts}
+
+
+
+

What inventory did (consumer)

+

+ Reads the stream in the "inventory" consumer group. Each event is acted on once, even if delivered twice. +

+
+ + + + + + + + + + {view?.consumed?.slice(0, 8).map((e) => ( + + + + + + ))} + +
EventHandledResult
+ {e.type} + {time(e.handled_at)}{e.outcome}
+
+
+
+
+ ); +} diff --git a/frontend/src/patterns/demos/SagaDemo.tsx b/frontend/src/patterns/demos/SagaDemo.tsx new file mode 100644 index 0000000..f0b8f71 --- /dev/null +++ b/frontend/src/patterns/demos/SagaDemo.tsx @@ -0,0 +1,163 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router-dom"; +import { SagaRun, StockRow, asyncApi, savedToken } from "../../api/gatewayClient"; + +const STEP_LABEL: Record = { + price: "1. Price the cart (catalog)", + reserve_stock: "2. Hold the stock (inventory)", + charge_payment: "3. Charge the card (payment)", + confirm: "4. Confirm the order", + release_stock: "Undo step 2: release the stock", +}; + +const PILL: Record = { done: "done", failed: "failed", compensated: "running" }; +const DEMO_PRODUCT = 1; + +export default function SagaDemo() { + const token = savedToken(); + const [decline, setDecline] = useState(false); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState<{ ok: boolean; message: string } | null>(null); + const [sagas, setSagas] = useState([]); + const [stock, setStock] = useState(null); + + async function refresh() { + if (!token) return; + const [history, view] = await Promise.all([asyncApi.sagas(token).catch(() => null), asyncApi.events().catch(() => null)]); + if (history) setSagas(history.results); + setStock(view?.stock?.find((s) => s.product_id === DEMO_PRODUCT) ?? null); + } + + useEffect(() => { + refresh(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + if (!token) { + return ( +

+ This demo checks out a real cart, so it needs a signed-in user. Register or log in on the Shop + page, then come back. +

+ ); + } + + async function buy() { + if (!token) return; + setBusy(true); + setResult(null); + try { + await asyncApi.addToCart(token, DEMO_PRODUCT, 2); + const { status, body } = await asyncApi.checkout(token, decline); + setResult({ ok: status === 200, message: status === 200 ? "Order placed. Payment approved." : body.message ?? `HTTP ${status}` }); + } catch (err) { + setResult({ ok: false, message: String(err) }); + } finally { + await refresh(); + setTimeout(refresh, 1500); // inventory sells the hold a moment later, when it reads the event + setBusy(false); + } + } + + const latest = sagas[0]; + + return ( +
+
+
+

Step 1. Buy 2 Contoso Mugs

+

Adds 2 mugs to your cart and checks out. The order service runs the saga below.

+ + + {result && ( +

+ {result.ok ? "Completed" : "Rolled back"}{" "} + {result.message} +

+ )} +
+
+

Mug stock in inventory

+ {stock ? ( + + + + + + + + + + + + + + + +
On hand{stock.on_hand}
Held for checkouts in progress{stock.reserved}
Available to sell{stock.available}
+ ) : ( +

Inventory is unreachable.

+ )} +

+ A declined card leaves all 3 numbers as they were. An approved one lowers on hand by 2, once inventory reads + the OrderConfirmed event. +

+
+
+ + {latest && ( +
+

Step 2. What the saga did

+

+ Saga {latest.saga_id.slice(0, 8)} for order #{latest.order_id}:{" "} + + {latest.outcome} + +

+
    + {latest.steps.map((s, i) => ( +
  1. + {s.status} + {STEP_LABEL[s.step] ?? s.step} + {s.detail && · {s.detail}} +
  2. + ))} +
+
+ )} + + {sagas.length > 1 && ( +
+

Your earlier checkouts

+ + + + + + + + + + + {sagas.slice(1).map((s) => ( + + + + + + + ))} + +
SagaOrderOutcomeStopped at
+ {s.saga_id.slice(0, 8)} + #{s.order_id}{s.outcome}{s.steps.find((x) => x.status === "failed")?.step ?? "–"}
+
+ )} +
+ ); +} diff --git a/frontend/src/theme/contoso.css b/frontend/src/theme/contoso.css index 579ec6f..4fe916d 100644 --- a/frontend/src/theme/contoso.css +++ b/frontend/src/theme/contoso.css @@ -337,3 +337,24 @@ h3 { padding-left: 1.1rem; font-size: 0.85rem; } + +/* Saga timeline (pattern 8) */ +.saga-timeline { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 0.4rem; +} +.saga-step { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + align-items: baseline; + padding: 0.5rem 0.75rem; + border-left: 3px solid var(--contoso-success); + background: var(--contoso-bg); + border-radius: 4px; +} +.saga-step.failed { border-left-color: var(--contoso-danger); } +.saga-step.compensated { border-left-color: var(--contoso-warning); } diff --git a/gateway/.dockerignore b/gateway/.dockerignore new file mode 100644 index 0000000..39daf35 --- /dev/null +++ b/gateway/.dockerignore @@ -0,0 +1,10 @@ +# Keeps the build context (what az acr build uploads) to source only. Pattern 9. +.venv/ +instance/ +tests/ +__pycache__/ +*.pyc +*.sqlite3 +*.pem +.env +.pytest_cache/ diff --git a/gateway/Dockerfile b/gateway/Dockerfile index f949e6b..4ec3257 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,18 +1,29 @@ -# Built exclusively via `az acr build` (cloud-side) — no local Docker daemon required. -FROM python:3.12-slim - -WORKDIR /app +# Pattern 9: built only in the cloud (az acr build) or in GitHub Actions, for linux/amd64. +# Multi-stage: dependencies in a virtualenv, then a clean image running as an unprivileged user. +FROM python:3.12-slim AS build +WORKDIR /build +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH COPY requirements.txt . -RUN pip install --no-cache-dir --root-user-action=ignore -r requirements.txt - -COPY . . -# shared/ (shop_common) is staged into ./_shared by infra/provision.py before `az acr build`. -RUN pip install --no-cache-dir --root-user-action=ignore "./_shared[fastapi]" - -ENV RUN_MODE=azure \ - PYTHONUNBUFFERED=1 +RUN pip install --no-cache-dir -r requirements.txt +# shared/ (shop_common) is staged into ./_shared before the build (infra/provision.py, CI). +COPY _shared ./_shared +RUN pip install --no-cache-dir "./_shared[fastapi]" +FROM python:3.12-slim +ARG GIT_SHA=unknown +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + RUN_MODE=azure \ + GIT_SHA=$GIT_SHA +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /opt/venv /opt/venv +COPY --chown=10001:10001 . . +RUN rm -rf _shared +USER 10001 EXPOSE 8100 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8100"] diff --git a/gateway/app/observability.py b/gateway/app/observability.py index ab65b4f..042aedc 100644 --- a/gateway/app/observability.py +++ b/gateway/app/observability.py @@ -13,10 +13,11 @@ from opentelemetry import trace from shop_common import metrics +from shop_common.runtime import runtime_info from shop_common.telemetry import REQUEST_ID_HEADER, SPAN_STORE, current_trace_id, get_logger, init_tracing SERVICE_NAME = "gateway" -_QUIET_PATHS = ("/health", "/metrics", "/debug/spans") +_QUIET_PATHS = ("/health", "/metrics", "/debug/spans", "/debug/runtime") def install(app: FastAPI) -> None: @@ -54,6 +55,10 @@ async def prometheus_metrics(): body, content_type = metrics.render() return Response(content=body, media_type=content_type) + @app.get("/debug/runtime", include_in_schema=False) + async def debug_runtime(): + return runtime_info(SERVICE_NAME, SERVICE_NAME) + @app.get("/debug/spans", include_in_schema=False) async def debug_spans(trace_id: str | None = None): spans = SPAN_STORE.by_trace(trace_id) if trace_id else SPAN_STORE.recent() diff --git a/gateway/app/routers/resilience.py b/gateway/app/routers/resilience.py index 488cca0..d20caa5 100644 --- a/gateway/app/routers/resilience.py +++ b/gateway/app/routers/resilience.py @@ -1,5 +1,5 @@ -"""Live demo endpoints for the M2 pattern pages: 4 (Circuit Breaker), 10 (Stateless Services) and -12 (Service Discovery). The stop and break switches work only with CHAOS_ENABLED=1 (make run).""" +"""Live demo endpoints for patterns 4 (Circuit Breaker), 7 (Event-Driven), 10 (Stateless Services) +and 12 (Service Discovery). The stop, break and bus switches work only with CHAOS_ENABLED=1 (make run).""" import asyncio import time @@ -8,6 +8,8 @@ from fastapi.responses import JSONResponse from pydantic import BaseModel +from shop_common.runtime import runtime_info + from .. import config from ..observability import upstream_headers @@ -120,3 +122,73 @@ async def set_payment_mode(body: PaymentMode): except httpx.HTTPError as exc: return JSONResponse({"message": f"payment unreachable ({exc.__class__.__name__})"}, status_code=503) return JSONResponse(response.json(), status_code=response.status_code) + + +# --- Pattern 7: Event-Driven Architecture ------------------------------------------------------ + +@router.get("/events") +async def events(request: Request): + """Both ends of the bus: order's outbox (published or waiting) and what inventory acted on, + plus current stock, so the page can show a hold turning into a sale.""" + async def get(service: str, path: str, base: str | None = None): + try: + response = await client.get(f"{base or config.url_for(service)}{path}", headers=upstream_headers(request)) + return response.json() if response.status_code == 200 else None + except httpx.HTTPError: + return None + + async with _client() as client: + # Every order replica reports its own bus switch; they share one outbox table, so any + # replica's answer is the outbox. + consumed, stock, reservations, *outboxes = await asyncio.gather( + get("inventory", "/api/inventory/events"), + get("inventory", "/api/inventory"), + get("inventory", "/api/inventory/reservations"), + *(get("order", "/api/order/outbox", base=b) for b in config.urls_for("order")), + ) + buses = [o for o in outboxes if o] + outbox = buses[0] if buses else None + return { + "outbox": outbox, + "consumed": consumed["results"] if consumed else None, + "stock": stock["results"] if stock else None, + "reservations": reservations["results"] if reservations else None, + "bus_connected": [b["bus_connected"] for b in buses] or None, + "chaos_enabled": config.CHAOS_ENABLED, + } + + +class BusSwitch(BaseModel): + connected: bool + + +@router.post("/events/bus") +async def set_bus(body: BusSwitch): + """Disconnects (or reconnects) every order replica's outbox relay from the bus.""" + if not config.CHAOS_ENABLED: + return JSONResponse(_CHAOS_OFF, status_code=403) + async with _client() as client: + results = await asyncio.gather( + *(client.post(f"{base}/debug/bus", json={"connected": body.connected}) for base in config.urls_for("order")), + return_exceptions=True, + ) + switched = sum(1 for r in results if isinstance(r, httpx.Response) and r.status_code == 200) + return {"bus_connected": body.connected, "replicas_switched": switched} + + +# --- Pattern 9: Containerization --------------------------------------------------------------- + +@router.get("/runtime") +async def runtime(request: Request): + """How every process runs: as root or not, in a container or not, which build (git SHA).""" + async def one(service: str, base: str) -> dict: + try: + response = await client.get(f"{base}/debug/runtime", headers=upstream_headers(request)) + return response.json() + except (httpx.HTTPError, ValueError): + return {"service": service, "instance": base, "unreachable": True} + + async with _client() as client: + targets = [(n, url) for n in config.SERVICE_BASE_URLS for url in config.urls_for(n)] + services = await asyncio.gather(*(one(n, url) for n, url in targets)) + return {"processes": [runtime_info("gateway", "gateway"), *services]} diff --git a/gateway/tests/test_api.py b/gateway/tests/test_api.py index 0ee81b3..e99fd3d 100644 --- a/gateway/tests/test_api.py +++ b/gateway/tests/test_api.py @@ -119,3 +119,20 @@ async def fake_request(self, method, url, **kwargs): assert r.status_code == 200 assert r.headers["X-Instance-Id"] == "catalog-alive" assert "http://dead/api/products" in tried + + +def test_events_view_reports_unreachable_services_as_none(monkeypatch): + from app import config + + for name in config.SERVICE_BASE_URLS: + monkeypatch.setitem(config.SERVICE_BASE_URLS, name, "http://127.0.0.1:9") + body = client.get("/api/patterns/events").json() + assert body["outbox"] is None and body["stock"] is None + assert client.post("/api/patterns/events/bus", json={"connected": False}).status_code == 403 + + +def test_runtime_reports_the_gateway_without_a_user_name(): + gateway = client.get("/api/patterns/runtime").json()["processes"][0] + assert gateway["service"] == "gateway" + assert set(gateway) >= {"uid", "root", "in_container", "arch", "git_sha"} + assert "user" not in gateway diff --git a/registry/.dockerignore b/registry/.dockerignore new file mode 100644 index 0000000..39daf35 --- /dev/null +++ b/registry/.dockerignore @@ -0,0 +1,10 @@ +# Keeps the build context (what az acr build uploads) to source only. Pattern 9. +.venv/ +instance/ +tests/ +__pycache__/ +*.pyc +*.sqlite3 +*.pem +.env +.pytest_cache/ diff --git a/registry/Dockerfile b/registry/Dockerfile new file mode 100644 index 0000000..6b25c45 --- /dev/null +++ b/registry/Dockerfile @@ -0,0 +1,25 @@ +# Pattern 9: built only in the cloud (az acr build) or in GitHub Actions, for linux/amd64. +# Multi-stage: dependencies in a virtualenv, then a clean image running as an unprivileged user. + +FROM python:3.12-slim AS build +WORKDIR /build +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +FROM python:3.12-slim +ARG GIT_SHA=unknown +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + RUN_MODE=azure \ + GIT_SHA=$GIT_SHA +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /opt/venv /opt/venv +COPY --chown=10001:10001 . . +USER 10001 +EXPOSE 8500 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8500"] diff --git a/scripts/run_local.py b/scripts/run_local.py index 5b5c4a8..51d8e10 100644 --- a/scripts/run_local.py +++ b/scripts/run_local.py @@ -5,11 +5,14 @@ the gateway, then the frontend, each gated on its /health endpoint. The list comes from scripts/service_table.py. +Redis (the event bus, patterns 7 and 8) starts first, in memory only, if redis-server is installed. A service process that exits (for example one stopped from the Stateless or Service Discovery demo) is restarted after RESTART_DELAY_SECONDS, so the registry shows it leave and come back. """ import os +import shutil import signal +import socket import subprocess import sys import time @@ -20,10 +23,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from _procutil import ensure_port_free # noqa: E402 from service_table import ( # noqa: E402 - FLASK_SERVICES, FRONTEND_DIR, FRONTEND_PORT, GATEWAY, REGISTRY, REPLICA_PORTS, Service, + FLASK_SERVICES, FRONTEND_DIR, FRONTEND_PORT, GATEWAY, REDIS_PORT, REDIS_URL, REGISTRY, REPLICA_PORTS, + REPO_ROOT, Service, ) RESTART_DELAY_SECONDS = 20 +redis_running = False processes: list[subprocess.Popen] = [] # pid -> how to start that process again (Flask services only) restartable: dict[int, tuple[str, object]] = {} @@ -51,9 +56,36 @@ def _service_url_env() -> dict[str, str]: """Every process gets every service's static URL (the fallback), the registry, and the demo switches.""" env = {f"{s.name.upper()}_SERVICE_BASE_URL": s.local_url for s in FLASK_SERVICES} env.update(REGISTRY_URL=REGISTRY.local_url, CHAOS_ENABLED="1") + if redis_running: + env["REDIS_URL"] = REDIS_URL return env +def start_redis() -> bool: + """Starts a throwaway redis-server: in memory only, no files written, stopped with the stack.""" + global redis_running + if shutil.which("redis-server") is None: + print(" ! redis-server not found (brew install redis). Events will wait in the outbox.") + return False + ensure_port_free(REDIS_PORT, "redis") + proc = subprocess.Popen( + ["redis-server", "--port", str(REDIS_PORT), "--bind", "127.0.0.1", "--save", "", "--appendonly", "no", + "--loglevel", "warning"], + cwd=str(REPO_ROOT), + ) + processes.append(proc) + print(f" redis starting on :{REDIS_PORT} (pid {proc.pid})…") + for _ in range(20): + try: + with socket.create_connection(("127.0.0.1", REDIS_PORT), timeout=0.5): + redis_running = True + return True + except OSError: + time.sleep(0.25) + print(" ! redis did not start. Events will wait in the outbox.") + return False + + def start_registry() -> subprocess.Popen: ensure_port_free(REGISTRY.local_port, REGISTRY.name) proc = subprocess.Popen( @@ -125,6 +157,7 @@ def main() -> int: signal.signal(signal.SIGTERM, shutdown) print("=== Starting local stack (no Docker) ===") + start_redis() start_registry() if not _wait_for_health(f"{REGISTRY.local_url}/health"): print("registry failed to become healthy — aborting.") @@ -152,6 +185,7 @@ def main() -> int: Frontend http://127.0.0.1:{FRONTEND_PORT} Gateway {GATEWAY.local_url} Registry {REGISTRY.local_url} + Redis {REDIS_URL if redis_running else "not running (events wait in the outbox)"} {services} Press Ctrl+C to stop everything. diff --git a/scripts/service_table.py b/scripts/service_table.py index 8c33890..229bec7 100644 --- a/scripts/service_table.py +++ b/scripts/service_table.py @@ -40,13 +40,18 @@ def local_url(self) -> str: PYTHON_SERVICES = FLASK_SERVICES + [GATEWAY, REGISTRY] +# The event bus (patterns 7 and 8): a Homebrew redis-server, no Docker. Offset from Redis' default +# 6379 so a Redis the owner already runs is never touched. +REDIS_PORT = int(os.environ.get("REDIS_PORT", 6390)) +REDIS_URL = f"redis://127.0.0.1:{REDIS_PORT}/0" + # Extra local replicas (pattern 10): service name -> extra ports. Each runs as its own process # from the same folder, and registers with the registry under its own instance id. REPLICA_PORTS: dict[str, list[int]] = {"order": [5113]} -# Suites whose tests need pact-python (pattern 11): order is the consumer, payment and catalog +# Suites whose tests need pact-python (pattern 11): order is the consumer; payment, catalog and inventory # verify they still honour its contracts. -CONTRACT_SUITES = {"order", "payment", "catalog"} +CONTRACT_SUITES = {"order", "payment", "catalog", "inventory"} CONTRACTS_DIR = REPO_ROOT / "contracts" # shop_common (shared/), installed into every Python venv by setup.py. diff --git a/services/catalog/.dockerignore b/services/catalog/.dockerignore new file mode 100644 index 0000000..39daf35 --- /dev/null +++ b/services/catalog/.dockerignore @@ -0,0 +1,10 @@ +# Keeps the build context (what az acr build uploads) to source only. Pattern 9. +.venv/ +instance/ +tests/ +__pycache__/ +*.pyc +*.sqlite3 +*.pem +.env +.pytest_cache/ diff --git a/services/catalog/Dockerfile b/services/catalog/Dockerfile index beafb77..5c37183 100644 --- a/services/catalog/Dockerfile +++ b/services/catalog/Dockerfile @@ -1,19 +1,34 @@ -# Built exclusively via `az acr build` (cloud-side) — no local Docker daemon required. -FROM python:3.12-slim - -WORKDIR /app +# Pattern 9: one image per service, built only in the cloud (az acr build) or in GitHub Actions, +# always for linux/amd64 (Azure Container Apps). Never built on the owner's arm64 Mac. +# +# Stage 1 installs dependencies into a virtualenv. Stage 2 copies only that virtualenv and the +# source into a clean image and runs as an unprivileged user, so no compiler cache, pip cache or +# root process ends up in production. +FROM python:3.12-slim AS build +WORKDIR /build +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH COPY requirements.txt . -RUN pip install --no-cache-dir --root-user-action=ignore -r requirements.txt - -COPY . . -# shared/ (shop_common) is staged into ./_shared by infra/provision.py before `az acr build`. -RUN pip install --no-cache-dir --root-user-action=ignore "./_shared[flask]" +RUN pip install --no-cache-dir -r requirements.txt +# shared/ (shop_common) is staged into ./_shared before the build (infra/provision.py, CI). +COPY _shared ./_shared +RUN pip install --no-cache-dir "./_shared[flask]" -ENV RUN_MODE=azure \ +FROM python:3.12-slim +ARG GIT_SHA=unknown +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + RUN_MODE=azure \ FLASK_ENV=production \ - PYTHONUNBUFFERED=1 - + GIT_SHA=$GIT_SHA +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /opt/venv /opt/venv +COPY --chown=10001:10001 . . +RUN rm -rf _shared && mkdir -p instance && chown 10001:10001 instance +USER 10001 EXPOSE 5102 # One process with threads, so the in-memory span store (pattern 6) sees every request. diff --git a/services/inventory/.dockerignore b/services/inventory/.dockerignore new file mode 100644 index 0000000..39daf35 --- /dev/null +++ b/services/inventory/.dockerignore @@ -0,0 +1,10 @@ +# Keeps the build context (what az acr build uploads) to source only. Pattern 9. +.venv/ +instance/ +tests/ +__pycache__/ +*.pyc +*.sqlite3 +*.pem +.env +.pytest_cache/ diff --git a/services/inventory/Dockerfile b/services/inventory/Dockerfile index 0c0ab19..1daa4af 100644 --- a/services/inventory/Dockerfile +++ b/services/inventory/Dockerfile @@ -1,19 +1,34 @@ -# Built exclusively via `az acr build` (cloud-side) — no local Docker daemon required. -FROM python:3.12-slim - -WORKDIR /app +# Pattern 9: one image per service, built only in the cloud (az acr build) or in GitHub Actions, +# always for linux/amd64 (Azure Container Apps). Never built on the owner's arm64 Mac. +# +# Stage 1 installs dependencies into a virtualenv. Stage 2 copies only that virtualenv and the +# source into a clean image and runs as an unprivileged user, so no compiler cache, pip cache or +# root process ends up in production. +FROM python:3.12-slim AS build +WORKDIR /build +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH COPY requirements.txt . -RUN pip install --no-cache-dir --root-user-action=ignore -r requirements.txt - -COPY . . -# shared/ (shop_common) is staged into ./_shared by infra/provision.py before `az acr build`. -RUN pip install --no-cache-dir --root-user-action=ignore "./_shared[flask]" +RUN pip install --no-cache-dir -r requirements.txt +# shared/ (shop_common) is staged into ./_shared before the build (infra/provision.py, CI). +COPY _shared ./_shared +RUN pip install --no-cache-dir "./_shared[flask]" -ENV RUN_MODE=azure \ +FROM python:3.12-slim +ARG GIT_SHA=unknown +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + RUN_MODE=azure \ FLASK_ENV=production \ - PYTHONUNBUFFERED=1 - + GIT_SHA=$GIT_SHA +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /opt/venv /opt/venv +COPY --chown=10001:10001 . . +RUN rm -rf _shared && mkdir -p instance && chown 10001:10001 instance +USER 10001 EXPOSE 5105 # One process with threads, so the in-memory span store (pattern 6) sees every request. diff --git a/services/inventory/application/__init__.py b/services/inventory/application/__init__.py index a497636..c200054 100644 --- a/services/inventory/application/__init__.py +++ b/services/inventory/application/__init__.py @@ -3,6 +3,9 @@ Bounded context (pattern 2): Inventory does not know a product's name or price. It knows a StockItem keyed by the catalog's product_id, with how many are on hand and how many are reserved. Catalog owns "what we sell"; Inventory owns "how many we have". + +M3: holds stock for the checkout saga (pattern 8) and finishes the sale when an OrderConfirmed +event arrives from the bus (pattern 7). """ import os @@ -31,8 +34,15 @@ def create_app(config_name: str | None = None) -> Flask: app.register_blueprint(inventory_api) db.create_all() - from .seed import seed_stock_if_empty + from .seed import seed_missing_stock - seed_stock_if_empty() + seed_missing_stock() install_observability(app, "inventory", db) + + # Pattern 7: act on OrderConfirmed events from the bus (no-op without REDIS_URL). + from shop_common.flask_support import instance_id + + from .consumer import start as start_consumer + + start_consumer(app, instance_id("inventory")) return app diff --git a/services/inventory/application/consumer.py b/services/inventory/application/consumer.py new file mode 100644 index 0000000..9102b84 --- /dev/null +++ b/services/inventory/application/consumer.py @@ -0,0 +1,44 @@ +"""Inventory's event handler (pattern 7): finishes a sale when the order service confirms it. + +OrderConfirmed -> commit the reservation (reserved units become sold, on hand goes down) +StockReleaseRequested -> release the reservation (the saga's fallback when its direct call failed) +Anything else is not inventory's business and is acknowledged without action. +""" +from . import db, stock +from .models import ConsumedEvent + +HANDLERS = {"OrderConfirmed": stock.commit, "StockReleaseRequested": stock.release} + + +def handle(event: dict) -> None: + action = HANDLERS.get(event.get("type")) + if action is None: + return + if db.session.get(ConsumedEvent, event["event_id"]) is not None: + return # seen before: at-least-once delivery sent it twice + reservation_id = event["payload"].get("reservation_id") + # action() commits on its own. If we crash before the ConsumedEvent row below, the event is + # redelivered and action() is a no-op the second time (it checks the reservation's status). + reservation = action(reservation_id) + outcome = f"reservation {reservation.status}" if reservation else "no such reservation" + db.session.add(ConsumedEvent(event_id=event["event_id"], type=event["type"], + reservation_id=reservation_id, outcome=outcome)) + db.session.commit() + + +def start(app, instance_id: str) -> None: + from shop_common.events import start_consumer + from shop_common.flask_support import is_serving_process + from shop_common.telemetry import get_logger + + if not is_serving_process(app): + return + + def in_app(event: dict) -> None: + with app.app_context(): + try: + handle(event) + finally: + db.session.remove() + + start_consumer("inventory", instance_id, in_app, log=get_logger()) diff --git a/services/inventory/application/models.py b/services/inventory/application/models.py index 4ca5a28..e77bf13 100644 --- a/services/inventory/application/models.py +++ b/services/inventory/application/models.py @@ -21,3 +21,54 @@ def to_json(self): "reserved": self.reserved, "available": self.available, } + + +def _utcnow(): + from datetime import datetime, timezone + + return datetime.now(timezone.utc) + + +class Reservation(db.Model): + """Stock held for one checkout attempt (pattern 8). Keyed by the order service's saga id, so a + retried request or a redelivered event finds the same row instead of holding stock twice.""" + + __tablename__ = "reservation" + + id = db.Column(db.String(36), primary_key=True) # the saga id + order_id = db.Column(db.Integer, nullable=False, index=True) + items = db.Column(db.JSON, nullable=False) # [{"product_id": 1, "quantity": 2}] + status = db.Column(db.String(10), nullable=False) # RESERVED -> SOLD | RELEASED + created_at = db.Column(db.DateTime, default=_utcnow) + updated_at = db.Column(db.DateTime, default=_utcnow, onupdate=_utcnow) + + def to_json(self): + return { + "reservation_id": self.id, + "order_id": self.order_id, + "items": self.items, + "status": self.status, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + + +class ConsumedEvent(db.Model): + """Every event inventory acted on (pattern 7). Also the dedupe record: an event id seen before + is skipped, which makes handling idempotent under at-least-once delivery.""" + + __tablename__ = "consumed_event" + + event_id = db.Column(db.String(36), primary_key=True) + type = db.Column(db.String(40), nullable=False) + reservation_id = db.Column(db.String(36), nullable=True) + outcome = db.Column(db.String(120), nullable=False) + handled_at = db.Column(db.DateTime, default=_utcnow) + + def to_json(self): + return { + "event_id": self.event_id, + "type": self.type, + "reservation_id": self.reservation_id, + "outcome": self.outcome, + "handled_at": self.handled_at.isoformat() if self.handled_at else None, + } diff --git a/services/inventory/application/routes.py b/services/inventory/application/routes.py index e3cb0cc..6587d5b 100644 --- a/services/inventory/application/routes.py +++ b/services/inventory/application/routes.py @@ -1,11 +1,24 @@ -from flask import Blueprint, jsonify, make_response +from flask import Blueprint, jsonify, make_response, request +from pydantic import BaseModel, Field, ValidationError +from shop_common.flask_support import chaos_enabled -from . import db -from .models import StockItem +from . import db, stock +from .models import ConsumedEvent, Reservation, StockItem inventory_api = Blueprint("inventory_api", __name__) +class Line(BaseModel): + product_id: int = Field(gt=0) + quantity: int = Field(gt=0) + + +class ReserveRequest(BaseModel): + reservation_id: str = Field(min_length=8, max_length=36) + order_id: int = Field(gt=0) + items: list[Line] = Field(min_length=1) + + @inventory_api.get("/api/inventory") def list_stock(): return jsonify({"results": [s.to_json() for s in StockItem.query.order_by(StockItem.product_id)]}) @@ -17,3 +30,57 @@ def get_stock(product_id: int): if item is None: return make_response(jsonify({"message": f"No stock record for product {product_id}"}), 404) return jsonify({"result": item.to_json()}) + + +@inventory_api.post("/api/inventory/reservations") +def reserve(): + try: + data = ReserveRequest.model_validate(request.get_json(silent=True) or {}) + except ValidationError as exc: + return make_response(jsonify({"message": "Invalid request", "errors": exc.errors(include_url=False)}), 400) + try: + reservation = stock.reserve(data.reservation_id, data.order_id, [line.model_dump() for line in data.items]) + except stock.OutOfStock as exc: + return make_response(jsonify({"message": "Not enough stock", "short": exc.args[0]}), 409) + return jsonify({"result": reservation.to_json()}) + + +@inventory_api.post("/api/inventory/reservations//release") +def release(reservation_id: str): + reservation = stock.release(reservation_id) + if reservation is None: + return make_response(jsonify({"message": "No such reservation"}), 404) + return jsonify({"result": reservation.to_json()}) + + +@inventory_api.get("/api/inventory/reservations") +def list_reservations(): + rows = Reservation.query.order_by(Reservation.updated_at.desc()).limit(20) + return jsonify({"results": [r.to_json() for r in rows]}) + + +@inventory_api.get("/api/inventory/events") +def consumed_events(): + rows = ConsumedEvent.query.order_by(ConsumedEvent.handled_at.desc()).limit(20) + return jsonify({"results": [e.to_json() for e in rows]}) + + +class RestockRequest(BaseModel): + quantity: int = Field(gt=0, le=1000) + + +@inventory_api.post("/api/inventory//restock") +def restock(product_id: int): + """Demo and smoke-test helper: adds units on hand. Only with CHAOS_ENABLED=1 (make run).""" + if not chaos_enabled(): + return make_response(jsonify({"message": "Demo switches are off (CHAOS_ENABLED is not 1)"}), 403) + try: + data = RestockRequest.model_validate(request.get_json(silent=True) or {}) + except ValidationError as exc: + return make_response(jsonify({"message": "Invalid request", "errors": exc.errors(include_url=False)}), 400) + item = db.session.get(StockItem, product_id) + if item is None: + return make_response(jsonify({"message": f"No stock record for product {product_id}"}), 404) + item.on_hand += data.quantity + db.session.commit() + return jsonify({"result": item.to_json()}) diff --git a/services/inventory/application/seed.py b/services/inventory/application/seed.py index 11cfbb4..ee92b0f 100644 --- a/services/inventory/application/seed.py +++ b/services/inventory/application/seed.py @@ -1,13 +1,17 @@ -"""Seeds stock for the catalog's 6 sample products (ids 1-6) on first local start. Idempotent.""" +"""Seeds stock for the catalog's 24 sample products (ids 1-24) on start. Idempotent: it only adds +rows that are missing, so counts that changed through sales are never reset.""" from . import db from .models import StockItem -SAMPLE_STOCK = {1: 25, 2: 40, 3: 60, 4: 15, 5: 10, 6: 30} +CATALOG_SAMPLE_COUNT = 24 +SAMPLE_STOCK = {1: 25, 2: 40, 3: 60, 4: 15, 5: 10, 6: 30} | { + product_id: 20 for product_id in range(7, CATALOG_SAMPLE_COUNT + 1) +} -def seed_stock_if_empty() -> None: - if StockItem.query.first() is not None: - return +def seed_missing_stock() -> None: + existing = {row.product_id for row in db.session.query(StockItem.product_id)} for product_id, on_hand in SAMPLE_STOCK.items(): - db.session.add(StockItem(product_id=product_id, on_hand=on_hand, reserved=0)) + if product_id not in existing: + db.session.add(StockItem(product_id=product_id, on_hand=on_hand, reserved=0)) db.session.commit() diff --git a/services/inventory/application/stock.py b/services/inventory/application/stock.py new file mode 100644 index 0000000..7bc538a --- /dev/null +++ b/services/inventory/application/stock.py @@ -0,0 +1,77 @@ +"""Stock rules for the checkout saga (pattern 8) and the event consumer (pattern 7). + +reserve: available -> reserved (saga step 1; all lines or none) +release: reserved -> available (saga compensation) +commit: reserved -> sold (on OrderConfirmed: on hand goes down) + +Each is idempotent: calling it again for the same reservation changes nothing. +""" +from sqlalchemy.exc import IntegrityError + +from . import db +from .models import Reservation, StockItem + + +class OutOfStock(Exception): + pass + + +def _lock(product_id: int) -> StockItem | None: + # Row lock on MySQL; SQLite serialises writers anyway. + return db.session.query(StockItem).filter_by(product_id=product_id).with_for_update().one_or_none() + + +def _in_lock_order(items: list[dict]) -> list[dict]: + """Always lock stock rows in product id order, so two checkouts for the same products cannot + each hold one row and wait for the other's (a deadlock on MySQL).""" + return sorted(items, key=lambda line: line["product_id"]) + + +def reserve(reservation_id: str, order_id: int, items: list[dict]) -> Reservation: + existing = db.session.get(Reservation, reservation_id) + if existing is not None: + return existing + items = _in_lock_order(items) + short = [] + for line in items: + stock = _lock(line["product_id"]) + if stock is None or stock.available < line["quantity"]: + short.append({"product_id": line["product_id"], "wanted": line["quantity"], + "available": stock.available if stock else 0}) + if short: + db.session.rollback() + raise OutOfStock(short) + for line in items: + _lock(line["product_id"]).reserved += line["quantity"] + reservation = Reservation(id=reservation_id, order_id=order_id, items=items, status="RESERVED") + db.session.add(reservation) + try: + db.session.commit() + except IntegrityError: # the same hold, sent twice at once, landed first: keep that one + db.session.rollback() + return db.session.get(Reservation, reservation_id) + return reservation + + +def release(reservation_id: str) -> Reservation | None: + reservation = db.session.get(Reservation, reservation_id) + if reservation is None or reservation.status != "RESERVED": + return reservation + for line in _in_lock_order(reservation.items): + _lock(line["product_id"]).reserved -= line["quantity"] + reservation.status = "RELEASED" + db.session.commit() + return reservation + + +def commit(reservation_id: str) -> Reservation | None: + reservation = db.session.get(Reservation, reservation_id) + if reservation is None or reservation.status != "RESERVED": + return reservation + for line in _in_lock_order(reservation.items): + stock = _lock(line["product_id"]) + stock.reserved -= line["quantity"] + stock.on_hand -= line["quantity"] + reservation.status = "SOLD" + db.session.commit() + return reservation diff --git a/services/inventory/requirements.txt b/services/inventory/requirements.txt index a12dc90..565f591 100644 --- a/services/inventory/requirements.txt +++ b/services/inventory/requirements.txt @@ -7,3 +7,4 @@ python-dotenv==1.0.1 gunicorn==23.0.0 pydantic==2.10.3 opentelemetry-instrumentation-flask==0.50b0 +redis==8.1.0 diff --git a/services/inventory/tests/test_contract_provider.py b/services/inventory/tests/test_contract_provider.py new file mode 100644 index 0000000..cdc1b8c --- /dev/null +++ b/services/inventory/tests/test_contract_provider.py @@ -0,0 +1,36 @@ +"""Provider side of pattern 11: inventory replays the order service's contract against itself.""" +from pathlib import Path + +from shop_common.contract_testing import verify_provider + +PACT_DIR = Path(__file__).resolve().parents[3] / "contracts" / "pacts" +HOLD_ID = "3f0c1d2e-0000-4000-8000-000000000001" + + +def test_inventory_honours_the_order_services_contract(app): + from application import db, stock + from application.models import Reservation + from application.seed import seed_missing_stock + + seed_missing_stock() + # Pact calls the state handlers on another thread. End this thread's transaction first, or + # SQLite's write lock makes the handlers wait forever. + db.session.commit() + db.session.remove() + + def no_hold_yet() -> None: + with app.app_context(): + if db.session.get(Reservation, HOLD_ID) is not None: + stock.release(HOLD_ID) + db.session.delete(db.session.get(Reservation, HOLD_ID)) + db.session.commit() + db.session.remove() + + def hold_exists() -> None: + no_hold_yet() + with app.app_context(): + stock.reserve(HOLD_ID, 42, [{"product_id": 1, "quantity": 2}]) + db.session.remove() + + verify_provider(app, provider="inventory", consumer="order", + states={"product 1 has stock": no_hold_yet, "a stock hold exists": hold_exists}, pact_dir=PACT_DIR) diff --git a/services/inventory/tests/test_event_reader.py b/services/inventory/tests/test_event_reader.py new file mode 100644 index 0000000..af66343 --- /dev/null +++ b/services/inventory/tests/test_event_reader.py @@ -0,0 +1,67 @@ +"""The bus reader in shop_common.events, driven with a fake Redis (pattern 7). + +One event that keeps failing must not stop the events behind it, and must not be retried forever. +""" +from shop_common import events + + +class FakeRedis: + """Just enough of a consumer group: new messages, a pending list, acks, a dead-letter stream.""" + + def __init__(self, messages): + self.new = list(messages) + self.pending = {} + self.acked = [] + self.dead = [] + + def xreadgroup(self, group, consumer, streams, count, block): + start = streams[events.STREAM] + if start == "0": + batch = list(self.pending.items()) + else: + batch, self.new = self.new[:count], self.new[count:] + self.pending.update(batch) + return [(events.STREAM, batch)] if batch else [] + + def xack(self, stream, group, message_id): + self.acked.append(message_id) + self.pending.pop(message_id, None) + + def xadd(self, stream, fields, **kwargs): + assert stream == events.DEAD_LETTER_STREAM + self.dead.append(fields["event_id"]) + + +def _event(n, event_type="OrderConfirmed"): + return f"{n}-0", events.to_fields(f"evt-{n}", event_type, {}, "test", "now") + + +def test_a_failing_event_does_not_block_the_events_behind_it(): + redis = FakeRedis([_event(1, "Poison")]) + handled = [] + + def handle(event): + if event["type"] == "Poison": + raise ValueError("cannot handle") + handled.append(event["event_id"]) + + reader = events.Reader("inventory", "inventory-1", handle) + reader.read_once(redis) # the poison event fails + redis.new.append(_event(2)) # a good event arrives after it + for _ in range(4): + reader.read_once(redis) + assert handled == ["evt-2"] + + +def test_an_event_that_keeps_failing_goes_to_the_dead_letter_stream(): + redis = FakeRedis([_event(1, "Poison")]) + + def handle(event): + raise ValueError("cannot handle") + + reader = events.Reader("inventory", "inventory-1", handle) + for _ in range(2 * events.MAX_ATTEMPTS): + reader.read_once(redis) + assert redis.dead == ["evt-1"] + assert redis.acked == ["1-0"] + assert redis.pending == {} diff --git a/services/inventory/tests/test_inventory.py b/services/inventory/tests/test_inventory.py index d16d86a..d7cbc10 100644 --- a/services/inventory/tests/test_inventory.py +++ b/services/inventory/tests/test_inventory.py @@ -1,5 +1,5 @@ from application.models import StockItem -from application.seed import SAMPLE_STOCK, seed_stock_if_empty +from application.seed import SAMPLE_STOCK, seed_missing_stock def test_available_is_on_hand_minus_reserved(): @@ -7,13 +7,13 @@ def test_available_is_on_hand_minus_reserved(): def test_seed_is_idempotent(app): - seed_stock_if_empty() - seed_stock_if_empty() + seed_missing_stock() + seed_missing_stock() assert StockItem.query.count() == len(SAMPLE_STOCK) def test_get_stock_and_404(client): - seed_stock_if_empty() + seed_missing_stock() r = client.get("/api/inventory/1") assert r.status_code == 200 assert r.get_json()["result"]["on_hand"] == SAMPLE_STOCK[1] @@ -23,4 +23,4 @@ def test_get_stock_and_404(client): def test_health_db_reports_only_own_tables(client): body = client.get("/health/db").get_json() assert body["service"] == "inventory" - assert body["tables"] == ["stock_item"] + assert body["tables"] == ["consumed_event", "reservation", "stock_item"] diff --git a/services/inventory/tests/test_saga_and_events.py b/services/inventory/tests/test_saga_and_events.py new file mode 100644 index 0000000..9a844d8 --- /dev/null +++ b/services/inventory/tests/test_saga_and_events.py @@ -0,0 +1,81 @@ +"""Stock holds for the checkout saga (pattern 8) and the OrderConfirmed handler (pattern 7).""" +import pytest + +from application import consumer, db, stock +from application.models import StockItem +from application.seed import seed_missing_stock + +RID = "saga-0001-aaaa" + + +@pytest.fixture +def seeded(app): + seed_missing_stock() + return app + + +def _stock(product_id): + return db.session.get(StockItem, product_id).to_json() + + +def _reserve(client, rid=RID, qty=2): + return client.post("/api/inventory/reservations", + json={"reservation_id": rid, "order_id": 7, "items": [{"product_id": 1, "quantity": qty}]}) + + +def test_reserve_holds_stock_and_is_idempotent(seeded, client): + assert _reserve(client).status_code == 200 + assert _reserve(client).status_code == 200 # a retried request holds nothing extra + assert _stock(1) == {"product_id": 1, "on_hand": 25, "reserved": 2, "available": 23} + + +def test_reserve_is_all_or_nothing(seeded, client): + r = client.post("/api/inventory/reservations", json={"reservation_id": RID, "order_id": 7, "items": [ + {"product_id": 1, "quantity": 1}, {"product_id": 5, "quantity": 99}]}) + assert r.status_code == 409 + assert r.get_json()["short"][0]["product_id"] == 5 + assert _stock(1)["reserved"] == 0 + + +def test_release_returns_the_stock_once(seeded, client): + _reserve(client) + for _ in range(2): + assert client.post(f"/api/inventory/reservations/{RID}/release").get_json()["result"]["status"] == "RELEASED" + assert _stock(1)["reserved"] == 0 + + +def test_order_confirmed_event_finishes_the_sale_once(seeded, client): + _reserve(client, qty=3) + event = {"event_id": "evt-1", "type": "OrderConfirmed", "payload": {"reservation_id": RID, "order_id": 7}} + consumer.handle(event) + consumer.handle(event) # redelivered: must not sell twice + assert _stock(1) == {"product_id": 1, "on_hand": 22, "reserved": 0, "available": 22} + assert client.get("/api/inventory/events").get_json()["results"][0]["outcome"] == "reservation SOLD" + + +def test_released_reservation_cannot_be_sold_later(seeded, client): + _reserve(client) + stock.release(RID) + consumer.handle({"event_id": "evt-2", "type": "OrderConfirmed", "payload": {"reservation_id": RID}}) + assert _stock(1)["on_hand"] == 25 + + +def test_other_events_are_ignored(seeded): + from application.models import ConsumedEvent + + consumer.handle({"event_id": "evt-3", "type": "CheckoutFailed", "payload": {}}) + assert ConsumedEvent.query.count() == 0 + + +def test_restock_is_a_demo_switch(seeded, client, monkeypatch): + monkeypatch.delenv("CHAOS_ENABLED", raising=False) + assert client.post("/api/inventory/1/restock", json={"quantity": 5}).status_code == 403 + monkeypatch.setenv("CHAOS_ENABLED", "1") + assert client.post("/api/inventory/1/restock", json={"quantity": 5}).get_json()["result"]["on_hand"] == 30 + + +def test_restock_rejects_bad_input_with_400(seeded, client, monkeypatch): + monkeypatch.setenv("CHAOS_ENABLED", "1") + assert client.post("/api/inventory/1/restock", json={"quantity": "lots"}).status_code == 400 + assert client.post("/api/inventory/1/restock", json={"quantity": 0}).status_code == 400 + assert client.post("/api/inventory/999/restock", json={"quantity": 5}).status_code == 404 diff --git a/services/order/.dockerignore b/services/order/.dockerignore new file mode 100644 index 0000000..39daf35 --- /dev/null +++ b/services/order/.dockerignore @@ -0,0 +1,10 @@ +# Keeps the build context (what az acr build uploads) to source only. Pattern 9. +.venv/ +instance/ +tests/ +__pycache__/ +*.pyc +*.sqlite3 +*.pem +.env +.pytest_cache/ diff --git a/services/order/Dockerfile b/services/order/Dockerfile index 906945d..ebb5da6 100644 --- a/services/order/Dockerfile +++ b/services/order/Dockerfile @@ -1,19 +1,34 @@ -# Built exclusively via `az acr build` (cloud-side) — no local Docker daemon required. -FROM python:3.12-slim - -WORKDIR /app +# Pattern 9: one image per service, built only in the cloud (az acr build) or in GitHub Actions, +# always for linux/amd64 (Azure Container Apps). Never built on the owner's arm64 Mac. +# +# Stage 1 installs dependencies into a virtualenv. Stage 2 copies only that virtualenv and the +# source into a clean image and runs as an unprivileged user, so no compiler cache, pip cache or +# root process ends up in production. +FROM python:3.12-slim AS build +WORKDIR /build +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH COPY requirements.txt . -RUN pip install --no-cache-dir --root-user-action=ignore -r requirements.txt - -COPY . . -# shared/ (shop_common) is staged into ./_shared by infra/provision.py before `az acr build`. -RUN pip install --no-cache-dir --root-user-action=ignore "./_shared[flask]" +RUN pip install --no-cache-dir -r requirements.txt +# shared/ (shop_common) is staged into ./_shared before the build (infra/provision.py, CI). +COPY _shared ./_shared +RUN pip install --no-cache-dir "./_shared[flask]" -ENV RUN_MODE=azure \ +FROM python:3.12-slim +ARG GIT_SHA=unknown +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + RUN_MODE=azure \ FLASK_ENV=production \ - PYTHONUNBUFFERED=1 - + GIT_SHA=$GIT_SHA +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /opt/venv /opt/venv +COPY --chown=10001:10001 . . +RUN rm -rf _shared && mkdir -p instance && chown 10001:10001 instance +USER 10001 EXPOSE 5103 # One process with threads, so the in-memory span store (pattern 6) sees every request. diff --git a/services/order/application/__init__.py b/services/order/application/__init__.py index e512bdb..047f088 100644 --- a/services/order/application/__init__.py +++ b/services/order/application/__init__.py @@ -3,6 +3,7 @@ This service has no access to the user table at all. It learns who the caller is from the signed login token (pattern 10), prices orders by asking the catalog, and charges through the payment service behind a circuit breaker (pattern 4). See order_api/api/clients.py. +Checkout is a saga (pattern 8, saga.py) whose events leave through an outbox (pattern 7, outbox.py). """ import os @@ -33,9 +34,18 @@ def create_app(config_name: str | None = None) -> Flask: # db.create_all() runs regardless of RUN_MODE (migrations/ has no actual Alembic revisions, so gating # this to local-only left Azure MySQL with no schema at all). - db.create_all() + # Two order replicas share one database and start together (pattern 10). + from shop_common.db_setup import create_tables + + create_tables(db) # /health, /health/db, /debug/spans, /metrics, tracing and JSON logs (patterns 1 and 6). install_observability(app, "order", db) + # Pattern 7: publish outbox events to the bus in the background (no-op without REDIS_URL). + from shop_common.flask_support import instance_id + + from .outbox import start as start_outbox_relay + + start_outbox_relay(app, instance_id("order")) return app diff --git a/services/order/application/models.py b/services/order/application/models.py index 1ec9560..7eafd49 100644 --- a/services/order/application/models.py +++ b/services/order/application/models.py @@ -37,3 +37,59 @@ def __init__(self, product_id, quantity): def to_json(self): return {"product_id": self.product_id, "quantity": self.quantity} + + +class SagaStep(db.Model): + """One line of a checkout saga's log (pattern 8). A saga is all rows sharing a saga_id.""" + + __tablename__ = "saga_step" + + id = db.Column(db.Integer, primary_key=True) + saga_id = db.Column(db.String(36), nullable=False, index=True) + order_id = db.Column(db.Integer, nullable=False) + user_id = db.Column(db.Integer, nullable=False, index=True) + step = db.Column(db.String(30), nullable=False) # price, reserve_stock, charge_payment, ... + status = db.Column(db.String(12), nullable=False) # done | failed | compensated | skipped + detail = db.Column(db.String(300), nullable=True) + at = db.Column(db.DateTime, default=_utcnow) + + def to_json(self): + return {"step": self.step, "status": self.status, "detail": self.detail, + "at": self.at.isoformat() if self.at else None} + + +class OutboxEvent(db.Model): + """An event waiting to be published (pattern 7). Written in the same transaction as the change + it describes; the relay in outbox.py publishes it to Redis and stamps published_at.""" + + __tablename__ = "outbox_event" + + id = db.Column(db.String(36), primary_key=True) # also the event id consumers dedupe on + type = db.Column(db.String(40), nullable=False) + payload = db.Column(db.JSON, nullable=False) + created_at = db.Column(db.DateTime, default=_utcnow, index=True) + published_at = db.Column(db.DateTime, nullable=True, index=True) + attempts = db.Column(db.Integer, default=0, nullable=False) + last_error = db.Column(db.String(200), nullable=True) + + def to_json(self): + return { + "event_id": self.id, + "type": self.type, + "payload": self.payload, + "created_at": self.created_at.isoformat() if self.created_at else None, + "published_at": self.published_at.isoformat() if self.published_at else None, + "attempts": self.attempts, + "last_error": self.last_error, + } + + +class CheckoutLock(db.Model): + """At most one checkout saga per order at a time. The primary key makes a second insert fail, + even from the other replica, so a double click cannot hold stock or charge twice.""" + + __tablename__ = "checkout_lock" + + order_id = db.Column(db.Integer, primary_key=True, autoincrement=False) + saga_id = db.Column(db.String(36), nullable=False) + started_at = db.Column(db.DateTime, nullable=False, default=_utcnow) diff --git a/services/order/application/order_api/api/clients.py b/services/order/application/order_api/api/clients.py index 40e48ba..9be3984 100644 --- a/services/order/application/order_api/api/clients.py +++ b/services/order/application/order_api/api/clients.py @@ -21,6 +21,7 @@ "user": os.environ.get("USER_SERVICE_BASE_URL", "http://127.0.0.1:5101"), "catalog": os.environ.get("CATALOG_SERVICE_BASE_URL", "http://127.0.0.1:5102"), "payment": os.environ.get("PAYMENT_SERVICE_BASE_URL", "http://127.0.0.1:5104"), + "inventory": os.environ.get("INVENTORY_SERVICE_BASE_URL", "http://127.0.0.1:5105"), } PAYMENT_TIMEOUT_SECONDS = float(os.environ.get("PAYMENT_TIMEOUT_SECONDS", "1.0")) @@ -49,6 +50,29 @@ def product_prices() -> dict[int, int]: return {p["id"]: p["price"] for p in response.json()["results"]} +# --- Inventory (saga steps, pattern 8) ------------------------------------------------------------ + +class OutOfStock(Exception): + def __init__(self, short: list[dict]): + super().__init__("not enough stock") + self.short = short + + +def reserve_stock(reservation_id: str, order_id: int, items: list[dict]) -> dict: + response = requests.post(f"{base_url('inventory')}/api/inventory/reservations", timeout=3, + json={"reservation_id": reservation_id, "order_id": order_id, "items": items}) + if response.status_code == 409: + raise OutOfStock(response.json().get("short", [])) + response.raise_for_status() + return response.json()["result"] + + +def release_stock(reservation_id: str) -> dict: + response = requests.post(f"{base_url('inventory')}/api/inventory/reservations/{reservation_id}/release", timeout=3) + response.raise_for_status() + return response.json()["result"] + + # --- Payment, behind the circuit breaker ---------------------------------------------------------- BREAKER = pybreaker.CircuitBreaker(fail_max=FAIL_MAX, reset_timeout=RESET_SECONDS, name="payment") @@ -84,8 +108,11 @@ def attempt() -> requests.Response: raise PaymentUnavailable("failed") -def charge(order_id: int, amount_cents: int) -> dict: - response = call_payment("POST", "/api/payment/charge", {"order_id": order_id, "amount_cents": amount_cents}) +def charge(order_id: int, amount_cents: int, simulate: str = "approve") -> dict: + body = {"order_id": order_id, "amount_cents": amount_cents} + if simulate != "approve": + body["simulate"] = simulate + response = call_payment("POST", "/api/payment/charge", body) response.raise_for_status() return response.json()["result"] diff --git a/services/order/application/order_api/routes.py b/services/order/application/order_api/routes.py index bd4f33e..f4e0002 100644 --- a/services/order/application/order_api/routes.py +++ b/services/order/application/order_api/routes.py @@ -1,11 +1,12 @@ import time -import requests from flask import jsonify, make_response, request from . import order_api_blueprint -from .. import db -from ..models import Order, OrderItem +from shop_common.flask_support import chaos_enabled + +from .. import db, outbox, saga +from ..models import Order, OrderItem, OutboxEvent from ..schemas import OrderAddItemRequest from ..validation import validate_form from .api import clients @@ -73,30 +74,40 @@ def checkout(): if order_model is None: return make_response(jsonify({"message": "No open order"}), 404) - try: - prices = clients.product_prices() - except requests.RequestException: - return make_response(jsonify({"message": "The catalog is unavailable, so the order cannot be priced. Your cart is kept."}), 503) - missing = [i.product_id for i in order_model.items if i.product_id not in prices] - if missing: - return make_response(jsonify({"message": f"Product(s) {missing} are no longer sold"}), 409) - amount = sum(prices[i.product_id] * i.quantity for i in order_model.items) + # Pattern 8 demo: {"simulate_decline": true} makes payment refuse the charge. + options = request.get_json(silent=True) or request.form + simulate = "decline" if str(options.get("simulate_decline", "")).lower() in ("1", "true", "yes") else "approve" + if simulate == "decline" and not chaos_enabled(): + return make_response(jsonify({"message": "Demo switches are off (CHAOS_ENABLED is not 1)"}), 403) + status, body = saga.run_checkout(order_model, u_id, simulate) + return make_response(jsonify(body), status) - try: - payment = clients.charge(order_model.id, amount) - except clients.PaymentUnavailable as exc: - state = clients.breaker_state() - wait = f" Try again in {state['retry_in_seconds']:.0f} s." if state["retry_in_seconds"] else "" - return make_response(jsonify({ - "message": f"Payment is unavailable right now, so nothing was charged. Your cart is kept.{wait}", - "reason": exc.reason, - "breaker": state, - }), 503) - - order_model.is_open = False - db.session.add(order_model) - db.session.commit() - return jsonify({"result": order_model.to_json(), "payment": payment}) + +@order_api_blueprint.route("/api/order/sagas", methods=["GET"]) +def sagas(): + u_id = _user_id() + if u_id is None: + return _not_logged_in() + return jsonify({"results": saga.recent_sagas(u_id)}) + + +# Pattern 7 demo: what is waiting in the outbox and what has been published. No personal data: +# events carry ids, quantities and amounts only. +@order_api_blueprint.route("/api/order/outbox", methods=["GET"]) +def outbox_events(): + rows = OutboxEvent.query.order_by(OutboxEvent.created_at.desc()).limit(20).all() + pending = OutboxEvent.query.filter(OutboxEvent.published_at.is_(None)).count() + return jsonify({"pending": pending, "results": [r.to_json() for r in rows], + "bus_connected": outbox.is_connected()}) + + +@order_api_blueprint.route("/debug/bus", methods=["POST"]) +def bus_switch(): + if not chaos_enabled(): + return make_response(jsonify({"message": "Demo switches are off (CHAOS_ENABLED is not 1)"}), 403) + connected = bool((request.get_json(silent=True) or {}).get("connected", True)) + outbox.set_connected(connected) + return jsonify({"bus_connected": connected}) # Pattern 4 demo: one cheap call to payment through the same breaker that protects checkout. diff --git a/services/order/application/outbox.py b/services/order/application/outbox.py new file mode 100644 index 0000000..ab56c46 --- /dev/null +++ b/services/order/application/outbox.py @@ -0,0 +1,73 @@ +"""The outbox relay (pattern 7): publishes saved events to the Redis bus, oldest first. + +Runs on a background thread in every order replica, once a second. Both replicas read the same +outbox table, so an event can occasionally be published twice; consumers dedupe on event_id +(at-least-once delivery). If Redis is down, events stay in the table, unpublished, and go out +once it is back. Nothing is lost and checkout never waits for the bus. + +Demo switch (CHAOS_ENABLED=1 only): disconnect() makes this replica's relay behave as if the bus +were down, so the page can show events piling up and then draining. +""" +import threading +import time +from datetime import datetime, timezone + +from shop_common import events + +from . import db +from .models import OutboxEvent + +BATCH = 50 +_bus = {"connected": True} + + +def set_connected(connected: bool) -> None: + _bus["connected"] = connected + + +def is_connected() -> bool: + return _bus["connected"] + + +def publish_pending(client, source: str) -> int: + """Publishes up to BATCH pending events. Returns how many went out.""" + pending = (OutboxEvent.query.filter(OutboxEvent.published_at.is_(None)) + .order_by(OutboxEvent.created_at).limit(BATCH).all()) + sent = 0 + for event in pending: + try: + if client is None or not _bus["connected"]: + raise ConnectionError("event bus unreachable" if client else "no REDIS_URL configured") + fields = events.to_fields(event.id, event.type, event.payload, source, event.created_at.isoformat()) + client.xadd(events.STREAM, fields, maxlen=10_000, approximate=True) + event.published_at = datetime.now(timezone.utc) + event.last_error = None + sent += 1 + except Exception as exc: # the bus is down: stop this pass, keep the events for later + event.attempts += 1 + event.last_error = str(exc)[:200] + break + db.session.commit() + return sent + + +def start(app, source: str) -> None: + from shop_common.flask_support import is_serving_process + + if not events.redis_url() or not is_serving_process(app): + return # events wait in the table; unit tests, the reloader's watcher and no-Redis land here + + def loop() -> None: + client = None + while True: + time.sleep(1) + with app.app_context(): + try: + client = client or events.connect() + publish_pending(client, source) + except Exception: + client = None + finally: + db.session.remove() + + threading.Thread(target=loop, name="outbox-relay", daemon=True).start() diff --git a/services/order/application/saga.py b/services/order/application/saga.py new file mode 100644 index 0000000..f098815 --- /dev/null +++ b/services/order/application/saga.py @@ -0,0 +1,182 @@ +"""The checkout saga (pattern 8), orchestrated by the order service. + + 1. price ask catalog for current prices + 2. reserve_stock inventory holds the units event StockReserved; undo: release_stock + 3. charge_payment payment charges the card event PaymentCharged + 4. confirm close the order and, in the SAME transaction, write OrderConfirmed to the + outbox. Inventory turns the hold into a sale when it reads that event (7). + Any failure: event CheckoutFailed. One checkout per order at a time (checkout_lock). + +If a later step fails, the earlier steps are undone in reverse order. Only step 2 changes another +service's data, so the only compensation is releasing the stock. If that direct call fails too, +a StockReleaseRequested event goes to the outbox, so inventory releases it once it can. +The cart stays open after a failed saga, so the shopper can try again. + +Every step is written to saga_step, which is what the Saga page draws. +""" +import uuid +from datetime import datetime, timedelta, timezone + +import requests +from sqlalchemy.exc import IntegrityError + +from . import db +from .models import CheckoutLock, Order, OutboxEvent, SagaStep +from .order_api.api import clients + + +class Saga: + def __init__(self, order: Order, user_id: int): + self.id = str(uuid.uuid4()) + self.order = order + self.user_id = user_id + + def emit(self, event_type: str, payload: dict | None = None) -> None: + """Adds an event to the outbox. It is committed together with the next log() line.""" + db.session.add(OutboxEvent(id=str(uuid.uuid4()), type=event_type, + payload={"saga_id": self.id, "order_id": self.order.id, **(payload or {})})) + + def log(self, step: str, status: str, detail: str | None = None, event: str | None = None, + payload: dict | None = None) -> None: + """Records a step and, in the same transaction, its event and any pending order change.""" + db.session.add(SagaStep(saga_id=self.id, order_id=self.order.id, user_id=self.user_id, + step=step, status=status, detail=(detail or "")[:300])) + if event: + self.emit(event, payload) + db.session.commit() + + def fail(self, step: str, detail: str, http_status: int, message: str, compensate: bool = False, + **extra) -> tuple[int, dict]: + """Logs the failed step, undoes the stock hold if asked, and reports CheckoutFailed.""" + self.log(step, "failed", detail) + if compensate: + _compensate(self) + self.emit("CheckoutFailed", {"failed_step": step, "reason": detail}) + db.session.commit() + return http_status, {"message": message, "saga_id": self.id, "failed_step": step, **extra} + + +# A lock older than this was left by a replica that crashed mid-saga; the next checkout may take it. +LOCK_TIMEOUT = timedelta(seconds=60) + + +def _acquire_lock(order: Order, saga_id: str) -> bool: + now = datetime.now(timezone.utc) + existing = db.session.get(CheckoutLock, order.id) + if existing is not None: + started = existing.started_at.replace(tzinfo=existing.started_at.tzinfo or timezone.utc) + if now - started < LOCK_TIMEOUT: + return False + db.session.delete(existing) + db.session.flush() + db.session.add(CheckoutLock(order_id=order.id, saga_id=saga_id, started_at=now)) + try: + db.session.commit() + return True + except IntegrityError: # the other click (or replica) got there first + db.session.rollback() + return False + + +def _release_lock(order_id: int) -> None: + lock = db.session.get(CheckoutLock, order_id) + if lock is not None: + db.session.delete(lock) + db.session.commit() + + +def run_checkout(order: Order, user_id: int, simulate: str = "approve") -> tuple[int, dict]: + saga = Saga(order, user_id) + if not _acquire_lock(order, saga.id): + return 409, {"message": "A checkout for this order is already in progress."} + try: + return _run_steps(saga, simulate) + finally: + _release_lock(order.id) + + +def _run_steps(saga: Saga, simulate: str) -> tuple[int, dict]: + order = saga.order + items = [{"product_id": i.product_id, "quantity": i.quantity} for i in order.items] + kept = "Your cart is kept." + + # 1. price + try: + prices = clients.product_prices() + except requests.RequestException as exc: + return saga.fail("price", f"catalog unreachable: {exc.__class__.__name__}", 503, + f"The catalog is unavailable, so the order cannot be priced. {kept}") + missing = [i["product_id"] for i in items if i["product_id"] not in prices] + if missing: + return saga.fail("price", f"unknown products {missing}", 409, f"Product(s) {missing} are no longer sold") + amount = sum(prices[i["product_id"]] * i["quantity"] for i in items) + saga.log("price", "done", f"{len(items)} line(s), total {amount} cents") + + # 2. reserve stock + try: + clients.reserve_stock(saga.id, order.id, items) + except clients.OutOfStock as exc: + return saga.fail("reserve_stock", f"short: {exc.short}", 409, + f"Not enough stock for your cart. Nothing was charged. {kept}", short=exc.short) + except requests.RequestException as exc: + # No answer does not mean no hold: inventory may have held the stock before the reply was + # lost. Ask it to release through the bus; releasing an unknown hold is a no-op. + saga.emit("StockReleaseRequested", {"reservation_id": saga.id}) + return saga.fail("reserve_stock", f"inventory unreachable: {exc.__class__.__name__}", 503, + f"Inventory is unavailable. Nothing was charged. {kept}") + units = sum(i["quantity"] for i in items) + saga.log("reserve_stock", "done", f"held {units} unit(s)", event="StockReserved", + payload={"reservation_id": saga.id, "items": items}) + + # 3. charge payment (behind the M2 circuit breaker) + try: + payment = clients.charge(order.id, amount, simulate) + except clients.PaymentUnavailable as exc: + breaker = clients.breaker_state() + if exc.reason == "open": # the breaker refused the call: payment was never contacted + wait = f" Try again in {breaker['retry_in_seconds']:.0f} s." if breaker["retry_in_seconds"] else "" + message = f"Payment is unavailable right now, so nothing was charged; the stock was released. {kept}{wait}" + else: # timeout or 5xx: the charge may or may not have happened + message = f"Payment could not be confirmed, so the order was not placed and the stock was released. {kept}" + return saga.fail("charge_payment", f"payment unavailable ({exc.reason})", 503, message, + compensate=True, breaker=breaker) + except requests.RequestException as exc: + return saga.fail("charge_payment", f"payment error: {exc.__class__.__name__}", 502, + f"Payment could not be confirmed, so the order was not placed and the stock was released. {kept}", + compensate=True) + if payment["status"] != "APPROVED": + return saga.fail("charge_payment", "declined", 402, + f"Your payment was declined. Nothing was charged; the stock was released. {kept}", + compensate=True) + saga.log("charge_payment", "done", f"payment {payment['id']} APPROVED, {amount} cents", event="PaymentCharged", + payload={"payment_id": payment["id"], "amount_cents": amount}) + + # 4. confirm: the order change, its log line and its event commit together, or not at all + order.is_open = False + saga.log("confirm", "done", "order closed; OrderConfirmed in the outbox", event="OrderConfirmed", + payload={"reservation_id": saga.id, "amount_cents": amount, "items": items}) + return 200, {"result": order.to_json(), "payment": payment, "saga_id": saga.id} + + +def _compensate(saga: Saga) -> None: + try: + clients.release_stock(saga.id) + saga.log("release_stock", "compensated", "stock returned to inventory") + except requests.RequestException as exc: + # Could not undo directly: ask inventory through the bus. It is idempotent, so a release + # that did go through and this event together are harmless. + saga.log("release_stock", "compensated", f"inventory unreachable ({exc.__class__.__name__}); release sent as an event", + event="StockReleaseRequested", payload={"reservation_id": saga.id}) + + +def recent_sagas(user_id: int, limit: int = 5) -> list[dict]: + saga_ids = [row.saga_id for row in db.session.query(SagaStep.saga_id, db.func.max(SagaStep.id).label("last")) + .filter_by(user_id=user_id).group_by(SagaStep.saga_id).order_by(db.desc("last")).limit(limit)] + result = [] + for saga_id in saga_ids: + steps = SagaStep.query.filter_by(saga_id=saga_id).order_by(SagaStep.id).all() + statuses = {s.status for s in steps} + outcome = "compensated" if "compensated" in statuses else "failed" if "failed" in statuses else "completed" + result.append({"saga_id": saga_id, "order_id": steps[0].order_id, "outcome": outcome, + "steps": [s.to_json() for s in steps]}) + return result diff --git a/services/order/requirements.txt b/services/order/requirements.txt index 55ce5bc..f3b24dc 100644 --- a/services/order/requirements.txt +++ b/services/order/requirements.txt @@ -10,3 +10,4 @@ pydantic==2.10.3 opentelemetry-instrumentation-flask==0.50b0 pybreaker==1.4.1 tenacity==9.2.1 +redis==8.1.0 diff --git a/services/order/tests/test_checkout_and_breaker.py b/services/order/tests/test_checkout_and_breaker.py index e3ad073..fd485ae 100644 --- a/services/order/tests/test_checkout_and_breaker.py +++ b/services/order/tests/test_checkout_and_breaker.py @@ -17,6 +17,9 @@ def _isolate(monkeypatch): monkeypatch.setattr(clients, "current_user_id", lambda header: 1 if header else None) monkeypatch.setitem(clients.STATIC_URLS, "payment", "http://127.0.0.1:9") # nothing listens on :9 monkeypatch.setattr(clients, "product_prices", lambda: {1: 1299, 2: 899}) + # Inventory is not running in unit tests: holds always succeed unless a test says otherwise. + monkeypatch.setattr(clients, "reserve_stock", lambda rid, order_id, items: {"status": "RESERVED"}) + monkeypatch.setattr(clients, "release_stock", lambda rid: {"status": "RELEASED"}) clients.BREAKER.close() yield clients.BREAKER.close() @@ -54,7 +57,7 @@ def test_checkout_keeps_the_cart_when_payment_is_down(client): def test_checkout_charges_the_catalog_price_and_closes_the_order(client, monkeypatch): charged = {} - def fake_charge(order_id, amount_cents): + def fake_charge(order_id, amount_cents, simulate="approve"): charged.update(order_id=order_id, amount_cents=amount_cents) return {"id": 1, "order_id": order_id, "amount_cents": amount_cents, "status": "APPROVED"} @@ -83,3 +86,19 @@ def down(): monkeypatch.setattr(clients, "product_prices", down) _cart(client) assert client.post("/api/order/checkout", headers=AUTH).status_code == 503 + + +def test_a_payment_timeout_never_claims_nothing_was_charged(client): + """Payment at :9 refuses the call, which looks like a timeout: the charge state is unknown.""" + _cart(client) + message = client.post("/api/order/checkout", headers=AUTH).get_json()["message"] + assert "could not be confirmed" in message + assert "nothing was charged" not in message + + +def test_an_open_breaker_does_say_nothing_was_charged(client): + _cart(client) + for _ in range(clients.FAIL_MAX): + client.get("/api/order/payment-check") + message = client.post("/api/order/checkout", headers=AUTH).get_json()["message"] + assert "nothing was charged" in message diff --git a/services/order/tests/test_contracts_consumer.py b/services/order/tests/test_contracts_consumer.py index e8a9c13..1159a16 100644 --- a/services/order/tests/test_contracts_consumer.py +++ b/services/order/tests/test_contracts_consumer.py @@ -37,6 +37,14 @@ def test_order_to_payment_contract(monkeypatch): "status": "APPROVED", }}, "application/json") ) + ( + pact.upon_receiving("a charge the bank declines") + .given("payment is healthy") + .with_request("POST", "/api/payment/charge") + .with_body({"order_id": match.int(43), "amount_cents": match.int(899), "simulate": "decline"}, "application/json") + .will_respond_with(200) + .with_body({"result": {"id": match.int(2), "status": "DECLINED"}}, "application/json") + ) ( pact.upon_receiving("a health check through the circuit breaker") .given("payment is healthy") @@ -49,6 +57,7 @@ def test_order_to_payment_contract(monkeypatch): payment = clients.charge(42, 2598) assert payment["status"] == "APPROVED" assert payment["amount_cents"] == 2598 + assert clients.charge(43, 899, "decline")["status"] == "DECLINED" assert clients.call_payment("GET", "/api/payment/ping").json() == {"status": "ok"} pact.write_file(PACT_DIR, overwrite=True) @@ -66,3 +75,43 @@ def test_order_to_catalog_contract(monkeypatch): monkeypatch.setitem(clients.STATIC_URLS, "catalog", str(server.url).rstrip("/")) assert clients.product_prices() == {1: 1299} pact.write_file(PACT_DIR, overwrite=True) + + +def test_order_to_inventory_contract(monkeypatch): + pact = Pact("order", "inventory").with_specification("V4") + rid = "3f0c1d2e-0000-4000-8000-000000000001" + ( + pact.upon_receiving("a stock hold for a checkout") + .given("product 1 has stock") + .with_request("POST", "/api/inventory/reservations") + # Exact request bodies (no matchers): the mock tells this request and the next one apart + # by their bodies, since both are POSTs to the same path. + .with_body({"reservation_id": rid, "order_id": 42, "items": [{"product_id": 1, "quantity": 2}]}, + "application/json") + .will_respond_with(200) + .with_body({"result": {"reservation_id": match.str(rid), "status": "RESERVED"}}, "application/json") + ) + ( + pact.upon_receiving("a stock hold that is more than inventory has") + .given("product 1 has stock") + .with_request("POST", "/api/inventory/reservations") + .with_body({"reservation_id": "3f0c1d2e-0000-4000-8000-000000000002", "order_id": 43, + "items": [{"product_id": 1, "quantity": 100000}]}, "application/json") + .will_respond_with(409) + .with_body({"short": match.each_like({"product_id": match.int(1), "available": match.int(25)}, min=1)}, + "application/json") + ) + ( + pact.upon_receiving("a release of a stock hold") + .given("a stock hold exists", reservation_id=rid) + .with_request("POST", f"/api/inventory/reservations/{rid}/release") + .will_respond_with(200) + .with_body({"result": {"reservation_id": rid, "status": "RELEASED"}}, "application/json") + ) + with pact.serve() as server: + monkeypatch.setitem(clients.STATIC_URLS, "inventory", str(server.url).rstrip("/")) + assert clients.reserve_stock(rid, 42, [{"product_id": 1, "quantity": 2}])["status"] == "RESERVED" + with pytest.raises(clients.OutOfStock): + clients.reserve_stock("3f0c1d2e-0000-4000-8000-000000000002", 43, [{"product_id": 1, "quantity": 100000}]) + assert clients.release_stock(rid)["status"] == "RELEASED" + pact.write_file(PACT_DIR, overwrite=True) diff --git a/services/order/tests/test_saga_and_outbox.py b/services/order/tests/test_saga_and_outbox.py new file mode 100644 index 0000000..2a546d7 --- /dev/null +++ b/services/order/tests/test_saga_and_outbox.py @@ -0,0 +1,163 @@ +"""The checkout saga (pattern 8) and the outbox relay (pattern 7), with every other service stubbed.""" +import pytest +import requests + +from application import outbox +from application.models import OutboxEvent, SagaStep +from application.order_api.api import clients + +AUTH = {"Authorization": "Bearer test"} + + +@pytest.fixture +def calls(monkeypatch): + log = [] + monkeypatch.setenv("CHAOS_ENABLED", "1") # the decline demo switch, as under make run + monkeypatch.setattr(clients, "current_user_id", lambda header: 1 if header else None) + monkeypatch.setattr(clients, "product_prices", lambda: {1: 1299}) + monkeypatch.setattr(clients, "reserve_stock", lambda rid, oid, items: log.append(("reserve", rid)) or {}) + monkeypatch.setattr(clients, "release_stock", lambda rid: log.append(("release", rid)) or {}) + monkeypatch.setattr(clients, "charge", lambda oid, amount, simulate="approve": log.append(("charge", simulate)) or { + "id": 9, "status": "DECLINED" if simulate == "decline" else "APPROVED"}) + clients.BREAKER.close() + return log + + +def _checkout(client, **body): + client.post("/api/order/add-item", data={"product_id": 1, "qty": 2}, headers=AUTH) + return client.post("/api/order/checkout", json=body, headers=AUTH) + + +def _steps(): + return [(s.step, s.status) for s in SagaStep.query.order_by(SagaStep.id)] + + +def _events(): + return [e.type for e in OutboxEvent.query.order_by(OutboxEvent.created_at)] + + +def test_happy_path_confirms_and_queues_order_confirmed(client, calls): + r = _checkout(client) + assert r.status_code == 200 + assert _steps() == [("price", "done"), ("reserve_stock", "done"), ("charge_payment", "done"), ("confirm", "done")] + assert _events() == ["StockReserved", "PaymentCharged", "OrderConfirmed"] # one event per step + event = OutboxEvent.query.filter_by(type="OrderConfirmed").one() + assert event.payload["reservation_id"] == r.get_json()["saga_id"] # inventory sells exactly this hold + assert event.published_at is None # nothing published inside the request + + +def test_declined_payment_releases_the_stock_and_keeps_the_cart(client, calls): + r = _checkout(client, simulate_decline=True) + assert r.status_code == 402 + assert [c[0] for c in calls] == ["reserve", "charge", "release"] + assert _steps()[-2:] == [("charge_payment", "failed"), ("release_stock", "compensated")] + assert _events() == ["StockReserved", "CheckoutFailed"] + assert client.get("/api/order", headers=AUTH).get_json()["result"]["is_open"] is True + + +def test_out_of_stock_stops_before_payment(client, calls, monkeypatch): + def short(rid, oid, items): + raise clients.OutOfStock([{"product_id": 1, "wanted": 2, "available": 0}]) + + monkeypatch.setattr(clients, "reserve_stock", short) + r = _checkout(client) + assert r.status_code == 409 + assert not any(c[0] == "charge" for c in calls) + + +def test_release_falls_back_to_an_event_when_inventory_is_down(client, calls, monkeypatch): + def down(rid): + raise requests.ConnectionError("inventory down") + + monkeypatch.setattr(clients, "release_stock", down) + _checkout(client, simulate_decline=True) + assert _events() == ["StockReserved", "StockReleaseRequested", "CheckoutFailed"] + + +def test_saga_history_is_per_user(client, calls): + _checkout(client, simulate_decline=True) + sagas = client.get("/api/order/sagas", headers=AUTH).get_json()["results"] + assert sagas[0]["outcome"] == "compensated" + + +class FakeBus: + def __init__(self): + self.sent = [] + + def xadd(self, stream, fields, **kwargs): + self.sent.append(fields["type"]) + + +def test_relay_publishes_pending_events_once(client, calls): + _checkout(client) + bus = FakeBus() + assert outbox.publish_pending(bus, "order-test") == 3 + assert outbox.publish_pending(bus, "order-test") == 0 + assert bus.sent == ["StockReserved", "PaymentCharged", "OrderConfirmed"] + + +def test_events_wait_while_the_bus_is_down(client, calls): + _checkout(client) + outbox.set_connected(False) + try: + assert outbox.publish_pending(FakeBus(), "order-test") == 0 + assert client.get("/api/order/outbox").get_json()["pending"] == 3 + finally: + outbox.set_connected(True) + assert outbox.publish_pending(FakeBus(), "order-test") == 3 + + +def test_a_second_checkout_while_one_is_running_is_refused(client, calls): + from datetime import datetime, timezone + + from application import db + from application.models import CheckoutLock, Order + + client.post("/api/order/add-item", data={"product_id": 1, "qty": 2}, headers=AUTH) + order = Order.query.one() + # The first click's saga is still running: it holds the order's checkout lock. + db.session.add(CheckoutLock(order_id=order.id, saga_id="first-click", started_at=datetime.now(timezone.utc))) + db.session.commit() + r = client.post("/api/order/checkout", json={}, headers=AUTH) + assert r.status_code == 409 + assert "already in progress" in r.get_json()["message"] + assert calls == [] # no hold, no charge + + +def test_the_lock_is_released_when_a_saga_ends(client, calls, monkeypatch): + monkeypatch.setenv("CHAOS_ENABLED", "1") + assert _checkout(client, simulate_decline=True).status_code == 402 + assert client.post("/api/order/checkout", json={}, headers=AUTH).status_code == 200 + + +def test_a_lock_left_by_a_crashed_replica_expires(client, calls): + from datetime import datetime, timedelta, timezone + + from application import db + from application.models import CheckoutLock, Order + + client.post("/api/order/add-item", data={"product_id": 1, "qty": 2}, headers=AUTH) + order = Order.query.one() + stale = datetime.now(timezone.utc) - timedelta(minutes=5) + db.session.add(CheckoutLock(order_id=order.id, saga_id="crashed", started_at=stale)) + db.session.commit() + assert client.post("/api/order/checkout", json={}, headers=AUTH).status_code == 200 + + +def test_the_decline_switch_is_off_unless_demo_switches_are_on(client, calls, monkeypatch): + monkeypatch.delenv("CHAOS_ENABLED") + r = _checkout(client, simulate_decline=True) + assert r.status_code == 403 + assert calls == [] + + +def test_a_timed_out_hold_is_released_through_the_bus(client, calls, monkeypatch): + """Inventory may have held the stock before the answer was lost: ask it to release, idempotently.""" + def timeout(rid, oid, items): + raise requests.Timeout("inventory slow") + + monkeypatch.setattr(clients, "reserve_stock", timeout) + r = _checkout(client) + assert r.status_code == 503 + assert _events() == ["StockReleaseRequested", "CheckoutFailed"] + assert OutboxEvent.query.filter_by(type="StockReleaseRequested").one().payload["reservation_id"] == r.get_json()["saga_id"] diff --git a/services/order/tests/test_startup.py b/services/order/tests/test_startup.py new file mode 100644 index 0000000..71519f8 --- /dev/null +++ b/services/order/tests/test_startup.py @@ -0,0 +1,64 @@ +"""Two order replicas start at the same moment and share one database (pattern 10).""" +import pytest +from sqlalchemy.exc import OperationalError + + +def test_replica_survives_another_replica_creating_the_tables_first(app, monkeypatch): + from application import db + from shop_common.db_setup import create_tables + + real_create_all = db.create_all + calls = [] + + def racing_create_all(): + calls.append(1) + if len(calls) == 1: # the other replica won the race between "check" and "create" + raise OperationalError("CREATE TABLE saga_step", {}, Exception("table saga_step already exists")) + real_create_all() + + monkeypatch.setattr(db, "create_all", racing_create_all) + create_tables(db) + assert len(calls) == 2 + + +def test_other_database_errors_still_stop_startup(app, monkeypatch): + from application import db + from shop_common.db_setup import create_tables + + def broken(): + raise OperationalError("CREATE TABLE x", {}, Exception("disk I/O error")) + + monkeypatch.setattr(db, "create_all", broken) + with pytest.raises(OperationalError): + create_tables(db) + + +def _relay_threads(): + import threading + + return [t for t in threading.enumerate() if t.name == "outbox-relay"] + + +def test_reloader_watcher_process_does_not_publish_events(app, monkeypatch): + """With app.run(debug=True), Flask runs a watcher process plus a worker process, and both build + the app. Only the worker serves requests, so only it may publish, or the bus switch (which + reaches the worker) would not stop publishing.""" + from application import outbox + + monkeypatch.setenv("REDIS_URL", "redis://127.0.0.1:1/0") + monkeypatch.delenv("WERKZEUG_RUN_MAIN", raising=False) + app.debug = True + before = len(_relay_threads()) + outbox.start(app, "order-test") + assert len(_relay_threads()) == before + + +def test_worker_process_publishes_events(app, monkeypatch): + from application import outbox + + monkeypatch.setenv("REDIS_URL", "redis://127.0.0.1:1/0") + monkeypatch.setenv("WERKZEUG_RUN_MAIN", "true") + app.debug = True + before = len(_relay_threads()) + outbox.start(app, "order-test") + assert len(_relay_threads()) == before + 1 diff --git a/services/payment/.dockerignore b/services/payment/.dockerignore new file mode 100644 index 0000000..39daf35 --- /dev/null +++ b/services/payment/.dockerignore @@ -0,0 +1,10 @@ +# Keeps the build context (what az acr build uploads) to source only. Pattern 9. +.venv/ +instance/ +tests/ +__pycache__/ +*.pyc +*.sqlite3 +*.pem +.env +.pytest_cache/ diff --git a/services/payment/Dockerfile b/services/payment/Dockerfile index a4aec98..e4fadce 100644 --- a/services/payment/Dockerfile +++ b/services/payment/Dockerfile @@ -1,19 +1,34 @@ -# Built exclusively via `az acr build` (cloud-side) — no local Docker daemon required. -FROM python:3.12-slim - -WORKDIR /app +# Pattern 9: one image per service, built only in the cloud (az acr build) or in GitHub Actions, +# always for linux/amd64 (Azure Container Apps). Never built on the owner's arm64 Mac. +# +# Stage 1 installs dependencies into a virtualenv. Stage 2 copies only that virtualenv and the +# source into a clean image and runs as an unprivileged user, so no compiler cache, pip cache or +# root process ends up in production. +FROM python:3.12-slim AS build +WORKDIR /build +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH COPY requirements.txt . -RUN pip install --no-cache-dir --root-user-action=ignore -r requirements.txt - -COPY . . -# shared/ (shop_common) is staged into ./_shared by infra/provision.py before `az acr build`. -RUN pip install --no-cache-dir --root-user-action=ignore "./_shared[flask]" +RUN pip install --no-cache-dir -r requirements.txt +# shared/ (shop_common) is staged into ./_shared before the build (infra/provision.py, CI). +COPY _shared ./_shared +RUN pip install --no-cache-dir "./_shared[flask]" -ENV RUN_MODE=azure \ +FROM python:3.12-slim +ARG GIT_SHA=unknown +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + RUN_MODE=azure \ FLASK_ENV=production \ - PYTHONUNBUFFERED=1 - + GIT_SHA=$GIT_SHA +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /opt/venv /opt/venv +COPY --chown=10001:10001 . . +RUN rm -rf _shared && mkdir -p instance && chown 10001:10001 instance +USER 10001 EXPOSE 5104 # One process with threads, so the in-memory span store (pattern 6) sees every request. diff --git a/services/payment/application/routes.py b/services/payment/application/routes.py index ce6b9b0..8245e57 100644 --- a/services/payment/application/routes.py +++ b/services/payment/application/routes.py @@ -1,4 +1,5 @@ import time +from typing import Literal from flask import Blueprint, jsonify, make_response, request from pydantic import BaseModel, Field, ValidationError @@ -19,6 +20,9 @@ class ChargeRequest(BaseModel): order_id: int = Field(gt=0) amount_cents: int = Field(gt=0) + # Pattern 8 demo: "decline" acts like a card the bank refuses. A decline is a normal business + # answer (200, status DECLINED), not a failure, so it never trips the order service's breaker. + simulate: Literal["approve", "decline"] = "approve" @payment_api.before_request @@ -49,7 +53,8 @@ def charge(): data = ChargeRequest.model_validate(raw) except ValidationError as exc: return make_response(jsonify({"message": "Invalid request", "errors": exc.errors(include_url=False)}), 400) - payment = Payment(order_id=data.order_id, amount_cents=data.amount_cents, status="APPROVED") + status = "DECLINED" if data.simulate == "decline" else "APPROVED" + payment = Payment(order_id=data.order_id, amount_cents=data.amount_cents, status=status) db.session.add(payment) db.session.commit() return jsonify({"result": payment.to_json()}) diff --git a/services/payment/tests/test_payment.py b/services/payment/tests/test_payment.py index 225526b..e38febd 100644 --- a/services/payment/tests/test_payment.py +++ b/services/payment/tests/test_payment.py @@ -32,3 +32,9 @@ def test_failing_mode_returns_500_until_healed(client, monkeypatch): def test_chaos_switch_is_off_unless_enabled(client, monkeypatch): monkeypatch.delenv("CHAOS_ENABLED", raising=False) assert client.post("/api/payment/chaos", json={"mode": "failing"}).status_code == 403 + + +def test_a_simulated_decline_is_a_normal_answer(client): + r = client.post("/api/payment/charge", json={"order_id": 4, "amount_cents": 900, "simulate": "decline"}) + assert r.status_code == 200 + assert r.get_json()["result"]["status"] == "DECLINED" diff --git a/services/user/.dockerignore b/services/user/.dockerignore new file mode 100644 index 0000000..39daf35 --- /dev/null +++ b/services/user/.dockerignore @@ -0,0 +1,10 @@ +# Keeps the build context (what az acr build uploads) to source only. Pattern 9. +.venv/ +instance/ +tests/ +__pycache__/ +*.pyc +*.sqlite3 +*.pem +.env +.pytest_cache/ diff --git a/services/user/Dockerfile b/services/user/Dockerfile index a5ea4ec..1848427 100644 --- a/services/user/Dockerfile +++ b/services/user/Dockerfile @@ -1,19 +1,34 @@ -# Built exclusively via `az acr build` (cloud-side) — no local Docker daemon required. -FROM python:3.12-slim - -WORKDIR /app +# Pattern 9: one image per service, built only in the cloud (az acr build) or in GitHub Actions, +# always for linux/amd64 (Azure Container Apps). Never built on the owner's arm64 Mac. +# +# Stage 1 installs dependencies into a virtualenv. Stage 2 copies only that virtualenv and the +# source into a clean image and runs as an unprivileged user, so no compiler cache, pip cache or +# root process ends up in production. +FROM python:3.12-slim AS build +WORKDIR /build +RUN python -m venv /opt/venv +ENV PATH=/opt/venv/bin:$PATH COPY requirements.txt . -RUN pip install --no-cache-dir --root-user-action=ignore -r requirements.txt - -COPY . . -# shared/ (shop_common) is staged into ./_shared by infra/provision.py before `az acr build`. -RUN pip install --no-cache-dir --root-user-action=ignore "./_shared[flask]" +RUN pip install --no-cache-dir -r requirements.txt +# shared/ (shop_common) is staged into ./_shared before the build (infra/provision.py, CI). +COPY _shared ./_shared +RUN pip install --no-cache-dir "./_shared[flask]" -ENV RUN_MODE=azure \ +FROM python:3.12-slim +ARG GIT_SHA=unknown +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + RUN_MODE=azure \ FLASK_ENV=production \ - PYTHONUNBUFFERED=1 - + GIT_SHA=$GIT_SHA +RUN useradd --uid 10001 --no-create-home --shell /usr/sbin/nologin app +WORKDIR /app +COPY --from=build /opt/venv /opt/venv +COPY --chown=10001:10001 . . +RUN rm -rf _shared && mkdir -p instance && chown 10001:10001 instance +USER 10001 EXPOSE 5101 # One process with threads, so the in-memory span store (pattern 6) sees every request. diff --git a/shared/shop_common/db_setup.py b/shared/shop_common/db_setup.py new file mode 100644 index 0000000..9c8cdcb --- /dev/null +++ b/shared/shop_common/db_setup.py @@ -0,0 +1,18 @@ +"""Creating a service's tables at startup, safely when several replicas start at once.""" +from sqlalchemy.exc import OperationalError + + +def create_tables(db) -> None: + """db.create_all(), tolerating another replica creating the same table at the same moment. + + create_all() checks which tables exist, then creates the missing ones. Two replicas sharing a + database can both pass the check, and the slower one then fails with "already exists". On that + error only, run it once more: the check now sees the table and skips it. + """ + try: + db.create_all() + except OperationalError as exc: + if "already exists" not in str(exc): + raise + db.session.rollback() + db.create_all() diff --git a/shared/shop_common/events.py b/shared/shop_common/events.py new file mode 100644 index 0000000..4b82e03 --- /dev/null +++ b/shared/shop_common/events.py @@ -0,0 +1,114 @@ +"""The event bus (pattern 7): one Redis Stream, `shop.events`, shared by every service. + +- Publishers never write to Redis inside a request. They save the event in their own database's + outbox table in the same transaction as the business change, and a relay publishes it later + (see services/order/application/outbox.py). So an event is never lost and never sent for a change + that was rolled back. +- Consumers read with a consumer group, so each event is handled once per group, and acknowledge + only after handling it. Unacknowledged events are redelivered after a restart. +- Delivery is at least once. Every handler must be idempotent: handling the same event twice must + change nothing the second time. + +Needs the `redis` package (order and inventory list it). Without REDIS_URL nothing is published or +read, and outbox events simply wait. +""" +import json +import os +import threading +import time +from typing import Callable + +STREAM = "shop.events" + + +def redis_url() -> str | None: + return os.environ.get("REDIS_URL") or None + + +def connect(): + """A Redis client with short timeouts, or None when no bus is configured.""" + url = redis_url() + if not url: + return None + import redis + + return redis.Redis.from_url(url, socket_timeout=3, socket_connect_timeout=2, decode_responses=True) + + +def to_fields(event_id: str, event_type: str, payload: dict, source: str, occurred_at: str) -> dict: + return {"event_id": event_id, "type": event_type, "payload": json.dumps(payload), "source": source, + "occurred_at": occurred_at} + + +def from_fields(fields: dict) -> dict: + return {**fields, "payload": json.loads(fields.get("payload") or "{}")} + + +MAX_ATTEMPTS = 5 +DEAD_LETTER_STREAM = "shop.events.dead" + + +class Reader: + """One consumer in a consumer group. read_once() is one pass; start_consumer() loops it. + + Alternates between new events (">") and this consumer's own unacknowledged ones ("0"), so a + failing event is retried without holding back the events behind it. After MAX_ATTEMPTS it is + copied to the dead-letter stream and acknowledged, for a person to look at. + """ + + def __init__(self, group: str, consumer: str, handle: Callable[[dict], None], log=None): + self.group, self.consumer, self.handle, self.log = group, consumer, handle, log + self.start_id = "0" # first pass: our own unacked events from before a restart + self.failures: dict[str, int] = {} + + def read_once(self, client) -> None: + start_id = self.start_id + batches = client.xreadgroup(self.group, self.consumer, {STREAM: start_id}, count=20, block=2000) + messages = batches[0][1] if batches else [] + for message_id, fields in messages: + try: + self.handle(from_fields(fields)) + client.xack(STREAM, self.group, message_id) + self.failures.pop(message_id, None) + except Exception as exc: + attempts = self.failures.get(message_id, 0) + 1 + self.failures[message_id] = attempts + if self.log: + self.log.warning(f"event {message_id} failed (attempt {attempts}): {exc}") + if attempts >= MAX_ATTEMPTS: + client.xadd(DEAD_LETTER_STREAM, {**fields, "group": self.group, "error": str(exc)[:200]}) + client.xack(STREAM, self.group, message_id) + self.failures.pop(message_id, None) + # Retry failures on the next pass, new events on the one after: neither starves the other. + self.start_id = "0" if start_id == ">" and self.failures else ">" + + +def start_consumer(group: str, consumer: str, handle: Callable[[dict], None], log=None) -> threading.Thread | None: + """Runs a Reader on a background thread. Returns None when no bus is configured.""" + if not redis_url(): + return None + reader = Reader(group, consumer, handle, log) + + def loop() -> None: + import redis + + client = None + while True: + try: + if client is None: + client = connect() + try: + client.xgroup_create(STREAM, group, id="0", mkstream=True) + except redis.ResponseError as exc: + if "BUSYGROUP" not in str(exc): + raise + reader.read_once(client) + if reader.failures: + time.sleep(1) # do not spin on an event that keeps failing + except (redis.ConnectionError, redis.TimeoutError, OSError): + client = None + time.sleep(2) + + thread = threading.Thread(target=loop, name=f"consumer-{group}", daemon=True) + thread.start() + return thread diff --git a/shared/shop_common/flask_support.py b/shared/shop_common/flask_support.py index 06f07b6..d37c33e 100644 --- a/shared/shop_common/flask_support.py +++ b/shared/shop_common/flask_support.py @@ -5,6 +5,7 @@ GET /health/db which database this service owns (pattern 1); credentials never shown GET /debug/spans this process's recent spans, optionally filtered by ?trace_id= (pattern 6) GET /metrics Prometheus text format (pattern 6) + GET /debug/runtime what this process runs as: uid, container or not, image git SHA (pattern 9) POST /debug/crash stops this process, for the replica and discovery demos (CHAOS_ENABLED=1 only) Every response carries X-Instance-Id, so a demo can show which replica answered (pattern 10). @@ -19,6 +20,7 @@ from . import metrics from .discovery import DISCOVERY +from .runtime import runtime_info from .telemetry import REQUEST_ID_HEADER, SPAN_STORE, current_trace_id, get_logger, init_tracing @@ -30,6 +32,16 @@ def instance_id(service_name: str) -> str: return os.environ.get("INSTANCE_ID") or os.environ.get("CONTAINER_APP_REPLICA_NAME") or f"{service_name}-{os.getpid()}" +def is_serving_process(app: Flask) -> bool: + """False in the file-watcher process of app.run(debug=True). + + The reloader runs a watcher plus a worker (marked WERKZEUG_RUN_MAIN=true), and both build the + app. Only the worker serves requests, so only it may run background threads (registry + heartbeats, outbox relay, event consumer). Under gunicorn there is no reloader. + """ + return not app.debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true" + + def chaos_enabled() -> bool: return os.environ.get("CHAOS_ENABLED") == "1" @@ -55,7 +67,7 @@ def _start_timer(): def _record(response): response.headers[INSTANCE_HEADER] = this_instance route = request.url_rule.rule if request.url_rule else "unmatched" - if route in ("/health", "/debug/spans", "/metrics"): + if route in ("/health", "/debug/spans", "/metrics", "/debug/runtime"): return response elapsed = time.perf_counter() - g.get("request_started", time.perf_counter()) metrics.observe(service_name, request.method, route, response.status_code, elapsed) @@ -97,6 +109,10 @@ def prometheus_metrics(): body, content_type = metrics.render() return Response(body, content_type=content_type) + @app.get("/debug/runtime") + def debug_runtime(): + return jsonify(runtime_info(service_name, this_instance)) + @app.post("/debug/crash") def debug_crash(): if not chaos_enabled(): @@ -107,5 +123,5 @@ def debug_crash(): return jsonify({"stopping": this_instance}) public_url = os.environ.get("SERVICE_PUBLIC_URL") - if public_url: + if public_url and is_serving_process(app): DISCOVERY.register(service_name, this_instance, public_url) diff --git a/shared/shop_common/runtime.py b/shared/shop_common/runtime.py new file mode 100644 index 0000000..b5e5e40 --- /dev/null +++ b/shared/shop_common/runtime.py @@ -0,0 +1,21 @@ +"""Facts about the running process (pattern 9). No Flask or FastAPI import, so both can use it.""" +import os + + +def runtime_info(service_name: str, instance: str) -> dict: + """Facts about the running process, for the Containerization page. Never the user name.""" + import platform + from pathlib import Path + + uid = os.getuid() if hasattr(os, "getuid") else None + return { + "service": service_name, + "instance": instance, + "uid": uid, + "root": uid == 0, + "in_container": Path("/.dockerenv").exists() or bool(os.environ.get("CONTAINER_APP_NAME")), + "os": platform.system(), + "arch": platform.machine(), + "python": platform.python_version(), + "git_sha": os.environ.get("GIT_SHA", "local"), + } diff --git a/tests/smoke/test_post_run.py b/tests/smoke/test_post_run.py index da61a37..65800a7 100644 --- a/tests/smoke/test_post_run.py +++ b/tests/smoke/test_post_run.py @@ -35,13 +35,17 @@ def _checkout_flow(user_url: str, catalog_url: str, order_url: str) -> None: product_id = r.json()["product"]["id"] headers = {"Authorization": f"Bearer {login_token}"} - r = httpx.post(f"{order_url}/order/add-item", data={"product_id": product_id, "qty": 2}, headers=headers) + # Buy a seeded product: a product created a moment ago has no stock record in inventory, so the + # saga's reserve step would (correctly) refuse it. + assert product_id > 0 + httpx.post(f"{order_url}/inventory/1/restock", json={"quantity": 10}) + r = httpx.post(f"{order_url}/order/add-item", data={"product_id": 1, "qty": 2}, headers=headers) assert r.status_code == 200, r.text # order verifies the token itself, no call to user r = httpx.post(f"{order_url}/order/checkout", headers=headers) assert r.status_code == 200, r.text assert r.json()["result"]["is_open"] is False - assert r.json()["payment"]["amount_cents"] == 3000 # priced by catalog, charged by payment + assert r.json()["payment"]["amount_cents"] == 2 * 1299 # priced by catalog, charged by payment def test_services_full_flow_direct(): @@ -73,7 +77,7 @@ def test_each_service_owns_only_its_tables(): services = {s["service"]: s for s in httpx.get(f"{GATEWAY_URL}/api/patterns/databases").json()["services"]} assert set(services) == set(URLS) assert services["catalog"]["tables"] == ["product"] - assert services["inventory"]["tables"] == ["stock_item"] + assert services["inventory"]["tables"] == ["consumed_event", "reservation", "stock_item"] assert "product" not in services["order"]["tables"] @@ -101,6 +105,8 @@ def test_one_request_is_traced_across_services(): # --- M2 pattern demos (patterns 10, 12, 4), through the gateway ---------------------------------- def _login_with_cart() -> dict: + # Top up stock first, so repeated smoke runs never run product 1 out of stock. + httpx.post(f"{GATEWAY_URL}/api/shop/inventory/1/restock", json={"quantity": 10}) tag = uuid.uuid4().hex[:8] shop = f"{GATEWAY_URL}/api/shop" httpx.post(f"{shop}/user/create", data={"username": f"m2{tag}", "email": f"m2{tag}@example.com", "password": "secret123"}) @@ -158,3 +164,68 @@ def test_breaker_opens_when_payment_fails_and_closes_after_it_heals(): outcomes = {httpx.get(f"{api}/call").json()["outcome"] for _ in range(2)} assert outcomes == {"ok"} assert {b["state"] for b in httpx.get(f"{api}/state").json()["breakers"]} == {"closed"} + + +# --- M3a pattern demos (patterns 8, 7, 9), through the gateway ---------------------------------- + +def _stock(product_id: int) -> dict: + return next(s for s in httpx.get(f"{GATEWAY_URL}/api/shop/inventory").json()["results"] if s["product_id"] == product_id) + + +def _wait_for(condition, seconds: float = 8.0) -> bool: + import time + + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if condition(): + return True + time.sleep(0.5) + return False + + +def test_saga_sale_holds_then_sells_through_the_event_bus(): + _require_stack() + headers = _login_with_cart() # 3 units of product 1 + before = _stock(1) + r = httpx.post(f"{GATEWAY_URL}/api/shop/order/checkout", headers=headers, timeout=15) + assert r.status_code == 200, r.text + steps = httpx.get(f"{GATEWAY_URL}/api/shop/order/sagas", headers=headers).json()["results"][0]["steps"] + assert [s["step"] for s in steps] == ["price", "reserve_stock", "charge_payment", "confirm"] + # Inventory sells the hold when it reads OrderConfirmed from Redis, a moment later. + assert _wait_for(lambda: _stock(1)["on_hand"] == before["on_hand"] - 3), _stock(1) + assert _stock(1)["reserved"] == before["reserved"] + + +def test_declined_payment_is_compensated_and_stock_is_unchanged(): + _require_stack() + headers = _login_with_cart() + before = _stock(1) + r = httpx.post(f"{GATEWAY_URL}/api/shop/order/checkout", json={"simulate_decline": True}, headers=headers, timeout=15) + assert r.status_code == 402, r.text + saga = httpx.get(f"{GATEWAY_URL}/api/shop/order/sagas", headers=headers).json()["results"][0] + assert saga["outcome"] == "compensated" + assert _stock(1) == before + assert httpx.get(f"{GATEWAY_URL}/api/shop/order", headers=headers).json()["result"]["is_open"] is True + + +def test_events_wait_in_the_outbox_while_the_bus_is_down_then_drain(): + _require_stack() + api = f"{GATEWAY_URL}/api/patterns/events" + assert httpx.post(f"{api}/bus", json={"connected": False}).json()["replicas_switched"] == 2 + try: + headers = _login_with_cart() + before = _stock(1) + assert httpx.post(f"{GATEWAY_URL}/api/shop/order/checkout", headers=headers, timeout=15).status_code == 200 + assert _wait_for(lambda: httpx.get(api).json()["outbox"]["pending"] >= 1) + assert _stock(1)["reserved"] == before["reserved"] + 3 # held, not yet sold + finally: + httpx.post(f"{api}/bus", json={"connected": True}) + assert _wait_for(lambda: httpx.get(api).json()["outbox"]["pending"] == 0) + assert _wait_for(lambda: _stock(1)["on_hand"] == before["on_hand"] - 3) + + +def test_every_process_reports_how_it_runs(): + _require_stack() + processes = httpx.get(f"{GATEWAY_URL}/api/patterns/runtime").json()["processes"] + assert len(processes) == 7 # gateway + 5 services, order twice + assert all("uid" in p for p in processes)