diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99da61e..12e6d65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,54 +8,56 @@ on: workflow_dispatch: jobs: + changes: + # Pattern 3: decide which suites and images this change needs (scripts/ci_changes.py). + # Pull requests run only what they touch; pushes to main and manual runs run everything. + name: What changed + runs-on: ubuntu-latest + outputs: + suites: ${{ steps.plan.outputs.suites }} + images: ${{ steps.plan.outputs.images }} + frontend: ${{ steps.plan.outputs.frontend }} + bicep: ${{ steps.plan.outputs.bicep }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Plan the jobs + id: plan + env: + # Through env, never pasted into the script: a branch name is user-controlled text. + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail + if [ "$EVENT" = "pull_request" ]; then + git diff --name-only "origin/$BASE_REF...HEAD" | tee changed.txt | python scripts/ci_changes.py + else + python scripts/ci_changes.py --all + fi + unit-tests: name: Unit tests (${{ matrix.suite }}) + needs: changes + if: needs.changes.outputs.suites != '[]' runs-on: ubuntu-latest strategy: fail-fast: false # Each suite gets its own install: gateway/app and every # service's config.py share top-level module names, so they cannot # run in one interpreter (see CLAUDE.md, "collision note"). + # Built from scripts/service_table.py by scripts/ci_changes.py: one entry per suite to run. matrix: - include: - - suite: repo-root - requirements: requirements-dev.txt - tests: tests/unit - - suite: gateway - requirements: gateway/requirements.txt - tests: gateway/tests - shared: fastapi - - suite: registry - requirements: registry/requirements.txt - tests: registry/tests - - suite: user - requirements: services/user/requirements.txt - tests: services/user/tests - shared: flask - - suite: catalog - requirements: services/catalog/requirements.txt - tests: services/catalog/tests - shared: flask - contracts: true - - suite: order - requirements: services/order/requirements.txt - tests: services/order/tests - shared: flask - contracts: true - - suite: payment - requirements: services/payment/requirements.txt - tests: services/payment/tests - shared: flask - contracts: true - - suite: inventory - requirements: services/inventory/requirements.txt - tests: services/inventory/tests - shared: flask - contracts: true + include: ${{ fromJSON(needs.changes.outputs.suites) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: "pip" @@ -83,14 +85,16 @@ jobs: frontend: name: Frontend (build) + needs: changes + if: needs.changes.outputs.frontend == 'true' runs-on: ubuntu-latest defaults: run: working-directory: frontend steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: "20" cache: "npm" @@ -106,20 +110,15 @@ jobs: # Pattern 9. Builds every image exactly as Azure will run it (linux/amd64), starts it, and checks # it answers /health and does not run as root. Nothing is pushed anywhere. name: Container image (${{ matrix.image }}) + needs: changes + if: needs.changes.outputs.images != '[]' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - include: - - { image: user, dir: services/user, port: 5101, shared: true } - - { image: catalog, dir: services/catalog, port: 5102, shared: true } - - { image: order, dir: services/order, port: 5103, shared: true } - - { image: payment, dir: services/payment, port: 5104, shared: true } - - { image: inventory, dir: services/inventory, port: 5105, shared: true } - - { image: gateway, dir: gateway, port: 8100, shared: true } - - { image: registry, dir: registry, port: 8500, shared: false } + include: ${{ fromJSON(needs.changes.outputs.images) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Stage shared package (shop_common) if: matrix.shared @@ -148,10 +147,30 @@ jobs: bicep: name: Bicep (build) + needs: changes + if: needs.changes.outputs.bicep == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 # Compiles the templates only; nothing is deployed and no login is needed. - name: Build main.bicep run: az bicep build --file infra/bicep/main.bicep --stdout > /dev/null + + ci-ok: + # The one check to require before merging. Matrix jobs that a change skips report under names + # that never resolve, so requiring them would block forever; this job always runs and fails + # only if a job that did run failed or was cancelled. + name: CI ok + if: always() + needs: [changes, unit-tests, frontend, containers, bicep] + runs-on: ubuntu-latest + steps: + - name: Every job that ran passed + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + echo "Job results: $RESULTS" + for result in $RESULTS; do + case "$result" in success|skipped) ;; *) exit 1 ;; esac + done diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..2df5472 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,112 @@ +name: Deploy + +# Pattern 3: the manual "Deploy" button. Nothing deploys on its own: a person presses +# "Run workflow" on main. Rebuilds only the services whose code changed since the commit they run +# (scripts/deploy_plan.py), tags each image with this commit, and updates those Container Apps. +# +# Signs in with OIDC (#8, make github-identity): GitHub asks Azure for a short-lived token, so no +# Azure password exists anywhere. The 3 ids it needs are secrets only so they stay masked in this +# public repo's logs. + +on: + workflow_dispatch: + +permissions: + id-token: write # to request the OIDC token + contents: read + +concurrency: + group: deploy + cancel-in-progress: false + +jobs: + deploy: + name: Deploy changed services to Azure + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 # deploy_plan diffs against the commit each app runs + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Sign in to Azure (OIDC, no secret) + uses: azure/login@7184910d9eb2b1c5e48f7073824a90609bb9b6d6 # v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Find the registry and what each app runs now + id: running + run: | + set -euo pipefail + # Services, folders and resource group all come from scripts/service_table.py. + RESOURCE_GROUP=$(python scripts/deploy_plan.py --resource-group) + echo "RESOURCE_GROUP=$RESOURCE_GROUP" >> "$GITHUB_ENV" + python scripts/deploy_plan.py --services > services.txt + acr=$(az acr list -g "$RESOURCE_GROUP" --query "[0].name" -o tsv) + if [ -z "$acr" ]; then echo "Nothing provisioned in $RESOURCE_GROUP. Run make provision first."; exit 1; fi + echo "acr=$acr" >> "$GITHUB_OUTPUT" + running="{}" + while read -r app _dir; do + image=$(az containerapp show -n "$app" -g "$RESOURCE_GROUP" --query "properties.template.containers[0].image" -o tsv) + tag="${image##*:}" + # 'latest' or anything that is not a commit: unknown, so deploy_plan rebuilds it. + if [[ "$tag" =~ ^[0-9a-f]{7,40}$ ]]; then value="\"$tag\""; else value=null; fi + running=$(jq -c --arg app "$app" --argjson commit "$value" '.[$app] = $commit' <<< "$running") + done < services.txt + echo "$running" > running.json + cat running.json + + - name: Decide what to rebuild + id: plan + run: | + set -euo pipefail + head=$(git rev-parse --short=12 HEAD) + services=$(python scripts/deploy_plan.py "$head" < running.json | tr '\n' ' ') + echo "head=$head" >> "$GITHUB_OUTPUT" + echo "services=$services" >> "$GITHUB_OUTPUT" + echo "Deploying commit $head: ${services:-nothing changed}" + + - name: Build, push and roll out each changed service + if: steps.plan.outputs.services != '' + env: + ACR: ${{ steps.running.outputs.acr }} + HEAD_SHA: ${{ steps.plan.outputs.head }} + SERVICES: ${{ steps.plan.outputs.services }} + run: | + set -euo pipefail + echo "| Service | Commit | Image size | Health |" >> "$GITHUB_STEP_SUMMARY" + echo "|---|---|---|---|" >> "$GITHUB_STEP_SUMMARY" + for svc in $SERVICES; do + dir=$(awk -v s="$svc" '$1 == s {print $2}' services.txt) + cp -r shared "$dir/_shared" + az acr build --registry "$ACR" --resource-group "$RESOURCE_GROUP" \ + --image "$svc:$HEAD_SHA" --build-arg "GIT_SHA=$HEAD_SHA" --platform linux/amd64 "$dir" + rm -rf "$dir/_shared" + size=$(az acr manifest show --registry "$ACR" --name "$svc:$HEAD_SHA" -o json | python scripts/deploy_plan.py --size) + az containerapp update -n "$svc" -g "$RESOURCE_GROUP" \ + --image "$ACR.azurecr.io/$svc:$HEAD_SHA" --set-env-vars "IMAGE_SIZE_MB=${size:-0}" -o none + # "Update accepted" is not "running": wait until the service itself answers with the + # new commit (this also wakes a scaled-to-zero app). A crash-looping revision fails the run. + fqdn=$(az containerapp show -n "$svc" -g "$RESOURCE_GROUP" --query properties.configuration.ingress.fqdn -o tsv) + running="" + for attempt in $(seq 1 30); do + running=$(curl -sf --max-time 10 "https://$fqdn/debug/runtime" | jq -r .git_sha || true) + [ "$running" = "$HEAD_SHA" ] && break + sleep 10 + done + if [ "$running" != "$HEAD_SHA" ]; then + echo "::error::$svc did not answer with $HEAD_SHA within 5 minutes (last seen: ${running:-nothing})" + exit 1 + fi + echo "| $svc | \`$HEAD_SHA\` | ${size:-?} MB | answering |" >> "$GITHUB_STEP_SUMMARY" + done + + - name: Nothing to deploy + if: steps.plan.outputs.services == '' + run: echo "Every service already runs ${{ steps.plan.outputs.head }}." >> "$GITHUB_STEP_SUMMARY" diff --git a/CLAUDE.md b/CLAUDE.md index 9d7d8f9..f06cc5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,8 @@ history, monolith dropped). The full agreed plan is in [`docs/PLAN.md`](docs/PLA (FastAPI), `registry/` (FastAPI), `shared/` (`shop_common`), `contracts/`, `frontend/`, `infra/`, `scripts/`, `tests/`. - **One service table**: `scripts/service_table.py` lists every local process, folder and port, plus - `REPLICA_PORTS` (order also on 5113) and `CONTRACT_SUITES`. `setup.py`, `run_local.py`, + `REPLICA_PORTS` (order also on 5113), `CONTRACT_SUITES`, each service's `azure`/`shared` flags, + `AZURE_SERVICES` and `AZURE_RESOURCE_GROUP`. CI matrices, provision and the Deploy workflow read it. `setup.py`, `run_local.py`, `provision.py` and the smoke tests all read it. Add a service there. - Ports: user 5101, catalog 5102, order 5103 + 5113, payment 5104, inventory 5105, gateway 8100, registry 8500, Redis 6390, frontend 5273. @@ -47,7 +48,20 @@ history, monolith dropped). The full agreed plan is in [`docs/PLAN.md`](docs/PLA - Background threads (heartbeat, relay, consumer) start only where `is_serving_process(app)` is true: Flask's debug reloader runs a watcher plus a worker, and both build the app. - Containers (pattern 9): multi-stage Dockerfiles, uid 10001, `.dockerignore` per service, `GIT_SHA` - build arg, `/debug/runtime`. CI job "Container image" builds, runs and checks all 7 images. + build arg, `/debug/runtime` (also `image_size_mb` from `IMAGE_SIZE_MB`). CI job "Container image" + builds, runs and checks the images a change touches. +- CI-CD (pattern 3, M3b): `scripts/ci_changes.py` turns changed files into the CI matrices (suites, + images, frontend, bicep) from the service table. `.github/workflows/deploy.yml` is manual, main + only, OIDC via `infra/github_identity.py` (`make github-identity`, run by the owner; ids stored as + GitHub secrets so they stay masked in public logs); `scripts/deploy_plan.py` picks services whose + code changed since their running commit (CI-only files rebuild nothing) and waits until each service +answers `/debug/runtime` with the new commit. All workflow actions are pinned to commit SHAs. "CI ok" +is the one job to require in branch protection. +- Azure (M3b): provision keeps `image_tag`, `image_sizes` and `jwt_key_b64` in infra/.state.json so + `--skip-build` and re-runs neither break image pulls nor log users out. It tags images with the 12-char commit, imports `redis:7.4-alpine` into ACR, + generates the JWT key (`openssl`, passed base64 as a secure param, user reads `JWT_PRIVATE_KEY_B64`), + and re-grants the deploy identity's role on the new resource group. Redis is an internal TCP app + (`redis://redis:6379/0`), 1 replica; `make resume` keeps it at exactly 1. Registry is not deployed. - **`shop_common`** (`shared/`): every service calls `shop_common.flask_support.install(app, name, db)` inside its app context. It adds `/health`, `/health/db`, `/debug/spans`, `/metrics`, `POST /debug/crash` (only with `CHAOS_ENABLED=1`), the `X-Instance-Id` header, tracing, JSON logs, diff --git a/Makefile b/Makefile index d2fad5a..952236c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: setup run provision pause resume teardown verify test contracts smoke clean +.PHONY: setup run github-identity provision pause resume teardown verify test contracts smoke clean setup: python3.12 scripts/setup.py @@ -6,6 +6,10 @@ setup: run: python3.12 scripts/run_local.py +# Once, by the owner (#8): the Azure identity the Deploy workflow signs in as. No secrets created. +github-identity: + python3.12 infra/github_identity.py + provision: python3.12 infra/provision.py --budget-email "$(BUDGET_EMAIL)" $(ARGS) diff --git a/README.md b/README.md index 4fce399..e7e79cb 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,24 @@ collector. Without it, traces stay in each process's memory and the Observabilit ## Cloud -Azure Container Apps (one app per service plus the gateway, all scale-to-zero) + MySQL Flexible -Server, provisioned with `make provision` and removed with `make teardown`. About $0.21 for an -8-hour session. See [cost_estimate.md](cost_estimate.md). Nothing from M1, M2 or M3a has been deployed -yet. In Azure, discovery uses Container Apps DNS (no registry), and the demo switches stay off. +Azure Container Apps (one app per service plus the gateway, all scale-to-zero; Redis as an +always-on internal bus) + MySQL Flexible Server. About $0.21 for an 8-hour session; see +[cost_estimate.md](cost_estimate.md). In Azure, discovery uses Container Apps DNS (no registry), and +the demo switches stay off. Redis is reachable only from apps inside the environment and has no +password, which is fine for a demo where every app is ours. Nothing has been deployed yet: the first +Azure run is issue #11. + +```bash +make github-identity # once: the Azure identity the Deploy workflow uses (OIDC, no secret) +make provision BUDGET_EMAIL=you@example.com # everything, images tagged with the current commit +# then GitHub → Actions → Deploy → Run workflow: rebuilds only the services that changed +make teardown && make verify # remove everything, then prove nothing is left +``` + +### CI-CD (pattern 3) +- **CI on every pull request** runs only what the change touches (`scripts/ci_changes.py`, driven + by the service table): a change to one service runs its tests and its image; a change to shared + code, CI or the service table runs everything. Pushes to main run everything. +- **Deploy** is a manual workflow on main. It signs in to Azure with OIDC (no password stored), + rebuilds only services whose code changed since the commit they run, tags each image with the + commit, and records each image's size. diff --git a/cost_estimate.md b/cost_estimate.md index 849c702..d8a3eee 100644 --- a/cost_estimate.md +++ b/cost_estimate.md @@ -18,11 +18,15 @@ locally (plain Python/Node processes, SQLite, zero Docker, zero Azure) at **$0 c | Azure Database for MySQL Flexible Server | **Burstable B1ms** (1 vCore/2GiB), single server hosting 4 logical databases (monolith_db, user_db, product_db, order_db) provisioned upfront | Cheapest managed MySQL tier (no free tier exists for managed MySQL); sharing one server across 4 logical DBs avoids paying for 4 separate servers (documented tradeoff: less isolation than 4 servers, acceptable for a demo) | $0.019 (compute) | $0.15 | **~$14.50** (compute + 20GB storage) | SQLite (already used for local dev; not used in Azure to keep the "real MySQL in prod" story) | | Azure Static Web Apps | **Free tier** | 100GB bandwidth/mo, 1GB storage, free SSL | $0 | $0 | **$0** | N/A — already free | | Application Insights / Log Analytics | Pay-as-you-go, workspace-based | First 5GB/day ingestion free; demo-scale traffic stays inside this | $0 | $0 | **~$0** | N/A | +| Redis Container App (event bus, M3a/M3b) | **Consumption plan, 0.25 vCPU / 0.5 GiB, exactly 1 replica** (never scales to zero: undelivered events live in its memory) | Azure Cache for Redis starts near $16/mo; a Container App inside the same environment, internal TCP only, costs nothing within the monthly free grant | ~$0 (free grant) | **$0** (7,200 vCPU-s and 14,400 GiB-s, inside the free grant) | **~$4 idle to ~$14 busy** (after the 180,000 vCPU-s / 360,000 GiB-s free grant; consumption rates of about $0.000003 idle or $0.000024 active per vCPU-s and $0.000003 per GiB-s, re-check before provisioning) | None cheaper for an always-on bus; `make pause` scales it to 0 | | Cost Management budget alert | n/a | Free, created by `provision.py` before any billable resource | $0 | $0 | $0 | — | | Contributor role assignment (bff's managed identity, scoped to this one resource group only) | n/a | Free — control-plane only, lets the bff create the microservices' Container Apps itself during a live migration | $0 | $0 | $0 | — | **Total estimated cost for an 8-hour test session: ~$0.21** -**Total estimated cost if resources are accidentally left running for 30 days: ~$19.51** +**Total estimated cost if resources are accidentally left running for 30 days: ~$19.51 before M3b, +~$24 to ~$34 with the always-on Redis (M3b).** The upper end is above the $25/month budget alert, +which then fires: that is the alert doing its job. Run `make teardown` (or at least `make pause`) +after every session. These figures are **unchanged** from the original estimate — the only thing that changed is *when* each Container App gets created (monolith + bff at `make provision`; user-/product-/order-service diff --git a/frontend/src/api/gatewayClient.ts b/frontend/src/api/gatewayClient.ts index c0dc709..70dcd2c 100644 --- a/frontend/src/api/gatewayClient.ts +++ b/frontend/src/api/gatewayClient.ts @@ -289,6 +289,7 @@ export interface RuntimeInfo { arch?: string; python?: string; git_sha?: string; + image_size_mb?: number | null; unreachable?: boolean; } diff --git a/frontend/src/patterns/demos/ContainerizationDemo.tsx b/frontend/src/patterns/demos/ContainerizationDemo.tsx index 0aa7276..e49002f 100644 --- a/frontend/src/patterns/demos/ContainerizationDemo.tsx +++ b/frontend/src/patterns/demos/ContainerizationDemo.tsx @@ -50,6 +50,7 @@ export default function ContainerizationDemo() { Platform Python Build + Image @@ -60,7 +61,7 @@ export default function ContainerizationDemo() { {p.instance} {p.unreachable ? ( - + unreachable ) : ( @@ -76,6 +77,7 @@ export default function ContainerizationDemo() { {p.git_sha === "local" ? "local" : p.git_sha?.slice(0, 7)} + {p.image_size_mb ? `${p.image_size_mb} MB` : "–"} )} @@ -100,6 +102,10 @@ export default function ContainerizationDemo() {
  • Starts each image and waits for /health to answer.
  • Fails if the process inside runs as root (uid 0).
  • Writes each image's size to the run summary.
  • +
  • + The manual Deploy workflow records each image's compressed size in the app, shown in the Image column in + Azure. +
  • A .dockerignore in each service keeps local virtualenvs, databases and keys out of the build. diff --git a/gateway/tests/test_api.py b/gateway/tests/test_api.py index e99fd3d..4ea49be 100644 --- a/gateway/tests/test_api.py +++ b/gateway/tests/test_api.py @@ -136,3 +136,10 @@ def test_runtime_reports_the_gateway_without_a_user_name(): assert gateway["service"] == "gateway" assert set(gateway) >= {"uid", "root", "in_container", "arch", "git_sha"} assert "user" not in gateway + + +def test_runtime_reports_the_image_size_the_deploy_recorded(monkeypatch): + monkeypatch.setenv("IMAGE_SIZE_MB", "61") + assert client.get("/api/patterns/runtime").json()["processes"][0]["image_size_mb"] == 61 + monkeypatch.delenv("IMAGE_SIZE_MB") + assert client.get("/api/patterns/runtime").json()["processes"][0]["image_size_mb"] is None diff --git a/infra/bicep/main.bicep b/infra/bicep/main.bicep index f5502be..8b0ad64 100644 --- a/infra/bicep/main.bicep +++ b/infra/bicep/main.bicep @@ -1,8 +1,8 @@ @description(''' Deploys the whole stack: ACR (Basic), Container Apps Environment (Consumption), MySQL Flexible Server (Burstable B1ms, one logical DB per service), a Static Web App (Free) for the React -frontend, and one Container App per service (user, catalog, order, payment, inventory) plus -the API gateway. +frontend, one Container App per service (user, catalog, order, payment, inventory), the API +gateway, and Redis (internal TCP only) as the event bus. Every SKU is the cheapest tier that meets the need. Names are resolved by infra/provision.py BEFORE this deployment runs, after checking for collisions. ''') @@ -24,6 +24,16 @@ param mysqlAdminPassword string @description('Image tag to deploy for every service — set by provision.py after az acr build pushes images.') param imageTag string = 'latest' +@description('The user service signing key (RSA PEM, base64, one line). Generated by provision.py on every provision; never stored in git.') +@secure() +param jwtPrivateKeyB64 string + +@description('Compressed image size in MB per service, measured by provision.py after az acr build; shown on the Containerization page.') +param imageSizesMb object = {} + +@description('Redis image, imported into this ACR by provision.py (pattern 7 event bus).') +param redisImageTag string = '7.4-alpine' + @description('Set true only when Container Apps should be created with minReplicas=1 for a live demo. Default false = scale-to-zero, effectively $0 idle cost.') param keepWarm bool = false @@ -81,6 +91,9 @@ var commonEnv = [ { name: 'AZURE_MYSQL_ADMIN_PASSWORD', secretRef: 'mysql-password' } ] +// Pattern 7: the event bus. Other apps in the environment reach an internal TCP app by its name. +var redisUrl = 'redis://redis:6379/0' + var acrPassword = listCredentials(resourceId('Microsoft.ContainerRegistry/registries', acrName), '2023-11-01-preview').passwords[0].value module user 'modules/container-app.bicep' = { @@ -97,7 +110,11 @@ module user 'modules/container-app.bicep' = { imageTag: imageTag targetPort: 5101 minReplicas: minReplicas - envVars: commonEnv + extraSecrets: { items: [ { name: 'jwt-private-key', value: jwtPrivateKeyB64 } ] } + envVars: concat(commonEnv, [ + { name: 'JWT_PRIVATE_KEY_B64', secretRef: 'jwt-private-key' } + { name: 'IMAGE_SIZE_MB', value: string(imageSizesMb.?user ?? 0) } + ]) } } @@ -115,7 +132,9 @@ module catalog 'modules/container-app.bicep' = { imageTag: imageTag targetPort: 5102 minReplicas: minReplicas - envVars: commonEnv + envVars: concat(commonEnv, [ + { name: 'IMAGE_SIZE_MB', value: string(imageSizesMb.?catalog ?? 0) } + ]) } } @@ -138,6 +157,9 @@ module order 'modules/container-app.bicep' = { { name: 'USER_SERVICE_BASE_URL', value: 'https://${user.outputs.fqdn}' } { name: 'CATALOG_SERVICE_BASE_URL', value: 'https://${catalog.outputs.fqdn}' } { name: 'PAYMENT_SERVICE_BASE_URL', value: 'https://${payment.outputs.fqdn}' } + { name: 'INVENTORY_SERVICE_BASE_URL', value: 'https://${inventory.outputs.fqdn}' } + { name: 'REDIS_URL', value: redisUrl } + { name: 'IMAGE_SIZE_MB', value: string(imageSizesMb.?order ?? 0) } ]) } } @@ -156,7 +178,9 @@ module payment 'modules/container-app.bicep' = { imageTag: imageTag targetPort: 5104 minReplicas: minReplicas - envVars: commonEnv + envVars: concat(commonEnv, [ + { name: 'IMAGE_SIZE_MB', value: string(imageSizesMb.?payment ?? 0) } + ]) } } @@ -174,7 +198,34 @@ module inventory 'modules/container-app.bicep' = { imageTag: imageTag targetPort: 5105 minReplicas: minReplicas - envVars: commonEnv + envVars: concat(commonEnv, [ + { name: 'REDIS_URL', value: redisUrl } + { name: 'IMAGE_SIZE_MB', value: string(imageSizesMb.?inventory ?? 0) } + ]) + } +} + +// One replica, always on while provisioned: the stream lives in memory, so scaling to zero would +// drop undelivered events. Internal TCP only: nothing outside the environment can reach it. +// No Redis password: every app in this environment can read and write the streams. Acceptable for +// this demo, where all apps are ours; a shared environment would need AUTH or a managed Redis. +module redis 'modules/container-app.bicep' = { + name: 'redis' + params: { + location: location + name: 'redis' + environmentId: containerAppsEnv.outputs.environmentId + acrLoginServer: acr.outputs.loginServer + acrUsername: acr.outputs.registryName + acrPassword: acrPassword + imageName: 'redis' + imageTag: redisImageTag + targetPort: 6379 + exposedPort: 6379 + transport: 'tcp' + external: false + minReplicas: 1 + maxReplicas: 1 } } @@ -200,6 +251,7 @@ module gateway 'modules/container-app.bicep' = { { name: 'PAYMENT_SERVICE_BASE_URL', value: 'https://${payment.outputs.fqdn}' } { name: 'INVENTORY_SERVICE_BASE_URL', value: 'https://${inventory.outputs.fqdn}' } { name: 'FRONTEND_ORIGIN', value: 'https://${staticWebApp.outputs.defaultHostname}' } + { name: 'IMAGE_SIZE_MB', value: string(imageSizesMb.?gateway ?? 0) } ] } } diff --git a/infra/bicep/modules/container-app.bicep b/infra/bicep/modules/container-app.bicep index 5fc55f7..0ad0fd9 100644 --- a/infra/bicep/modules/container-app.bicep +++ b/infra/bicep/modules/container-app.bicep @@ -1,4 +1,4 @@ -@description('One reusable Container App module — instantiated once per service from main.bicep (user, catalog, order, payment, inventory, gateway). Explicit cheap defaults: Consumption plan, minReplicas 0 (scale-to-zero), 0.25 vCPU / 0.5Gi.') +@description('One reusable Container App module — instantiated once per service from main.bicep (user, catalog, order, payment, inventory, gateway, redis). Explicit cheap defaults: Consumption plan, minReplicas 0 (scale-to-zero), 0.25 vCPU / 0.5Gi.') param location string param name string param environmentId string @@ -16,9 +16,16 @@ param cpu string = '0.25' param memory string = '0.5Gi' param envVars array = [] param external bool = true -@description('Secret value for the AZURE_MYSQL_ADMIN_PASSWORD env var, referenced via secretRef so it never appears in plain env vars or deployment output.') +@description('auto = HTTP(S). tcp for non-HTTP backends such as Redis; internal TCP needs no custom VNet.') +param transport string = 'auto' +@description('TCP only: the port other apps in the environment connect to. 0 = not set (HTTP apps).') +param exposedPort int = 0 +@description('Extra secrets for this app only, e.g. the user service signing key: [{ name, value }].') @secure() -param mysqlAdminPassword string +param extraSecrets object = {} +@description('Secret value for the AZURE_MYSQL_ADMIN_PASSWORD env var, referenced via secretRef so it never appears in plain env vars or deployment output. Empty for apps without a database (Redis), which then get no MySQL secret at all.') +@secure() +param mysqlAdminPassword string = '' @description('Grants a system-assigned managed identity. Off for every app today; kept for later milestones.') param enableSystemIdentity bool = false @@ -34,7 +41,8 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { ingress: { external: external targetPort: targetPort - transport: 'auto' + transport: transport + exposedPort: exposedPort == 0 ? null : exposedPort } registries: [ { @@ -43,16 +51,17 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { passwordSecretRef: 'acr-password' } ] - secrets: [ + secrets: concat([ { name: 'acr-password' value: acrPassword } + ], empty(mysqlAdminPassword) ? [] : [ { name: 'mysql-password' value: mysqlAdminPassword } - ] + ], extraSecrets.?items ?? []) } template: { containers: [ diff --git a/infra/github_identity.py b/infra/github_identity.py new file mode 100644 index 0000000..4bb49ae --- /dev/null +++ b/infra/github_identity.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""make github-identity — the Azure identity the "Deploy" GitHub workflow signs in as (pattern 3). + +Creates, only where missing: + 1. an app registration named APP_NAME, and its service principal; + 2. a federated credential (OIDC) trusting only this repo's main branch, so GitHub signs in with a + short-lived token and no client secret exists anywhere; + 3. the Contributor role on the project resource group only, never the subscription. + +Then stores the three ids (client, tenant, subscription) as GitHub Actions secrets. They are not +passwords, but this repo is public and Actions logs are public: secrets are masked in logs, +variables are not, and the subscription id must never appear in public. + +Safe to run again: it changes nothing that already exists. The role assignment disappears with the +resource group on `make teardown`; `make provision` grants it again (grant_role_if_identity_exists). + +Runs with the owner's own `az login` and `gh auth`. To remove everything it created: + az ad app delete --id $(az ad app list --display-name ms12p-github-deploy --query "[0].appId" -o tsv) + gh secret delete AZURE_CLIENT_ID; gh secret delete AZURE_TENANT_ID; gh secret delete AZURE_SUBSCRIPTION_ID +""" +import json +import re +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from service_table import AZURE_RESOURCE_GROUP # noqa: E402 + +APP_NAME = "ms12p-github-deploy" +CREDENTIAL_NAME = "github-main-branch" +ISSUER = "https://token.actions.githubusercontent.com" +RESOURCE_GROUP = AZURE_RESOURCE_GROUP +ROLE = "Contributor" +STEPS = ["app", "service_principal", "credential", "role", "github_secrets"] +GITHUB_SECRETS = ["AZURE_CLIENT_ID", "AZURE_TENANT_ID", "AZURE_SUBSCRIPTION_ID"] + + +def federated_credential(repo: str) -> dict: + return { + "name": CREDENTIAL_NAME, + "issuer": ISSUER, + "subject": f"repo:{repo}:ref:refs/heads/main", + "audiences": ["api://AzureADTokenExchange"], + "description": "GitHub Actions on main of this repo only (Deploy workflow).", + } + + +def role_scope(subscription_id: str, resource_group: str) -> str: + return f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}" + + +def missing_steps(found: dict) -> list[str]: + """What still needs creating. The role needs the resource group, which only exists while + provisioned. A newly created app has a new client id, so the GitHub secrets are rewritten.""" + todo = [step for step in STEPS if not found[step] and (step != "role" or found["resource_group"])] + if "app" in todo and "github_secrets" not in todo: + todo.append("github_secrets") + return todo + + +def role_assignment_args(sp_object_id: str, scope: str) -> list[str]: + """By object id: right after creating a service principal, Entra has often not replicated it, + and an assignment by app id fails with "cannot find principal".""" + return ["role", "assignment", "create", "--assignee-object-id", sp_object_id, + "--assignee-principal-type", "ServicePrincipal", "--role", ROLE, "--scope", scope] + + +def pick_app(apps: list[dict]) -> str | None: + """`az ad app list --display-name` matches by prefix: keep only the exact name.""" + return next((app["appId"] for app in apps if app["displayName"] == APP_NAME), None) + + +def repo_from_remote(url: str) -> str: + match = re.search(r"github\.com[:/]([^/]+/[^/]+?)(?:\.git)?$", url.strip()) + if not match: + raise ValueError(f"not a GitHub remote: {url}") + return match.group(1) + + +def _run(args: list[str]) -> str: + return subprocess.run(args, capture_output=True, text=True, check=True).stdout.strip() + + +def _az_json(args: list[str]): + return json.loads(_run(["az", *args, "--output", "json"]) or "null") + + +def app_id() -> str | None: + return pick_app(_az_json(["ad", "app", "list", "--display-name", APP_NAME]) or []) + + +def sp_object_id(client_id: str) -> str | None: + principals = _az_json(["ad", "sp", "list", "--filter", f"appId eq '{client_id}'"]) or [] + return principals[0]["id"] if principals else None + + +def grant_role_if_identity_exists(subscription_id: str, resource_group: str) -> None: + """Called by make provision right after it creates the resource group.""" + client_id = app_id() + object_id = sp_object_id(client_id) if client_id else None + if object_id is None: + return + scope = role_scope(subscription_id, resource_group) + if not _az_json(["role", "assignment", "list", "--assignee", object_id, "--role", ROLE, "--scope", scope]): + _run(["az", *role_assignment_args(object_id, scope)]) + print(f"Deploy identity {APP_NAME}: {ROLE} granted on {resource_group}.") + + +def main() -> int: + account = _az_json(["account", "show"]) + subscription_id, tenant_id = account["id"], account["tenantId"] + repo = repo_from_remote(_run(["git", "remote", "get-url", "origin"])) + scope = role_scope(subscription_id, RESOURCE_GROUP) + + client_id = app_id() + object_id = sp_object_id(client_id) if client_id else None + secret_names = {item["name"] for item in json.loads(_run(["gh", "secret", "list", "--repo", repo, "--json", "name"]) or "[]")} + found = { + "app": client_id is not None, + "service_principal": object_id is not None, + "credential": bool(client_id and any(c["name"] == CREDENTIAL_NAME for c in + _az_json(["ad", "app", "federated-credential", "list", "--id", client_id]))), + "resource_group": _run(["az", "group", "exists", "--name", RESOURCE_GROUP]) == "true", + "role": False, + "github_secrets": set(GITHUB_SECRETS) <= secret_names, + } + if object_id and found["resource_group"]: + found["role"] = bool(_az_json(["role", "assignment", "list", "--assignee", object_id, "--role", ROLE, "--scope", scope])) + + todo = missing_steps(found) + print(f"Repo {repo}, identity {APP_NAME}. To create: {', '.join(todo) or 'nothing, already set up'}.") + if "app" in todo: + client_id = _az_json(["ad", "app", "create", "--display-name", APP_NAME])["appId"] + if "service_principal" in todo: + object_id = _az_json(["ad", "sp", "create", "--id", client_id])["id"] + if "credential" in todo: + _run(["az", "ad", "app", "federated-credential", "create", "--id", client_id, + "--parameters", json.dumps(federated_credential(repo))]) + if "role" in todo: + _run(["az", *role_assignment_args(object_id, scope)]) + if not found["resource_group"]: + print(f"{RESOURCE_GROUP} does not exist yet: make provision grants the role when it creates it.") + if "github_secrets" in todo: + # Not passwords, but masked in this public repo's logs, and passed on stdin so they never + # show in the process list. + for name, value in zip(GITHUB_SECRETS, [client_id, tenant_id, subscription_id]): + subprocess.run(["gh", "secret", "set", name, "--repo", repo], input=value, text=True, check=True, + capture_output=True) + print(f"GitHub secrets {', '.join(GITHUB_SECRETS)} are set.") + print("Done.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/provision.py b/infra/provision.py index b8690c4..a19fc61 100644 --- a/infra/provision.py +++ b/infra/provision.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -"""make provision — stands up the whole stack on Azure: user, catalog and order services -plus the API gateway, each as its own Container App, and the React frontend. +"""make provision — stands up the whole stack on Azure: the 5 services and the API gateway, each +as its own Container App, Redis as the event bus, and the React frontend. Every image is tagged +with the commit it was built from. Thin orchestrator: resolves collision-safe resource names, builds every service image in the cloud via `az acr build` (no local Docker daemon, and @@ -16,6 +17,7 @@ config get defaults.location`, and only as a last resort the constant below. """ import argparse +import base64 import json import os import secrets @@ -35,7 +37,6 @@ BICEP_FILE = REPO_ROOT / "infra" / "bicep" / "main.bicep" ENV_FILE = REPO_ROOT / ".env" -DEFAULT_RESOURCE_GROUP = "rg-microservices-12-patterns" FALLBACK_LOCATION = "eastus" # only used if neither .env nor `az config` set one DEFAULT_NAME_PREFIX = "ms12p" MYSQL_ADMIN_USERNAME = "flaskadmin" @@ -58,10 +59,71 @@ def load_dotenv_defaults() -> None: os.environ[key] = value sys.path.insert(0, str(REPO_ROOT / "scripts")) -from service_table import PYTHON_SERVICES # noqa: E402 +from service_table import AZURE_RESOURCE_GROUP as DEFAULT_RESOURCE_GROUP # noqa: E402 +from service_table import AZURE_SERVICES as _AZURE_SERVICE_ROWS # noqa: E402 -# Image name -> build context. Image names match Container App names in main.bicep. -SERVICES = {svc.name: svc.path for svc in PYTHON_SERVICES} +# Image name -> build context, for every image Azure runs. Image names match Container App names in +# main.bicep. The registry is local only: in Azure, Container Apps DNS does service discovery. +AZURE_SERVICES = {svc.name: svc.path for svc in _AZURE_SERVICE_ROWS} + + +REDIS_IMAGE = "redis:7.4-alpine" # keep in step with redisImageTag in main.bicep + + +def redis_import_args(resource_group: str, acr_name: str) -> list[str]: + """Copies Redis from Docker Hub into our ACR once, so Container Apps never pulls from Docker + Hub (rate limits). The image is multi-arch; Container Apps picks its linux/amd64 variant.""" + return ["acr", "import", "--name", acr_name, "--resource-group", resource_group, + "--source", f"docker.io/library/{REDIS_IMAGE}", "--image", REDIS_IMAGE, "--force"] + + +def generate_jwt_key_b64() -> str: + """A fresh RSA key for the user service to sign login tokens (pattern 10), base64 on one line + so it passes cleanly as a deployment parameter. Uses the openssl CLI: provision runs on plain + python3.12, without the cryptography package.""" + pem = subprocess.run(["openssl", "genpkey", "-algorithm", "RSA", "-pkeyopt", "rsa_keygen_bits:2048"], + capture_output=True, check=True).stdout + return base64.b64encode(pem).decode() + + +def container_apps_for_state() -> dict[str, str]: + """Every Container App provision creates, for pause/resume/teardown to find. Includes Redis.""" + return {name: name for name in [*AZURE_SERVICES, "redis"]} + + +def choose_image_tag(skip_build: bool, head: str, previous: str | None) -> str: + """The tag the apps should run. --skip-build builds nothing, so it must reuse the last build: + pointing apps at an image that was never pushed makes every pull fail.""" + if not skip_build: + return head + if previous is None: + raise SystemExit("No previous image build is recorded in infra/.state.json: run make provision without --skip-build.") + return previous + + +def choose_jwt_key(previous: str | None) -> str: + """Keep the signing key across re-runs of provision, so nobody is logged out; new on a fresh stack.""" + return previous or generate_jwt_key_b64() + + +def measure_image_sizes(acr_name: str, tag: str) -> dict[str, int]: + """Compressed size of each image just pushed, for the Containerization page (pattern 9).""" + from deploy_plan import image_size_mb + + sizes = {} + for service_name in AZURE_SERVICES: + manifest = json.loads(run_az(["acr", "manifest", "show", "--registry", acr_name, + "--name", f"{service_name}:{tag}"]).stdout) + size = image_size_mb(manifest) + if size is not None: + sizes[service_name] = size + return sizes + + +def image_tag() -> str: + """The commit being deployed, so every image and every running app says what it was built from.""" + return subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=REPO_ROOT, + capture_output=True, text=True, check=True).stdout.strip() def _redact_secrets(args: list[str]) -> list[str]: @@ -190,8 +252,8 @@ def wait_for_acr_ready(resource_group: str, acr_name: str, timeout: float = 60.0 print(f"Warning: ACR '{acr_name}' still not queryable after {timeout}s — proceeding anyway.") -def build_images(resource_group: str, acr_name: str) -> None: - for service_name, service_dir in SERVICES.items(): +def build_images(resource_group: str, acr_name: str, tag: str) -> None: + for service_name, service_dir in AZURE_SERVICES.items(): print(f"\n=== az acr build: {service_name} (cloud-side, no local Docker) ===") # `az acr build` only uploads the service's own folder, so stage a copy of shared/ # (shop_common) inside it as _shared/. Each Dockerfile installs ./_shared. Gitignored. @@ -199,37 +261,33 @@ def build_images(resource_group: str, acr_name: str) -> None: shutil.rmtree(staged, ignore_errors=True) shutil.copytree(REPO_ROOT / "shared", staged, ignore=shutil.ignore_patterns("*.egg-info", "__pycache__")) try: - _acr_build(resource_group, acr_name, service_name, service_dir) + run_az(acr_build_args(resource_group, acr_name, service_name, service_dir, tag), capture=False) finally: shutil.rmtree(staged, ignore_errors=True) -def _acr_build(resource_group: str, acr_name: str, service_name: str, service_dir: Path) -> None: - run_az([ +def acr_build_args(resource_group: str, acr_name: str, service_name: str, service_dir: Path, tag: str) -> list[str]: + return [ "acr", "build", "--registry", acr_name, "--resource-group", resource_group, - "--image", f"{service_name}:latest", + "--image", f"{service_name}:{tag}", + # Inside the image: GIT_SHA, which /debug/runtime reports (pattern 9). + "--build-arg", f"GIT_SHA={tag}", # Explicit: Container Apps runs amd64, and the owner's Mac is arm64. "--platform", "linux/amd64", str(service_dir), - ], capture=False) + ] -def deploy_bicep(resource_group: str, location: str, names: dict, mysql_password: str, keep_warm: bool) -> dict: - print("\n=== Deploying infra/bicep/main.bicep ===") - # Static Web Apps is only available in a handful of regions (Central US, - # East US 2, West US 2, West Europe, East Asia) — NOT every region that - # supports Container Apps/MySQL/etc (plain "East US" doesn't support it, - # which is a real failure discovered running this for the first time). - # Kept independently configurable via .env so it's still never hardcoded - # as the only option. +def bicep_parameters(location: str, names: dict, mysql_password: str, keep_warm: bool, tag: str, + jwt_key_b64: str, image_sizes: dict[str, int] | None = None) -> list[str]: + # Static Web Apps is only available in a handful of regions (Central US, East US 2, West US 2, + # West Europe, East Asia) — NOT every region that supports Container Apps/MySQL/etc (plain + # "East US" doesn't support it, a real failure found the first time this ran). Kept + # configurable via .env so it is never hardcoded as the only option. static_web_app_location = os.environ.get("AZURE_STATIC_WEB_APP_LOCATION", "eastus2").strip() or "eastus2" - result = run_az([ - "deployment", "group", "create", - "--resource-group", resource_group, - "--template-file", str(BICEP_FILE), - "--parameters", + return [ f"location={location}", f"staticWebAppLocation={static_web_app_location}", f"acrName={names['acr_name']}", @@ -240,6 +298,19 @@ def deploy_bicep(resource_group: str, location: str, names: dict, mysql_password f"mysqlAdminUsername={MYSQL_ADMIN_USERNAME}", f"mysqlAdminPassword={mysql_password}", f"keepWarm={'true' if keep_warm else 'false'}", + f"imageTag={tag}", + f"jwtPrivateKeyB64={jwt_key_b64}", + f"imageSizesMb={json.dumps(image_sizes or {})}", + ] + + +def deploy_bicep(resource_group: str, parameters: list[str]) -> dict: + print("\n=== Deploying infra/bicep/main.bicep ===") + result = run_az([ + "deployment", "group", "create", + "--resource-group", resource_group, + "--template-file", str(BICEP_FILE), + "--parameters", *parameters, ]) return json.loads(result.stdout)["properties"]["outputs"] @@ -327,6 +398,10 @@ def main() -> int: print(f"Subscription: {subscription_id} Location: {location}") ensure_resource_group(args.resource_group, location) + # Teardown deletes the group and with it the Deploy identity's role (#8): grant it again. + from github_identity import grant_role_if_identity_exists + + grant_role_if_identity_exists(subscription_id, args.resource_group) if args.reuse_names: print("\n=== Reusing existing resource names (--reuse-names) ===") @@ -354,7 +429,14 @@ def main() -> int: for key, value in names.items(): print(f" {key}: {value}") + previous = json.loads(STATE_FILE.read_text()) if STATE_FILE.exists() else {} + if previous.get("resource_group") != args.resource_group: + previous = {} mysql_password = generate_mysql_password() + jwt_key_b64 = choose_jwt_key(previous.get("jwt_key_b64")) + tag = choose_image_tag(args.skip_build, image_tag(), previous.get("image_tag")) + image_sizes = previous.get("image_sizes", {}) if args.skip_build else {} + print(f" image tag: {tag} ({'reused from the last build' if args.skip_build else 'the commit being deployed'})") if not args.skip_build: run_az([ @@ -373,17 +455,21 @@ def main() -> int: "--admin-enabled", "true", ]) wait_for_acr_ready(args.resource_group, names["acr_name"]) - build_images(args.resource_group, names["acr_name"]) + print("\n=== az acr import: redis (event bus, pattern 7) ===") + run_az(redis_import_args(args.resource_group, names["acr_name"]), capture=False) + build_images(args.resource_group, names["acr_name"], tag) + image_sizes = measure_image_sizes(names["acr_name"], tag) try: - outputs = deploy_bicep(args.resource_group, location, names, mysql_password, args.keep_warm) + parameters = bicep_parameters(location, names, mysql_password, args.keep_warm, tag, jwt_key_b64, image_sizes) + outputs = deploy_bicep(args.resource_group, parameters) except subprocess.CalledProcessError as exc: # Never let the real password leak into a traceback or log — redact # it from stderr/stdout before printing (caught for real: the first # run of this script hit a genuine Bicep failure — Static Web Apps # isn't available in every region — and the unhandled traceback here # didn't show the actual az error at all, only a stack trace). - stderr = (exc.stderr or "").replace(mysql_password, "***REDACTED***") + stderr = (exc.stderr or "").replace(mysql_password, "***REDACTED***").replace(jwt_key_b64, "***REDACTED***") print(f"\nBicep deployment failed:\n{stderr}") return 1 output_values = {k: v["value"] for k, v in outputs.items()} @@ -405,7 +491,12 @@ def main() -> int: "mysql_admin_username": MYSQL_ADMIN_USERNAME, "mysql_admin_password": mysql_password, "acr_password": acr_password, - "container_apps": {name: name for name in SERVICES}, + # Reused by the next provision (--skip-build, retries) so apps keep their images and + # nobody is logged out. Gitignored, like the passwords above. + "image_tag": tag, + "image_sizes": image_sizes, + "jwt_key_b64": jwt_key_b64, + "container_apps": container_apps_for_state(), } STATE_FILE.parent.mkdir(parents=True, exist_ok=True) STATE_FILE.write_text(json.dumps(state, indent=2)) diff --git a/infra/resume.py b/infra/resume.py index 0bfc8dd..c14ce7f 100644 --- a/infra/resume.py +++ b/infra/resume.py @@ -16,6 +16,14 @@ STATE_FILE = REPO_ROOT / "infra" / ".state.json" MAX_REPLICAS_DEFAULT = "3" +# Apps that must never scale to zero: Redis keeps undelivered events in memory (pattern 7). +ALWAYS_ON = {"redis"} + + +def replica_args(app_name: str) -> list[str]: + if app_name in ALWAYS_ON: + return ["--min-replicas", "1", "--max-replicas", "1"] + return ["--min-replicas", "0", "--max-replicas", MAX_REPLICAS_DEFAULT] def _wait_for_mysql_ready(mysql_name: str, resource_group: str, timeout: float = 180.0) -> bool: @@ -59,10 +67,10 @@ def main() -> int: container_apps = state.get("container_apps", {}) for app_name in container_apps.values(): - print(f"=== Restoring Container App '{app_name}' scale range (min=0, max={MAX_REPLICAS_DEFAULT}) ===") + print(f"=== Restoring Container App '{app_name}' scale range ({' '.join(replica_args(app_name))}) ===") subprocess.run( ["az", "containerapp", "update", "--name", app_name, "--resource-group", resource_group, - "--min-replicas", "0", "--max-replicas", MAX_REPLICAS_DEFAULT, "--output", "json"], + *replica_args(app_name), "--output", "json"], ) print("\nResumed. Give the services a few seconds to cold-start on first request.") diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py new file mode 100644 index 0000000..18b96e3 --- /dev/null +++ b/scripts/ci_changes.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Which CI jobs a change needs (pattern 3): only the services it touches. + +Reads the changed file paths (one per line on stdin) or `--all`, and writes the plan as GitHub +Actions outputs: `suites` and `images` (JSON lists for job matrices), `frontend` and `bicep` +(true/false). The suite and image lists come from scripts/service_table.py, so a new service needs +no CI edit. + +Rules: +- shared code, the service table, CI itself, pytest settings or Pact's version: everything runs. +- a service's folder: its unit tests and its image. +- the order service is the contract consumer, so changing it also runs each provider's contract + check (payment, catalog, inventory); a contract file runs the consumer and that one provider. +- infra/ and scripts/: the repo-root tests; infra/bicep/ also the Bicep compile. +- frontend/ (and the contracts it displays): the frontend build. +- anything else (docs, README, smoke tests that CI does not run): nothing. +""" +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from service_table import CONTRACT_SUITES, PYTHON_SERVICES, REPO_ROOT # noqa: E402 + +# Changes every service's code or image: CI and Deploy both treat them as touching everything. +EVERY_SERVICE = ("shared/", "scripts/service_table.py") +# Changes only how CI runs (workflows, this planner, test settings, Pact's version): run every CI job, +# but a Deploy rebuilds nothing for them. +CI_ONLY = ("scripts/ci_changes.py", ".github/workflows/", "contracts/requirements.txt", "pytest.ini") +CONSUMER = "order" + + +def _rel(path: Path) -> str: + return path.relative_to(REPO_ROOT).as_posix() + + +def _suite(name: str) -> dict: + if name == "repo-root": + return {"suite": name, "requirements": "requirements-dev.txt", "tests": "tests/unit", "shared": "", + "contracts": False} + svc = next(s for s in PYTHON_SERVICES if s.name == name) + shared = ("fastapi" if svc.kind == "fastapi" else "flask") if svc.shared else "" + return {"suite": name, "requirements": f"{_rel(svc.path)}/requirements.txt", "tests": f"{_rel(svc.path)}/tests", + "shared": shared, "contracts": name in CONTRACT_SUITES} + + +def _image(name: str) -> dict: + svc = next(s for s in PYTHON_SERVICES if s.name == name) + return {"image": name, "dir": _rel(svc.path), "port": svc.port, "shared": svc.shared} + + +def plan(changed: list[str] | None, for_deploy: bool = False) -> dict: + """changed=None means run everything (a push to main, a manual run). + + for_deploy=True ignores CI-only files: they change no running code, so nothing is rebuilt. + """ + if changed is not None and for_deploy: + changed = [path for path in changed if not path.startswith(CI_ONLY)] + run_everything = EVERY_SERVICE if for_deploy else EVERY_SERVICE + CI_ONLY + every_service = [s.name for s in PYTHON_SERVICES] + suites: set[str] = set() + images: set[str] = set() + frontend = bicep = False + + if changed is None or any(path.startswith(run_everything) for path in changed): + suites, images, frontend, bicep = {"repo-root", *every_service}, set(every_service), True, True + else: + folders = {_rel(s.path) + "/": s.name for s in PYTHON_SERVICES} + for path in changed: + owner = next((name for folder, name in folders.items() if path.startswith(folder)), None) + if owner: + suites.add(owner) + images.add(owner) + if owner == CONSUMER: + suites.update(CONTRACT_SUITES) + elif path.startswith("contracts/pacts/"): + provider = Path(path).stem.removeprefix(f"{CONSUMER}-") + if provider not in CONTRACT_SUITES: # never skip a contract check silently + raise ValueError(f"{path}: no provider '{provider}' in CONTRACT_SUITES") + suites.update({CONSUMER, provider}) + frontend = True # the Contract Testing page displays these files + elif path.startswith(("infra/", "scripts/", "tests/unit/", "tests/conftest.py", "requirements-dev.txt")): + suites.add("repo-root") + bicep = bicep or path.startswith("infra/bicep/") + elif path.startswith("frontend/"): + frontend = True + + return { + "suites": [_suite(name) for name in sorted(suites)], + "images": [_image(name) for name in sorted(images)], + "frontend": frontend, + "bicep": bicep, + } + + +def main() -> int: + changed = None if "--all" in sys.argv else [line.strip() for line in sys.stdin if line.strip()] + result = plan(changed) + lines = [f"suites={json.dumps(result['suites'])}", f"images={json.dumps(result['images'])}", + f"frontend={str(result['frontend']).lower()}", f"bicep={str(result['bicep']).lower()}"] + output = os.environ.get("GITHUB_OUTPUT") + if output: + with open(output, "a") as handle: + handle.write("\n".join(lines) + "\n") + print("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/deploy_plan.py b/scripts/deploy_plan.py new file mode 100644 index 0000000..f4bb012 --- /dev/null +++ b/scripts/deploy_plan.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Which services the Deploy workflow rebuilds (pattern 3): those whose code changed since the +commit they are running now. Uses the same rules as CI (ci_changes.plan), so a shared/ change +rebuilds every service, minus CI-only files, which rebuild nothing. + +CLI, used by .github/workflows/deploy.yml so the workflow hardcodes nothing: + deploy_plan.py --services every Azure service and its folder, "name folder" per line + deploy_plan.py --resource-group the project's Azure resource group + deploy_plan.py --size image size in MB, from `az acr manifest show` JSON on stdin + deploy_plan.py reads {"service": "running-commit-or-null"} JSON on stdin and + prints the services to deploy, one per line +""" +import json +import subprocess +import sys +from pathlib import Path +from typing import Callable + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ci_changes import plan # noqa: E402 +from service_table import AZURE_RESOURCE_GROUP, AZURE_SERVICES, REPO_ROOT # noqa: E402 + + +def services_to_deploy(running: dict[str, str | None], head: str, + changed_since: Callable[[str], list[str]]) -> list[str]: + deploy = [] + for service, commit in sorted(running.items()): + if commit == head: + continue + try: + files = changed_since(commit) if commit else None + except LookupError: # a commit this checkout does not know: rebuild to be safe + files = None + if files is None or service in {image["image"] for image in plan(files, for_deploy=True)["images"]}: + deploy.append(service) + return deploy + + +def image_size_mb(manifest: dict) -> int | None: + """Compressed size of a single-platform image: its config plus every layer. None for a + multi-platform index, which has no layers of its own (the page then shows "unknown").""" + if "layers" not in manifest or "config" not in manifest: + return None + total = manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]) + return round(total / (1024 * 1024)) + + +def git_changed_since(commit: str) -> list[str]: + result = subprocess.run(["git", "diff", "--name-only", f"{commit}...HEAD"], cwd=REPO_ROOT, + capture_output=True, text=True) + if result.returncode != 0: + raise LookupError(commit) + return [line for line in result.stdout.splitlines() if line] + + +def main() -> int: + if sys.argv[1:2] == ["--services"]: + print("\n".join(f"{s.name} {s.path.relative_to(REPO_ROOT).as_posix()}" for s in AZURE_SERVICES)) + return 0 + if sys.argv[1:2] == ["--resource-group"]: + print(AZURE_RESOURCE_GROUP) + return 0 + if sys.argv[1:2] == ["--size"]: + size = image_size_mb(json.load(sys.stdin)) + print("" if size is None else size) + return 0 + head = sys.argv[1] + running = json.load(sys.stdin) + print("\n".join(services_to_deploy(running, head, git_changed_since))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/service_table.py b/scripts/service_table.py index 229bec7..53dce4c 100644 --- a/scripts/service_table.py +++ b/scripts/service_table.py @@ -18,6 +18,8 @@ class Service: port: int # default local port port_env: str # env var the process reads its port from kind: str # "flask" or "fastapi" + azure: bool = True # deployed as a Container App (the registry is local only) + shared: bool = True # installs shared/ (shop_common); the registry does not @property def local_port(self) -> int: @@ -36,9 +38,14 @@ def local_url(self) -> str: Service("inventory", REPO_ROOT / "services" / "inventory", 5105, "INVENTORY_SERVICE_PORT", "flask"), ] GATEWAY = Service("gateway", REPO_ROOT / "gateway", 8100, "GATEWAY_PORT", "fastapi") -REGISTRY = Service("registry", REPO_ROOT / "registry", 8500, "REGISTRY_PORT", "fastapi") +REGISTRY = Service("registry", REPO_ROOT / "registry", 8500, "REGISTRY_PORT", "fastapi", azure=False, shared=False) PYTHON_SERVICES = FLASK_SERVICES + [GATEWAY, REGISTRY] +AZURE_SERVICES = [s for s in PYTHON_SERVICES if s.azure] + +# The one Azure resource group (provision, teardown, the deploy identity and the Deploy workflow). +# Must never match the parent project's rg-flask-monolith-microservices: teardown deletes it whole. +AZURE_RESOURCE_GROUP = "rg-microservices-12-patterns" # The event bus (patterns 7 and 8): a Homebrew redis-server, no Docker. Offset from Redis' default # 6379 so a Redis the owner already runs is never touched. diff --git a/services/user/application/tokens.py b/services/user/application/tokens.py index 8cadb96..b3f166c 100644 --- a/services/user/application/tokens.py +++ b/services/user/application/tokens.py @@ -1,8 +1,11 @@ """Signs login tokens (pattern 10). Only the user service holds the private key. -Key source, in order: JWT_PRIVATE_KEY_PEM (Azure secret), else instance/jwt_private.pem, created -on first start. Keeping the key in a file means restarting the service does not log anyone out. +Key source, in order: + 1. JWT_PRIVATE_KEY_B64: base64 PEM, the Azure Container Apps secret that provision sets; + 2. JWT_PRIVATE_KEY_PEM; + 3. instance/jwt_private.pem, created on first start. Keeping the key in a file means restarting the service does not log anyone out. """ +import base64 import os from datetime import datetime, timedelta, timezone from pathlib import Path @@ -21,7 +24,9 @@ def _private_key_pem() -> bytes: global _private_pem if _private_pem is None: - if os.environ.get("JWT_PRIVATE_KEY_PEM"): + if os.environ.get("JWT_PRIVATE_KEY_B64"): + _private_pem = base64.b64decode(os.environ["JWT_PRIVATE_KEY_B64"]) + elif os.environ.get("JWT_PRIVATE_KEY_PEM"): _private_pem = os.environ["JWT_PRIVATE_KEY_PEM"].encode() elif KEY_FILE.exists(): _private_pem = KEY_FILE.read_bytes() diff --git a/services/user/tests/test_tokens.py b/services/user/tests/test_tokens.py index a458b21..9ff6f9f 100644 --- a/services/user/tests/test_tokens.py +++ b/services/user/tests/test_tokens.py @@ -34,3 +34,26 @@ def test_public_key_verifies_the_token(client): token = _register_and_login(client)["token"] key = client.get("/api/user/public-key").get_json()["public_key_pem"] assert jwt.decode(token, key, algorithms=["RS256"], issuer="ms12p-user")["username"] == "bob" + + +def test_signing_key_can_come_from_the_azure_secret(client, monkeypatch, tmp_path): + """In Azure, provision passes the key base64-encoded in JWT_PRIVATE_KEY_B64 (a Container Apps + secret), so every replica and every restart signs with the same key.""" + import base64 + + import jwt + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + from application import tokens + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + monkeypatch.setenv("JWT_PRIVATE_KEY_B64", base64.b64encode(pem).decode()) + monkeypatch.setattr(tokens, "_private_pem", None) + monkeypatch.setattr(tokens, "KEY_FILE", tmp_path / "unused.pem") + + token = _register_and_login(client, "carol")["token"] + public = key.public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo) + assert jwt.decode(token, public, algorithms=["RS256"], issuer="ms12p-user")["username"] == "carol" + assert not (tmp_path / "unused.pem").exists() diff --git a/shared/shop_common/runtime.py b/shared/shop_common/runtime.py index b5e5e40..b095dbf 100644 --- a/shared/shop_common/runtime.py +++ b/shared/shop_common/runtime.py @@ -18,4 +18,6 @@ def runtime_info(service_name: str, instance: str) -> dict: "arch": platform.machine(), "python": platform.python_version(), "git_sha": os.environ.get("GIT_SHA", "local"), + # Set by the Deploy workflow from the registry's manifest; unknown otherwise. + "image_size_mb": int(os.environ["IMAGE_SIZE_MB"]) if os.environ.get("IMAGE_SIZE_MB", "").isdigit() else None, } diff --git a/tests/unit/test_ci_changes.py b/tests/unit/test_ci_changes.py new file mode 100644 index 0000000..67d3a5e --- /dev/null +++ b/tests/unit/test_ci_changes.py @@ -0,0 +1,67 @@ +"""#7: CI runs only what a change touches. ci_changes.plan() turns the changed file list into +the unit-test suites, container images and other jobs to run.""" +from ci_changes import plan + + +def _suites(p): + return sorted(s["suite"] for s in p["suites"]) + + +def _images(p): + return sorted(i["image"] for i in p["images"]) + + +def test_a_change_to_one_service_runs_only_that_service(): + p = plan(["services/payment/application/routes.py"]) + assert _suites(p) == ["payment"] + assert _images(p) == ["payment"] + assert p["frontend"] is False and p["bicep"] is False + + +def test_changing_the_consumer_also_runs_every_provider_contract_check(): + p = plan(["services/order/application/saga.py"]) + assert _suites(p) == ["catalog", "inventory", "order", "payment"] + assert _images(p) == ["order"] + + +def test_shared_code_runs_everything(): + p = plan(["shared/shop_common/events.py"]) + assert _suites(p) == ["catalog", "gateway", "inventory", "order", "payment", "registry", "repo-root", "user"] + assert len(_images(p)) == 7 and p["frontend"] and p["bicep"] + + +def test_the_service_table_and_the_workflow_run_everything(): + for path in ["scripts/service_table.py", ".github/workflows/ci.yml", "contracts/requirements.txt"]: + assert len(plan([path])["suites"]) == 8, path + + +def test_a_contract_file_runs_the_consumer_and_that_provider(): + assert _suites(plan(["contracts/pacts/order-inventory.json"])) == ["inventory", "order"] + + +def test_frontend_infra_and_docs_changes(): + assert plan(["frontend/src/App.tsx"]) == {"suites": [], "images": [], "frontend": True, "bicep": False} + p = plan(["infra/bicep/main.bicep"]) + assert _suites(p) == ["repo-root"] and p["bicep"] + assert plan(["README.md", "docs/PLAN.md"]) == {"suites": [], "images": [], "frontend": False, "bicep": False} + + +def test_matrix_entries_carry_what_the_jobs_need(): + p = plan(["services/order/run.py", "gateway/app/main.py", "registry/app/main.py"]) + order = next(s for s in p["suites"] if s["suite"] == "order") + assert order == {"suite": "order", "requirements": "services/order/requirements.txt", + "tests": "services/order/tests", "shared": "flask", "contracts": True} + image = next(i for i in p["images"] if i["image"] == "registry") + assert image == {"image": "registry", "dir": "registry", "port": 8500, "shared": False} + assert next(i for i in p["images"] if i["image"] == "gateway")["shared"] is True + + +def test_pytest_settings_run_every_suite(): + assert len(plan(["pytest.ini"])["suites"]) == 8 + + +def test_a_contract_file_for_an_unknown_provider_fails_loudly(): + import pytest + + with pytest.raises(ValueError, match="no provider"): + plan(["contracts/pacts/order-shipping.json"]) diff --git a/tests/unit/test_deploy_plan.py b/tests/unit/test_deploy_plan.py new file mode 100644 index 0000000..5499e18 --- /dev/null +++ b/tests/unit/test_deploy_plan.py @@ -0,0 +1,49 @@ +"""#9: the Deploy button rebuilds only services whose code changed since the commit they run.""" +from deploy_plan import image_size_mb, services_to_deploy + +ALL = ["catalog", "gateway", "inventory", "order", "payment", "user"] + + +def _changes(files_by_commit): + def changed_since(commit): + if commit not in files_by_commit: + raise LookupError(commit) + return files_by_commit[commit] + return changed_since + + +def test_services_already_on_this_commit_are_left_alone(): + running = {name: "abc" for name in ALL} + assert services_to_deploy(running, "abc", _changes({})) == [] + + +def test_only_the_service_that_changed_is_rebuilt(): + running = {name: "old" for name in ALL} + assert services_to_deploy(running, "new", _changes({"old": ["services/payment/application/routes.py"]})) == ["payment"] + + +def test_a_shared_change_rebuilds_everything(): + running = {name: "old" for name in ALL} + assert services_to_deploy(running, "new", _changes({"old": ["shared/shop_common/events.py"]})) == ALL + + +def test_unknown_or_unparseable_running_versions_are_rebuilt(): + """Never deployed by commit yet ('latest'), or a commit this checkout does not know.""" + running = {name: "abc" for name in ALL} | {"user": None, "order": "gone"} + assert services_to_deploy(running, "abc", _changes({})) == ["order", "user"] + + +def test_image_size_is_the_sum_of_its_layers(): + manifest = {"config": {"size": 1_000_000}, "layers": [{"size": 30_000_000}, {"size": 20_428_800}]} + assert image_size_mb(manifest) == 49 + + +def test_a_ci_only_change_rebuilds_nothing(): + running = {name: "old" for name in ALL} + changed = _changes({"old": [".github/workflows/ci.yml", "scripts/ci_changes.py", "pytest.ini"]}) + assert services_to_deploy(running, "new", changed) == [] + + +def test_a_multi_platform_index_reports_unknown_size_instead_of_crashing(): + index = {"mediaType": "application/vnd.oci.image.index.v1+json", "manifests": [{"size": 1}]} + assert image_size_mb(index) is None diff --git a/tests/unit/test_github_identity.py b/tests/unit/test_github_identity.py new file mode 100644 index 0000000..25885c6 --- /dev/null +++ b/tests/unit/test_github_identity.py @@ -0,0 +1,58 @@ +"""#8: the Azure identity GitHub Actions deploys with. These test what the script would do, +without touching Azure: the owner runs the real thing.""" +import github_identity as gi + + +def test_only_this_repos_main_branch_can_sign_in_as_it(): + cred = gi.federated_credential("MAOFILHO/Microservices-12-Patterns") + assert cred["issuer"] == "https://token.actions.githubusercontent.com" + assert cred["subject"] == "repo:MAOFILHO/Microservices-12-Patterns:ref:refs/heads/main" + assert cred["audiences"] == ["api://AzureADTokenExchange"] + + +def test_access_is_this_resource_group_only_never_the_subscription(): + scope = gi.role_scope("sub-123", "rg-microservices-12-patterns") + assert scope == "/subscriptions/sub-123/resourceGroups/rg-microservices-12-patterns" + + +NOTHING = {"app": False, "service_principal": False, "credential": False, "resource_group": True, + "role": False, "github_secrets": False} +ALL = {step: True for step in NOTHING} + + +def test_first_run_creates_everything(): + assert gi.missing_steps(NOTHING) == ["app", "service_principal", "credential", "role", "github_secrets"] + + +def test_second_run_changes_nothing(): + """Not even the GitHub secrets are rewritten.""" + assert gi.missing_steps(ALL) == [] + + +def test_a_new_app_rewrites_the_secrets_even_if_old_ones_exist(): + found = NOTHING | {"github_secrets": True} + assert "github_secrets" in gi.missing_steps(found) + + +def test_role_waits_for_the_resource_group(): + """After make teardown the group is gone; make provision grants the role again.""" + assert gi.missing_steps(ALL | {"resource_group": False, "role": False}) == [] + + +def test_role_is_granted_by_object_id_so_a_brand_new_identity_works(): + """By app id, Azure often answers "cannot find principal" right after creating the identity.""" + args = gi.role_assignment_args("sp-object-id", "/subscriptions/s/resourceGroups/rg") + assert args[args.index("--assignee-object-id") + 1] == "sp-object-id" + assert args[args.index("--assignee-principal-type") + 1] == "ServicePrincipal" + assert "--assignee" not in args + + +def test_only_the_app_with_exactly_our_name_is_ours(): + apps = [{"displayName": "ms12p-github-deploy-old", "appId": "x"}, {"displayName": "ms12p-github-deploy", "appId": "y"}] + assert gi.pick_app(apps) == "y" + assert gi.pick_app(apps[:1]) is None + + +def test_repo_is_read_from_the_git_remote(): + assert gi.repo_from_remote("https://github.com/MAOFILHO/Microservices-12-Patterns.git") == "MAOFILHO/Microservices-12-Patterns" + assert gi.repo_from_remote("git@github.com:MAOFILHO/Microservices-12-Patterns.git") == "MAOFILHO/Microservices-12-Patterns" diff --git a/tests/unit/test_pause_resume_redis.py b/tests/unit/test_pause_resume_redis.py new file mode 100644 index 0000000..ba2a1ce --- /dev/null +++ b/tests/unit/test_pause_resume_redis.py @@ -0,0 +1,16 @@ +"""#6: Redis holds undelivered events in memory, so resume must bring it back as exactly one +always-on replica, while the services may scale to zero. And provision records it for pause.""" +import provision +import resume + + +def test_resume_keeps_redis_at_exactly_one_replica(): + assert resume.replica_args("redis") == ["--min-replicas", "1", "--max-replicas", "1"] + + +def test_resume_lets_services_scale_to_zero(): + assert resume.replica_args("order")[:2] == ["--min-replicas", "0"] + + +def test_provision_records_redis_so_pause_scales_it_down(): + assert "redis" in provision.container_apps_for_state() diff --git a/tests/unit/test_provision_images.py b/tests/unit/test_provision_images.py new file mode 100644 index 0000000..817de6c --- /dev/null +++ b/tests/unit/test_provision_images.py @@ -0,0 +1,31 @@ +"""#5: Azure images are tagged with the commit they were built from, and the local-only registry +is never built for Azure. No Azure calls: these test the arguments provision would send.""" +import subprocess +from pathlib import Path + +import provision + + +def test_azure_builds_every_service_and_the_gateway_but_not_the_registry(): + assert set(provision.AZURE_SERVICES) == {"user", "catalog", "order", "payment", "inventory", "gateway"} + + +def test_image_tag_is_the_current_commit(repo_root): + head = subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + assert provision.image_tag() == head + + +def test_acr_build_tags_the_image_and_passes_the_commit_inside(): + args = provision.acr_build_args("rg", "myacr", "order", Path("/src/order"), "abc123def456") + assert args[args.index("--image") + 1] == "order:abc123def456" + assert args[args.index("--build-arg") + 1] == "GIT_SHA=abc123def456" + assert args[args.index("--platform") + 1] == "linux/amd64" + assert args[-1] == "/src/order" + + +def test_bicep_deploys_that_same_tag(): + params = provision.bicep_parameters("eastus", {"acr_name": "a", "log_analytics_name": "l", + "static_web_app_name": "w", "mysql_server_name": "m"}, + "pw", keep_warm=False, tag="abc123def456", jwt_key_b64="K") + assert "imageTag=abc123def456" in params diff --git a/tests/unit/test_provision_m3a_config.py b/tests/unit/test_provision_m3a_config.py new file mode 100644 index 0000000..88a6607 --- /dev/null +++ b/tests/unit/test_provision_m3a_config.py @@ -0,0 +1,83 @@ +"""#6: the M3a features get what they need in Azure: a bus, the inventory address, a signing key. + +Checked against the compiled template and provision's arguments. No Azure calls, no spend. +""" +import base64 +import json +import shutil +import subprocess + +import pytest + +import provision + + +def test_signing_key_is_a_fresh_rsa_private_key_in_one_line(): + first, second = provision.generate_jwt_key_b64(), provision.generate_jwt_key_b64() + assert "\n" not in first and first != second + assert base64.b64decode(first).decode().startswith("-----BEGIN PRIVATE KEY-----") + + +def test_signing_key_never_reaches_the_console(): + params = provision.bicep_parameters("eastus", {"acr_name": "a", "log_analytics_name": "l", + "static_web_app_name": "w", "mysql_server_name": "m"}, + "pw", keep_warm=False, tag="t", jwt_key_b64="SECRETKEY") + printed = " ".join(provision._redact_secrets(params)) + assert "SECRETKEY" not in printed and "pw" not in printed.split("mysqlAdminPassword=")[1][:3] + + +def test_redis_is_imported_into_our_registry_for_linux_amd64(): + args = provision.redis_import_args("rg", "myacr") + assert args[args.index("--source") + 1] == f"docker.io/library/{provision.REDIS_IMAGE}" + assert args[args.index("--image") + 1] == provision.REDIS_IMAGE + + +@pytest.fixture(scope="module") +def template(): + if shutil.which("az") is None: + pytest.skip("Azure CLI not installed") + out = subprocess.run(["az", "bicep", "build", "--file", str(provision.BICEP_FILE), "--stdout"], + capture_output=True, text=True, check=True).stdout + return json.loads(out) + + +def _app(template, name): + """The compiled module for one Container App, with its parameters as plain values.""" + for resource in template["resources"]: + if resource.get("name") == name and resource["type"] == "Microsoft.Resources/deployments": + return resource["properties"]["parameters"] + raise AssertionError(f"no module {name}") + + +def _env_names(template, name): + env = _app(template, name)["envVars"]["value"] + text = json.dumps(env) + return text + + +def test_redis_is_internal_tcp_only(template): + redis = _app(template, "redis") + assert redis["external"]["value"] is False + assert redis["transport"]["value"] == "tcp" + assert redis["targetPort"]["value"] == 6379 + + +def test_order_and_inventory_get_the_bus_and_order_gets_inventory(template): + order, inventory = _env_names(template, "order"), _env_names(template, "inventory") + assert "REDIS_URL" in order and "REDIS_URL" in inventory + assert "INVENTORY_SERVICE_BASE_URL" in order + + +def test_user_reads_its_signing_key_from_a_secret(template): + user = _app(template, "user") + assert "JWT_PRIVATE_KEY_B64" in json.dumps(user["envVars"]["value"]) + assert "secretRef" in json.dumps(user["envVars"]["value"]) + + +def test_every_service_app_gets_its_image_size(template): + for name in ["user", "catalog", "order", "payment", "inventory", "gateway"]: + assert "IMAGE_SIZE_MB" in _env_names(template, name), name + + +def test_redis_does_not_receive_the_mysql_password(template): + assert "mysqlAdminPassword" not in _app(template, "redis") diff --git a/tests/unit/test_provision_reruns.py b/tests/unit/test_provision_reruns.py new file mode 100644 index 0000000..339df0c --- /dev/null +++ b/tests/unit/test_provision_reruns.py @@ -0,0 +1,34 @@ +"""Review fixes: re-running provision must not break or log out a live stack.""" +import json + +import pytest + +import provision + + +def test_a_normal_provision_deploys_the_commit_it_just_built(): + assert provision.choose_image_tag(skip_build=False, head="new", previous="old") == "new" + + +def test_skip_build_reuses_the_images_built_last_time(): + """Pointing apps at a tag that was never built would make every image pull fail.""" + assert provision.choose_image_tag(skip_build=True, head="new", previous="old") == "old" + + +def test_skip_build_without_a_previous_build_stops_with_a_clear_message(): + with pytest.raises(SystemExit, match="without --skip-build"): + provision.choose_image_tag(skip_build=True, head="new", previous=None) + + +def test_a_rerun_keeps_the_signing_key_so_nobody_is_logged_out(): + assert provision.choose_jwt_key(previous="KEEP") == "KEEP" + assert provision.choose_jwt_key(previous=None) != provision.choose_jwt_key(previous=None) + + +def test_image_sizes_reach_every_app(): + params = provision.bicep_parameters("eastus", {"acr_name": "a", "log_analytics_name": "l", + "static_web_app_name": "w", "mysql_server_name": "m"}, + "pw", keep_warm=False, tag="t", jwt_key_b64="K", + image_sizes={"user": 61, "gateway": 70}) + sizes = next(p for p in params if p.startswith("imageSizesMb=")) + assert json.loads(sizes.split("=", 1)[1]) == {"user": 61, "gateway": 70} diff --git a/tests/unit/test_single_sources.py b/tests/unit/test_single_sources.py new file mode 100644 index 0000000..8767872 --- /dev/null +++ b/tests/unit/test_single_sources.py @@ -0,0 +1,28 @@ +"""Review fixes: one place says which services Azure runs, where they live, and the resource group.""" +import subprocess +import sys + +import github_identity +import provision +import service_table + + +def test_the_service_table_says_which_services_azure_runs(): + assert [s.name for s in service_table.AZURE_SERVICES] == ["user", "catalog", "order", "payment", "inventory", "gateway"] + assert set(provision.AZURE_SERVICES) == {s.name for s in service_table.AZURE_SERVICES} + + +def test_deploy_reads_services_and_folders_from_the_table(repo_root): + out = subprocess.run([sys.executable, "scripts/deploy_plan.py", "--services"], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.split("\n") + assert "order services/order" in out and "gateway gateway" in out and len([l for l in out if l]) == 6 + + +def test_deploy_provision_and_identity_share_one_resource_group(repo_root): + out = subprocess.run([sys.executable, "scripts/deploy_plan.py", "--resource-group"], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + assert out == provision.DEFAULT_RESOURCE_GROUP == github_identity.RESOURCE_GROUP == service_table.AZURE_RESOURCE_GROUP + + +def test_no_leftover_services_alias(): + assert not hasattr(provision, "SERVICES")