From 6942c5cc0f5efb7f628cf1efc920081757c1fa7a Mon Sep 17 00:00:00 2001 From: Patrick Hudson <5728403+patrick-hudson@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:04:35 -0500 Subject: [PATCH 1/4] feat: add EKS platform with monitoring and GitHub deployments Share Terraform roots across dev and prod workspaces, keep worker nodes private, and expose Grafana through an HTTPS ALB and Route 53. Use S3 state locking and separate GitHub OIDC plan/apply roles. Support manual reruns for one environment, guard destructive cluster changes, and document AWS costs and a resource inventory audit for teardown. --- .github/workflows/terraform.yml | 76 +++ .gitignore | 19 + .terraform-version | 1 + README.md | 477 ++++++++++++++++++ bootstrap/.terraform.lock.hcl | 26 + bootstrap/backend.hcl.example | 5 + bootstrap/backend.tf.example | 3 + bootstrap/main.tf | 84 +++ bootstrap/outputs.tf | 19 + bootstrap/terraform.tfvars.example | 9 + bootstrap/tests/bootstrap.tftest.hcl | 67 +++ bootstrap/tests/fixtures.tfvars | 1 + bootstrap/tests/pipeline.tftest.hcl | 72 +++ bootstrap/variables.tf | 35 ++ bootstrap/versions.tf | 9 + docs/cleanup.md | 279 ++++++++++ docs/verification.md | 80 +++ environments/.terraform.lock.hcl | 26 + environments/backend.hcl.example | 6 + environments/locals.tf | 3 + environments/main.tf | 33 ++ .../observability/.terraform.lock.hcl | 67 +++ .../observability/backend.hcl.example | 6 + environments/observability/locals.tf | 3 + environments/observability/main.tf | 67 +++ environments/observability/outputs.tf | 29 ++ environments/observability/terraform.tfvars | 39 ++ .../observability/tests/monitoring.tftest.hcl | 56 ++ environments/observability/variables.tf | 40 ++ environments/observability/versions.tf | 15 + environments/outputs.tf | 49 ++ environments/terraform.tfvars | 47 ++ environments/tests/platform.tftest.hcl | 52 ++ environments/variables.tf | 40 ++ environments/versions.tf | 10 + .../load-balancer-controller-policy.json | 251 +++++++++ modules/cluster/main.tf | 219 ++++++++ modules/cluster/outputs.tf | 39 ++ modules/cluster/variables.tf | 55 ++ modules/cluster/versions.tf | 9 + modules/network/main.tf | 72 +++ modules/network/outputs.tf | 19 + modules/network/variables.tf | 35 ++ modules/network/versions.tf | 9 + modules/observability/ingress.tf | 102 ++++ modules/observability/main.tf | 75 +++ modules/observability/outputs.tf | 28 + modules/observability/variables.tf | 95 ++++ modules/observability/versions.tf | 8 + modules/pipeline/main.tf | 133 +++++ modules/pipeline/outputs.tf | 9 + modules/pipeline/variables.tf | 60 +++ modules/pipeline/versions.tf | 9 + scripts/audit.py | 228 +++++++++ scripts/check.sh | 11 + scripts/deploy.sh | 70 +++ scripts/test.sh | 17 + tests/test_audit.py | 65 +++ tests/test_deploy.py | 135 +++++ 59 files changed, 3603 insertions(+) create mode 100644 .github/workflows/terraform.yml create mode 100644 .gitignore create mode 100644 .terraform-version create mode 100644 README.md create mode 100644 bootstrap/.terraform.lock.hcl create mode 100644 bootstrap/backend.hcl.example create mode 100644 bootstrap/backend.tf.example create mode 100644 bootstrap/main.tf create mode 100644 bootstrap/outputs.tf create mode 100644 bootstrap/terraform.tfvars.example create mode 100644 bootstrap/tests/bootstrap.tftest.hcl create mode 100644 bootstrap/tests/fixtures.tfvars create mode 100644 bootstrap/tests/pipeline.tftest.hcl create mode 100644 bootstrap/variables.tf create mode 100644 bootstrap/versions.tf create mode 100644 docs/cleanup.md create mode 100644 docs/verification.md create mode 100644 environments/.terraform.lock.hcl create mode 100644 environments/backend.hcl.example create mode 100644 environments/locals.tf create mode 100644 environments/main.tf create mode 100644 environments/observability/.terraform.lock.hcl create mode 100644 environments/observability/backend.hcl.example create mode 100644 environments/observability/locals.tf create mode 100644 environments/observability/main.tf create mode 100644 environments/observability/outputs.tf create mode 100644 environments/observability/terraform.tfvars create mode 100644 environments/observability/tests/monitoring.tftest.hcl create mode 100644 environments/observability/variables.tf create mode 100644 environments/observability/versions.tf create mode 100644 environments/outputs.tf create mode 100644 environments/terraform.tfvars create mode 100644 environments/tests/platform.tftest.hcl create mode 100644 environments/variables.tf create mode 100644 environments/versions.tf create mode 100644 modules/cluster/load-balancer-controller-policy.json create mode 100644 modules/cluster/main.tf create mode 100644 modules/cluster/outputs.tf create mode 100644 modules/cluster/variables.tf create mode 100644 modules/cluster/versions.tf create mode 100644 modules/network/main.tf create mode 100644 modules/network/outputs.tf create mode 100644 modules/network/variables.tf create mode 100644 modules/network/versions.tf create mode 100644 modules/observability/ingress.tf create mode 100644 modules/observability/main.tf create mode 100644 modules/observability/outputs.tf create mode 100644 modules/observability/variables.tf create mode 100644 modules/observability/versions.tf create mode 100644 modules/pipeline/main.tf create mode 100644 modules/pipeline/outputs.tf create mode 100644 modules/pipeline/variables.tf create mode 100644 modules/pipeline/versions.tf create mode 100755 scripts/audit.py create mode 100755 scripts/check.sh create mode 100755 scripts/deploy.sh create mode 100755 scripts/test.sh create mode 100644 tests/test_audit.py create mode 100644 tests/test_deploy.py diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..3be50c8 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,76 @@ +name: Terraform +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + inputs: + environment: + description: Environment to run + required: true + type: choice + default: dev + options: [dev, prod] + operation: + description: Plan only, or plan and apply + required: true + type: choice + default: plan + options: [plan, apply] +permissions: + contents: read +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.16.5 + terraform_wrapper: false + - run: bash scripts/check.sh + - run: bash scripts/test.sh + deploy: + name: Terraform (${{ matrix.target }}) + needs: validate + # Fork code never receives an OIDC session or access to state. Its validate job still runs. + if: >- + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && + (github.event_name != 'workflow_dispatch' || inputs.operation == 'plan' || github.ref == 'refs/heads/main') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: ${{ github.event_name == 'workflow_dispatch' && fromJSON(format('["{0}"]', inputs.environment)) || fromJSON('["dev", "prod"]') }} + environment: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.operation == 'plan')) && format('{0}-plan', matrix.target) || matrix.target }} + permissions: + contents: read + id-token: write + concurrency: + group: terraform-${{ matrix.target }} + cancel-in-progress: false + env: + TF_IN_AUTOMATION: 'true' + TF_INPUT: 'false' + TARGET: ${{ matrix.target }} + EVENT: ${{ github.event_name }} + OPERATION: ${{ github.event_name == 'pull_request' && 'plan' || github.event_name == 'workflow_dispatch' && inputs.operation || 'apply' }} + AWS_REGION: ${{ vars.AWS_REGION }} + STATE_BUCKET: ${{ vars.TF_STATE_BUCKET }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.16.5 + terraform_wrapper: false + - uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: terraform-${{ github.run_id }} + role-duration-seconds: 3600 + allowed-account-ids: ${{ vars.AWS_ACCOUNT_ID }} + - name: Plan or apply platform, then monitoring + shell: bash + run: bash scripts/deploy.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fdf3ba9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +**/.terraform/* +*.tfstate +*.tfstate.* +*.tfplan +crash.log +crash.*.log +**/backend.hcl +**/backend.bootstrap.hcl +bootstrap/terraform.tfvars +*.auto.tfvars +*.auto.tfvars.json +*.tfvars.local +*.tfvars.local.json +.terraformrc +terraform.rc +bootstrap/backend.tf +audit/ +__pycache__/ +*.pyc diff --git a/.terraform-version b/.terraform-version new file mode 100644 index 0000000..0d92a10 --- /dev/null +++ b/.terraform-version @@ -0,0 +1 @@ +1.16.5 diff --git a/README.md b/README.md new file mode 100644 index 0000000..b77143b --- /dev/null +++ b/README.md @@ -0,0 +1,477 @@ +Terraform for an AWS EKS platform with private worker nodes, Prometheus, public HTTPS Grafana, and GitHub OIDC deployments. + +## What gets built + +```text +GitHub Actions -- OIDC --> plan / apply IAM roles + | + S3 state + lock files + | + Terraform platform root + | + VPC across availability zones + | | + public subnets private subnets + NAT gateways EKS managed nodes + Grafana ALB -- HTTP ----> Grafana + Prometheus + ^ | + | gp3 EBS + HTTPS / ACM certificate + | + Route 53: grafana..hudson.tf + +Terraform monitoring root --> Helm / Kubernetes API +EKS control-plane logs --> CloudWatch +CNI / EBS / ALB controller --> individual IRSA roles +``` + +Dev uses two zones, two small nodes, and one NAT gateway. Prod uses three +zones, three larger nodes, and a NAT gateway in each zone. Neither cluster +places worker nodes on the public internet. The public ALB forwards only +Grafana traffic; Prometheus stays inside the cluster. + +## Repository layout + +```text +bootstrap/ state bucket and GitHub OIDC identities +modules/network/ VPC, subnets, routing and NAT +modules/cluster/ EKS, managed nodes, add-ons and IRSA +modules/observability/ monitoring, ALB controller, TLS and DNS +modules/pipeline/ scoped state access and GitHub IAM roles +environments/ shared platform root + terraform.tfvars platform settings keyed by dev/prod workspace + observability/ shared monitoring root + terraform.tfvars monitoring settings keyed by dev/prod workspace +.github/workflows/terraform.yml validation, PR plans and main applies +scripts/ local checks and workflow deployment +``` + +Both roots use Terraform CLI workspaces named `dev` and `prod`. All environment +differences live in each root's `terraform.tfvars`, which Terraform loads +automatically. Each file contains an `environments` map; the root selects +`environments[terraform.workspace]`. Changing workspace selects both state and +settings, so plan and destroy need no `-var-file`. Variable validation rejects +`default` and any workspace without configured settings. + +The states are separate because Helm needs an existing Kubernetes API, valid +credentials, and ready nodes. A fresh deployment applies the platform before +monitoring. Destruction runs in the opposite order. No targeted apply is needed. + +## Before you run it + +You need Terraform 1.16.5, AWS CLI v2, kubectl compatible with the configured +Kubernetes version, and an AWS account with room for the nodes, NAT gateways, +and load balancers. Bash, jq, and Python 3 are used by the scripts; the Python code uses only the +standard library. Helm is only +needed if you want to inspect charts yourself; Terraform installs the releases. + +Use an AWS profile or SSO session with permission to bootstrap IAM and S3 and +to provision the platform. Configure a persistent IAM administrator role in +`administrator_arns` before applying; use its IAM ARN, not an STS session ARN. +The cluster does not grant implicit administrator access to its creator. + +The public `hudson.tf` hosted zone must already exist in the same account, with +its nameservers delegated correctly. Set its exact `hosted_zone_id` in bootstrap +and both monitoring tfvars files. +Terraform selects that ID and checks the zone name and public-zone setting +before adding Grafana alias records and ACM validation records. It does not create or replace +the zone or register the domain. Use unused Grafana hostnames; existing records +need to be imported or moved deliberately before this configuration owns them. + +Review both sets of tfvars. The shipped values use one AWS region; if you change +it, update the AZs, backend region, bootstrap inputs, and GitHub variables as +well. Verify that the configured Kubernetes version is available and in +[standard EKS support](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html). +The control plane uses the STANDARD upgrade policy, so AWS can upgrade it at +the end of standard support; schedule explicit upgrades before that happens. + +## Run dev first + +Start each shell block below at the repository root. The commands enter the +relevant Terraform folder and return to the root when finished. They provision +billable AWS resources. Read the [cost breakdown](#cost) before leaving a +cluster running. + +### Bootstrap state and CI roles + +Authenticate your AWS profile first using your configured login method +(`aws login` or `aws sso login`), then select it for this shell: + +```bash +export AWS_PROFILE=YOUR_PROFILE +aws sts get-caller-identity +cp bootstrap/terraform.tfvars.example bootstrap/terraform.tfvars +``` + +Edit that local tfvars file: choose a globally unique state bucket, set your +GitHub `owner/repository`, and confirm the cluster and role names match the +environment inputs. Confirm that `hosted_zone_id` identifies the zone whose +name servers are delegated for your domain; duplicate hosted-zone names are +allowed in AWS. If this account already has GitHub's OIDC provider, set +`github_oidc_provider_arn` to its ARN. There is only one provider per issuer per +account; this configuration can reuse it. + +```bash +cd bootstrap +terraform init +terraform plan -out=bootstrap.tfplan +terraform apply bootstrap.tfplan +terraform output +cd .. +``` + +Bootstrap starts with local state because the bucket does not exist yet. +Move that state into the bucket once it has been created: + +```bash +cp bootstrap/backend.tf.example bootstrap/backend.tf +cp bootstrap/backend.hcl.example bootstrap/backend.hcl +# Edit bootstrap/backend.hcl with the bucket output and region. +cd bootstrap +terraform init -migrate-state -backend-config=backend.hcl +cd .. +``` + +Confirm the migration prompt. Secure any local backup state left by the +migration. Bootstrap state lives at `bootstrap/terraform.tfstate`; environment +roles cannot read or modify it. Bucket deletion is blocked with `prevent_destroy`. +S3 versioning, encryption, TLS-only access, and public-access blocking are enabled. + +### Initialize the two roots + +```bash +cp environments/backend.hcl.example environments/backend.hcl +cp environments/observability/backend.hcl.example environments/observability/backend.hcl +# Set the same bucket and region in both files. Keep their distinct keys. +cd environments +terraform init -backend-config=backend.hcl +terraform workspace select -or-create dev +terraform workspace select -or-create prod +terraform workspace select dev +cd observability +terraform init -backend-config=backend.hcl +terraform workspace select -or-create dev +terraform workspace select -or-create prod +terraform workspace select dev +cd ../.. +``` + +Create both workspaces using your bootstrap credentials before enabling PR +plans. A read-only plan role cannot initialize an empty workspace state. The +backend uses native S3 locking with `use_lockfile = true`, without DynamoDB. +The [S3 backend documentation](https://developer.hashicorp.com/terraform/language/backend/s3) +explains lock permissions and workspace paths. + +| Workspace | Platform state | Monitoring state | +| --- | --- | --- | +| dev | `environments/dev/platform.tfstate` | `environments/dev/observability.tfstate` | +| prod | `environments/prod/platform.tfstate` | `environments/prod/observability.tfstate` | + +### Plan and apply dev + +In `environments/terraform.tfvars`, set `administrator_arns` inside the +`environments.dev` object to your human administrator's persistent IAM ARN: + +```hcl +administrator_arns = { + operator = "REPLACE_WITH_YOUR_PERSISTENT_IAM_ROLE_ARN" +} +``` + +Then apply using that role's AWS session: + +```bash +cd environments +terraform workspace select dev +terraform plan -out=dev.tfplan +terraform apply dev.tfplan +cd observability +terraform workspace select dev +terraform plan -out=dev.tfplan +terraform apply dev.tfplan +cd ../.. +``` + +The second apply waits for Helm, certificate validation, and ALB reconciliation. +An access-denied error is a reason to check your IAM principal and its EKS access +entry. Do not rerun monitoring with an unrelated administrator identity. + +### Open Grafana and check the cluster + +```bash +cd environments +cluster_region=$(terraform output -raw region) +cluster_name=$(terraform output -raw cluster_name) +aws eks update-kubeconfig --region "$cluster_region" --name "$cluster_name" +kubectl get nodes +kubectl -n monitoring get pods,pvc,ingress +cd observability +terraform output -raw grafana_url +kubectl -n monitoring get secret monitoring-grafana \ + -o jsonpath='{.data.admin-password}' | base64 --decode +cd ../.. +``` + +Log in as `admin` with that generated password. The dev URL is +`https://grafana.dev.hudson.tf`; prod uses `https://grafana.prod.hudson.tf`. +HTTP redirects to HTTPS. Anonymous access and account sign-up are disabled. +Keep the password private and change it after first login. Prometheus provides +node, pod, and Kubernetes object metrics; Grafana ships the chart's dashboards. + +The API endpoint is public with IAM authentication, and private access is also +enabled. The sample API CIDRs allow GitHub-hosted runners with changing egress +addresses. For a fixed-egress runner, restrict `public_access_cidrs` in platform +tfvars. `ingress_cidrs` separately controls who can reach Grafana's ALB. +For a long-lived shared Grafana deployment, add SSO before giving more people access. + +### Run prod + +Select `prod` in both roots and configure `environments.prod.administrator_arns` +in the platform tfvars. Use the same plan and apply commands; the selected +workspace supplies the prod settings. Name the saved plans `prod.tfplan`. +Do not copy Terraform code or change modules to deploy prod. + +## GitHub Actions + +Create GitHub environments `dev-plan`, `prod-plan`, `dev`, and `prod`. Each needs +these environment variables; they are configuration, not stored AWS secrets: + +| Variable | Value | +| --- | --- | +| `AWS_ROLE_ARN` | Corresponding role from bootstrap's `github_role_arns` output | +| `AWS_ACCOUNT_ID` | Account allowed by the AWS credentials action | +| `AWS_REGION` | Region matching tfvars and the state bucket | +| `TF_STATE_BUCKET` | Bootstrap state bucket output | + +Restrict deployment branches on `dev` and `prod` to `main`. Require reviewers +for `prod`. Restrict who can approve same-repository PRs against `dev-plan` and +`prod-plan`: plan roles can read state and Kubernetes Secrets to refresh Helm +releases. The plan environments must allow PR refs, not just `main`. Protect +`main`, require the validate job, and require review for changes to workflows, +Terraform, and IAM policies. Environment branch restrictions are essential: +the OIDC environment subject does not itself prove which branch requested it. + +Every PR runs formatting, validation, and offline plan tests. Same-repository +PRs also plan both environments using the read-only roles. Fork PRs run the +credential-free checks only. No `pull_request_target` execution is used. + +For an environment without a cluster yet, the platform plan runs and the live +monitoring plan is explicitly deferred. Static validation and offline monitoring +plans still run. Once a cluster exists, both live plans run. PR plans are unlocked +because their roles cannot write lock files; they can be stale. Main applies +create new saved plans under S3 locks and apply those exact files in the same job. +PR plan artifacts are never reused for deployment or published as artifacts. + +A merge to `main` applies platform then monitoring for both environments. +Manual runs select one environment and either `plan` or `apply`; they default +to a dev plan. Manual applies require `main`. Deployment jobs serialize per environment and +are never canceled mid-apply. The prod environment approval sits before AWS +credentials are requested. This is not a sequential dev-to-prod promotion gate. + +After the workflow is merged into `main`, use Actions > Terraform > Run workflow +to select the environment and operation, or run: + +```bash +gh workflow run terraform.yml --ref main -f environment=dev -f operation=plan +gh workflow run terraform.yml --ref main -f environment=prod -f operation=apply +``` + +Manual plans use the corresponding `dev-plan` or `prod-plan` environment and +read-only role. Manual applies use `dev` or `prod`, including its approval rules. +To retry a failed PR job before merging, use GitHub's Re-run failed jobs button +or `gh run rerun RUN_ID --failed`. To retry just one environment job, find its +`databaseId` with `gh run view RUN_ID --json jobs --jq '.jobs[] | {name, databaseId}'`, +then run `gh run rerun RUN_ID --job JOB_ID` using that ID. +Plans do not take a state lock, so an active local apply does not block them; +wait until the apply finishes for a reliable result. Applies wait up to five +minutes for the S3 lock. Retry after the other apply finishes if that wait expires. + +CI refuses plans that delete or replace the EKS cluster. Those changes need +monitoring-first cleanup and an AWS audit while its controllers still run; +follow [the cleanup procedure](docs/cleanup.md) before changing the cluster's +identity or other settings that require replacement. Initial creation and +in-place updates run normally. + +The apply roles can manage EC2 and EKS resources in the configured region, +cluster-prefixed IAM roles, and records in the chosen hosted zone. Direct IAM grants target cluster-prefixed roles, and state-object deletion is +not granted; lock files can be deleted. This is not a permissions boundary: +creating roles and attaching policies can allow privilege escalation beyond +those direct grants, including access to bootstrap identities. Treat apply +workflows and their approvers as fully trusted within this AWS account. These +are privileged infrastructure roles, not least-privilege application roles. Workspaces separate state, not accounts or failure domains. At larger +scale, use separate accounts and backends and narrow IAM permissions further. + +## Design decisions + +**Managed node groups.** EKS manages node registration, replacement, and rolling +updates. That removes work that would otherwise live in custom launch scripts +and ASG lifecycle hooks. On-demand capacity keeps interruption handling out of +this baseline. Dev uses burstable nodes; sustained CPU loads need different +instance types or incur CPU credit charges. Node counts are explicit: increasing +`max` does not install an autoscaler or automatically grow the cluster. + +**OIDC rather than stored AWS keys.** GitHub gets a short-lived session tied to +this repository and an exact GitHub environment. There is no IAM user access key +to rotate or leak in repository secrets. Plan and apply roles are separate, and +IRSA gives the CNI, EBS CSI driver, and ALB controller separate AWS identities. +IMDSv2 is required, with a hop limit that keeps normal pods from relying on node +credentials. The ALB controller receives its region and VPC explicitly. + +**Public and private subnets.** Nodes need outbound access for images and AWS +APIs but do not need public IPs. NAT provides that path. Dev shares one NAT to +save money, accepting an availability dependency and cross-zone traffic charges. +Prod routes each private subnet to its local NAT. Public subnets also host the +Grafana ALB. The controller is the standard AWS controller, not a custom operator. + +**S3 locking and workspaces.** S3 now supports Terraform lock files directly. +This avoids a separate lock database. Workspaces give the same code distinct +states; auto-loaded tfvars maps supply settings for the selected workspace. +Platform and monitoring are +separate roots because their provider dependencies and teardown order differ. +They look up the existing cluster through AWS, rather than granting monitoring +access to the entire platform state. + +**Monitoring storage and ingress.** Prometheus and Grafana use encrypted gp3 +volumes with delayed binding to a node's zone. Retention is bounded by time and +size. Neither deployment is replicated; a node failure can interrupt monitoring +while a replacement attaches the zonal volume. Grafana has a public TLS endpoint +because this reference includes internet access. The ALB terminates TLS and +forwards HTTP inside the VPC. Prometheus is not exposed, and alert routing is +not configured. Managed control-plane targets that EKS does not expose are +disabled in the chart instead of leaving permanently failing scrape targets. + +**What changes at larger scale.** Separate AWS accounts for prod, private API +access from a VPC runner, SSO for Grafana, a permissions boundary on deployment +roles, and stronger DNS ownership controls come first. Add autoscaling when the +workload requires it. Move metrics to durable remote storage when retention or +availability requirements exceed a single Prometheus volume. Add alert receivers, +backup and restore tests, application ingress policies, and budget alerts with +an actual owner. Measure traffic before adding paid VPC endpoints; an S3 gateway +endpoint is a useful first step to reduce NAT traffic. + +Provider versions and Helm charts are pinned. EKS add-ons use AWS's compatible +default for the configured minor version. That default can change; inspect its +version in every plan. Chart upgrades need their upstream upgrade notes, +particularly Prometheus CRD changes. The checked-in controller IAM policy is +from [AWS Load Balancer Controller v2.14.1](https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/v2.14.1/docs/install/iam_policy.json) +and must be reviewed alongside the controller chart pin. + +## Cost + +Estimate checked on October 14, 2026, for Linux on-demand instances in the sample +region, 730 hours per month, and the desired node counts. No free-tier credits, +Savings Plans, taxes, or support plan are included. Monitoring runs on the listed +nodes; there are no extra EC2 monitoring servers. + +| Component | Dev / month | Prod / month | Basis | +| --- | ---: | ---: | --- | +| EKS control plane | $73.00 | $73.00 | One cluster at $0.10/hour | +| Worker nodes | $60.74 | $210.24 | Two at $0.0416/hour; three at $0.096/hour | +| Node root disks | $4.80 | $7.20 | 30 GiB gp3 per node at $0.08/GiB-month | +| Monitoring disks | $1.20 | $2.80 | 15 / 35 GiB total gp3 | +| NAT gateways | $32.85 | $98.55 | One / three at $0.045/hour | +| NAT public IPv4 | $3.65 | $10.95 | One / three at $0.005/IP-hour | +| Grafana ALB | $16.43 | $16.43 | One at $0.0225/hour | +| ALB public IPv4 | $7.30 | $10.95 | At least one address per enabled zone: two / three | +| ALB capacity allowance | $5.84 | $5.84 | Budget assumption: average one LCU at $0.008/LCU-hour | +| ACM certificate | $0 | $0 | Non-exportable public certificate used by ALB | +| Route 53 alias record | $0 additional | $0 additional | Existing zone; alias queries to ELB are free | +| **Estimated subtotal** | **$205.80** | **$435.96** | Before usage costs below | + +The existing `hudson.tf` hosted zone costs $0.50/month once, not per environment; +it is already being paid for. Domain registration and renewal are separate. +State storage and S3 requests are usually well under $1/month for this project. +CloudWatch control-plane log ingestion and storage depend on traffic; allow a +few dollars initially and measure it. A reasonable light-use budget is roughly +**$210–225/month for dev** and **$440–465/month for prod**, or **$650–690/month +for both**. These are budget estimates, not limits or a bill guarantee. + +For a short verification run, use these subtotal estimates: + +| Runtime | Dev | Prod | Both | +| --- | ---: | ---: | ---: | +| 3 hours | $0.85 | $1.79 | $2.64 | +| 6 hours | $1.69 | $3.58 | $5.27 | +| 24 hours | $6.77 | $14.33 | $21.10 | + +Allow **$2–3 for a short dev test** or **$8–10 for a day of dev**, including some +startup downloads and logs. Those allowances assume light traffic and successful +cleanup; they are not caps. Timing starts when AWS resources are provisioned, +including time spent waiting for a deployment or teardown. + +Public Grafana adds about **$29.57/month in dev** or **$33.22/month in prod**, +including the one-LCU allowance and minimum public IPs. Capacity and IP counts +can grow under load. Without the LCU allowance, the fixed ingress floor is about +$23.73 / $27.38 per month. A 24-hour dev run is about **$6.77** at the subtotal +rate; prod is about **$14.33**, before variable charges. There is no separate +one-time EKS provisioning fee. Deployment, idle time, and teardown all count. + +Add $0.045 for each GB processed by NAT, plus applicable internet and cross-zone +data transfer. Image pulls during first startup use NAT too. Dev's shared NAT +makes cross-zone charges more likely. Burstable CPU credit use can add EC2 +charges. Extra nodes cost their instance rate and root disk storage. Retained +PVCs, snapshots, or ALBs left after a failed teardown continue to cost money. +If a cluster enters extended Kubernetes support, EKS's price rises to +$0.60/hour, adding $365/month; this configuration opts into STANDARD support. + +Sources: [EKS](https://aws.amazon.com/eks/pricing/), +[EC2 Linux regional rate data](https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/ec2/USD/current/ec2-ondemand-without-sec-sel/US%20East%20(N.%20Virginia)/Linux/index.json), +[EBS](https://aws.amazon.com/ebs/pricing/), +[NAT and public IPv4](https://aws.amazon.com/vpc/pricing/), +[ALB](https://aws.amazon.com/elasticloadbalancing/pricing/), +[ACM](https://aws.amazon.com/certificate-manager/pricing/), and +[Route 53](https://aws.amazon.com/route53/pricing/). + +## Checks + +```bash +bash scripts/check.sh +bash scripts/test.sh +``` + +Checks install the locked providers, format-check the repo, validate every root, +and run offline Terraform plans with AWS lookups replaced by fixtures. They do +not contact AWS or create resources. Helm chart rendering is a separate local +check described in [verification](docs/verification.md). Live permissions, +quotas, DNS delegation, cluster readiness, and a second no-change AWS plan must +be checked during the first real deployment. Static validation cannot prove those. + +## Tear it down + +Use the [full cleanup and API audit runbook](docs/cleanup.md) to capture an +inventory before teardown and verify that billable resources are gone afterward. +The brief commands below are only the normal destruction sequence. + +Select the same workspace in both roots. Save anything you want to keep from +Grafana and Prometheus, then destroy monitoring before the platform: + +```bash +cd environments/observability +terraform workspace select dev +terraform destroy +cd .. +terraform workspace select dev +terraform destroy +cd .. +``` + +The ingress is deleted before its controller. Namespace deletion removes any +Prometheus PVCs retained by Helm; the storage class uses a Delete reclaim policy. +Before removing the cluster, confirm that the ALB and monitoring EBS volumes +are gone. A failed controller cleanup can leave AWS resources behind. If that +happens, fix it while the cluster and controller still exist. Do not delete the +cluster first and assume the finalizers will finish later. + +The state bucket, bootstrap roles, and existing hosted zone remain. Keep state +versions until you no longer need recovery. Removing bootstrap protection is an +explicit cleanup task, not part of destroying dev or prod. + +## What this is not + +This is a small platform reference, not a production service or a claim that +workspaces isolate tenants. It has no application deployment, service mesh, +GitOps controller, custom operators, autoscaler, remote metrics storage, +monitoring high availability, disaster recovery, or staffed alert response. +The prod inputs improve node and NAT capacity; they do not create those missing +operating practices. Grafana's password login is a starting point, not a complete +identity system for a team. diff --git a/bootstrap/.terraform.lock.hcl b/bootstrap/.terraform.lock.hcl new file mode 100644 index 0000000..3622c58 --- /dev/null +++ b/bootstrap/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.67.0" + constraints = "~> 6.67.0" + hashes = [ + "h1:EB9ixYOZrSlYD7wtJxf88qwoyyWrlKDKxhzaCLIb3t4=", + "zh:111d5686a1f4ccbc888bb5e2229308bcdd9149898c6af97969fcdfb0e7bd2aa0", + "zh:21b3b7693bd9754c039fd546f03ed09e7510c6189aa5ee37176f144cf8dd5f95", + "zh:25e03c7f025ff4851889537605aa560d2e39bd738b33a5204b8037eb164b475f", + "zh:2817a046f1060e25bf04b0eb78f4e251130bb92dc82ec1264ce156dc9bf78e67", + "zh:2d318d674c3ec9dbd97ddb10b12ad4827be4f508c43ba2ae1e0523baa9e367e3", + "zh:2f07ab356718267299e10b6072472ded29d82753883027bf43bcd687ac72feb2", + "zh:32307e67f83bc0dc94d38cfcd23d782166a09e0e975e2a5aa7d7a3e7ca5d3da8", + "zh:4ce94853264097dd7f4542b3cfd18d2b98b06d23321db45d5e384ea275a57f63", + "zh:869656c41cc7c7412f213e488f98167cff61deafcb8cf245d25d056e8dfdc263", + "zh:8fd8c814e9d8edf152552047db23bf5b5d63f22e6b0b5d47b2af851376e99349", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9cbc02ceef9bd469da497a1e7065ff984bdacfbe919ac3cce804a86c13b6e2c6", + "zh:9ea55dda2767acfc1f6337fc7d29b1d4d79d6f8f04871f8ad99a6078d2865994", + "zh:d7149df0819fb57a160489db45d33951765b8f0118daa3b4cd549a0135bc97a1", + "zh:e5819937bb7043d08c9829b32d52d9fb55a5b9a4d8d55d550d47a3d7392db546", + "zh:f93bc38dbc0ad53842303f30eec7a22c525c4d56209b54ec33a8d375cfc4e4fd", + ] +} diff --git a/bootstrap/backend.hcl.example b/bootstrap/backend.hcl.example new file mode 100644 index 0000000..123c8b8 --- /dev/null +++ b/bootstrap/backend.hcl.example @@ -0,0 +1,5 @@ +bucket = "REPLACE-WITH-BOOTSTRAP-BUCKET" +key = "bootstrap/terraform.tfstate" +region = "us-east-1" +encrypt = true +use_lockfile = true diff --git a/bootstrap/backend.tf.example b/bootstrap/backend.tf.example new file mode 100644 index 0000000..12c0dbe --- /dev/null +++ b/bootstrap/backend.tf.example @@ -0,0 +1,3 @@ +terraform { + backend "s3" {} +} diff --git a/bootstrap/main.tf b/bootstrap/main.tf new file mode 100644 index 0000000..a872673 --- /dev/null +++ b/bootstrap/main.tf @@ -0,0 +1,84 @@ +provider "aws" { + region = var.region + default_tags { tags = { environment = "bootstrap", "managed-by" = "terraform", project = "eks-platform-reference" } } +} +resource "aws_s3_bucket" "state" { + bucket = var.state_bucket_name + lifecycle { prevent_destroy = true } +} +resource "aws_s3_bucket_versioning" "state" { + bucket = aws_s3_bucket.state.id + versioning_configuration { status = "Enabled" } +} +resource "aws_s3_bucket_server_side_encryption_configuration" "state" { + bucket = aws_s3_bucket.state.id + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} +resource "aws_s3_bucket_public_access_block" "state" { + bucket = aws_s3_bucket.state.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} +data "aws_iam_policy_document" "state" { + statement { + effect = "Deny" + actions = ["s3:*"] + resources = [aws_s3_bucket.state.arn, "${aws_s3_bucket.state.arn}/*"] + principals { + type = "*" + identifiers = ["*"] + } + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} +resource "aws_s3_bucket_policy" "state" { + bucket = aws_s3_bucket.state.id + policy = data.aws_iam_policy_document.state.json +} +resource "aws_iam_openid_connect_provider" "github" { + count = var.github_oidc_provider_arn == null ? 1 : 0 + url = "https://token.actions.githubusercontent.com" + client_id_list = ["sts.amazonaws.com"] +} +data "aws_route53_zone" "public" { + zone_id = var.hosted_zone_id + private_zone = false + lifecycle { + postcondition { + condition = !self.private_zone && lower(trimsuffix(self.name, ".")) == lower(trimsuffix(var.domain_name, ".")) + error_message = "The selected hosted zone must be public and match domain_name." + } + } +} +locals { + github_oidc_provider_arn = var.github_oidc_provider_arn != null ? var.github_oidc_provider_arn : aws_iam_openid_connect_provider.github[0].arn + roles = merge( + { for env, config in var.environments : "${env}-apply" => { environment = env, github_environment = env, role_name = config.deploy_role_name, cluster_name = config.cluster_name, read_only = false } }, + { for env, config in var.environments : "${env}-plan" => { environment = env, github_environment = "${env}-plan", role_name = config.plan_role_name, cluster_name = config.cluster_name, read_only = true } } + ) +} +module "pipeline" { + for_each = local.roles + source = "../modules/pipeline" + environment = each.value.environment + github_repository = var.github_repository + github_environment = each.value.github_environment + oidc_provider_arn = local.github_oidc_provider_arn + role_name = each.value.role_name + cluster_name = each.value.cluster_name + region = var.region + state_bucket_arn = aws_s3_bucket.state.arn + state_prefix = "environments/${each.value.environment}/" + hosted_zone_id = data.aws_route53_zone.public.zone_id + read_only = each.value.read_only +} diff --git a/bootstrap/outputs.tf b/bootstrap/outputs.tf new file mode 100644 index 0000000..e03526e --- /dev/null +++ b/bootstrap/outputs.tf @@ -0,0 +1,19 @@ +output "state_bucket_name" { + description = "S3 backend bucket." + value = aws_s3_bucket.state.id +} + +output "github_role_arns" { + description = "Apply and plan role ARNs by environment." + value = { for k, role in module.pipeline : k => role.role_arn } +} + +output "github_oidc_provider_arn" { + description = "Shared GitHub OIDC provider ARN." + value = local.github_oidc_provider_arn +} + +output "hosted_zone_id" { + description = "Public hosted zone selected for ingress and certificate validation." + value = data.aws_route53_zone.public.zone_id +} diff --git a/bootstrap/terraform.tfvars.example b/bootstrap/terraform.tfvars.example new file mode 100644 index 0000000..bd52463 --- /dev/null +++ b/bootstrap/terraform.tfvars.example @@ -0,0 +1,9 @@ +region = "us-east-1" +state_bucket_name = "REPLACE-WITH-A-GLOBALLY-UNIQUE-BUCKET" +github_repository = "REPLACE-OWNER/eks-platform-reference" +environments = { + dev = { cluster_name = "eks-platform-reference-dev", deploy_role_name = "github-eks-platform-dev-apply", plan_role_name = "github-eks-platform-dev-plan" } + prod = { cluster_name = "eks-platform-reference-prod", deploy_role_name = "github-eks-platform-prod-apply", plan_role_name = "github-eks-platform-prod-plan" } +} +domain_name = "hudson.tf" +hosted_zone_id = "Z0992412PWY8TUM0P6ZU" diff --git a/bootstrap/tests/bootstrap.tftest.hcl b/bootstrap/tests/bootstrap.tftest.hcl new file mode 100644 index 0000000..62de681 --- /dev/null +++ b/bootstrap/tests/bootstrap.tftest.hcl @@ -0,0 +1,67 @@ +variable "region" { type = string } +provider "aws" { + region = var.region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +override_data { + target = data.aws_route53_zone.public + values = { zone_id = "ZOFFLINETEST", name = "offline.invalid", private_zone = false } +} +override_data { + target = module.pipeline["dev-plan"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["dev-plan"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.pipeline["dev-apply"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["dev-apply"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.pipeline["prod-plan"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["prod-plan"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.pipeline["prod-apply"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["prod-apply"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +run "bootstrap_plan" { + command = plan + variables { + state_bucket_name = "offline-eks-reference-state" + github_repository = "offline/eks-platform-reference" + github_oidc_provider_arn = "arn:aws:iam::111122223333:oidc-provider/token.actions.githubusercontent.com" + domain_name = "offline.invalid" + hosted_zone_id = "ZOFFLINETEST" + environments = { + dev = { cluster_name = "offline-dev", deploy_role_name = "offline-dev-apply", plan_role_name = "offline-dev-plan" } + prod = { cluster_name = "offline-prod", deploy_role_name = "offline-prod-apply", plan_role_name = "offline-prod-plan" } + } + } + assert { + condition = aws_s3_bucket_versioning.state.versioning_configuration[0].status == "Enabled" && aws_s3_bucket_public_access_block.state.block_public_policy && aws_s3_bucket_public_access_block.state.block_public_acls + error_message = "State must have versioning and public access blocking." + } + assert { + condition = alltrue([for name, role in module.pipeline : role.oidc_subject == "repo:offline/eks-platform-reference:environment:${replace(name, "-apply", "")}"]) + error_message = "Bootstrap must wire each CI role to its matching GitHub environment." + } +} diff --git a/bootstrap/tests/fixtures.tfvars b/bootstrap/tests/fixtures.tfvars new file mode 100644 index 0000000..0d253dd --- /dev/null +++ b/bootstrap/tests/fixtures.tfvars @@ -0,0 +1 @@ +region = "us-east-1" diff --git a/bootstrap/tests/pipeline.tftest.hcl b/bootstrap/tests/pipeline.tftest.hcl new file mode 100644 index 0000000..f958fc2 --- /dev/null +++ b/bootstrap/tests/pipeline.tftest.hcl @@ -0,0 +1,72 @@ +variable "region" { type = string } +provider "aws" { + region = var.region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +override_data { + target = data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +variables { + environment = "dev" + github_repository = "offline/eks-platform-reference" + github_environment = "dev-plan" + oidc_provider_arn = "arn:aws:iam::111122223333:oidc-provider/token.actions.githubusercontent.com" + role_name = "offline-dev-plan" + cluster_name = "offline-dev" + state_bucket_arn = "arn:aws:s3:::offline-eks-reference-state" + state_prefix = "environments/dev/" + read_only = true + hosted_zone_id = "ZOFFLINETEST" +} +run "plan_role_trust" { + command = plan + module { source = "../modules/pipeline" } + assert { + condition = ( + length(jsondecode(aws_iam_role.this.assume_role_policy).Statement) == 1 && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Effect == "Allow" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Action == "sts:AssumeRoleWithWebIdentity" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Principal.Federated == var.oidc_provider_arn && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Condition == { + StringEquals = { + "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" + "token.actions.githubusercontent.com:sub" = "repo:offline/eks-platform-reference:environment:dev-plan" + } + } + ) + error_message = "The actual plan role must trust only the exact GitHub environment, audience and provider." + } +} +run "apply_role_trust" { + command = plan + module { source = "../modules/pipeline" } + variables { + github_environment = "dev" + role_name = "offline-dev-apply" + read_only = false + } + assert { + condition = ( + length(jsondecode(aws_iam_role.this.assume_role_policy).Statement) == 1 && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Effect == "Allow" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Action == "sts:AssumeRoleWithWebIdentity" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Principal.Federated == var.oidc_provider_arn && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Condition == { + StringEquals = { + "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" + "token.actions.githubusercontent.com:sub" = "repo:offline/eks-platform-reference:environment:dev" + } + } + ) + error_message = "The actual apply role must trust only the exact GitHub environment, audience and provider." + } +} diff --git a/bootstrap/variables.tf b/bootstrap/variables.tf new file mode 100644 index 0000000..6b70a4f --- /dev/null +++ b/bootstrap/variables.tf @@ -0,0 +1,35 @@ +variable "region" { + description = "AWS region for state and locking." + type = string +} + +variable "state_bucket_name" { + description = "Globally unique S3 state bucket name." + type = string +} + +variable "github_repository" { + description = "GitHub owner/repository that will run the workflows." + type = string +} + +variable "github_oidc_provider_arn" { + description = "Existing account GitHub provider ARN; null creates it here." + type = string + default = null +} + +variable "environments" { + description = "Environment names and the cluster and CI role names for each." + type = map(object({ cluster_name = string, deploy_role_name = string, plan_role_name = string })) +} + +variable "domain_name" { + description = "Existing public Route 53 hosted zone name." + type = string +} + +variable "hosted_zone_id" { + description = "Exact existing public Route 53 zone ID; zone names are not unique." + type = string +} diff --git a/bootstrap/versions.tf b/bootstrap/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/bootstrap/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/docs/cleanup.md b/docs/cleanup.md new file mode 100644 index 0000000..a9fd532 --- /dev/null +++ b/docs/cleanup.md @@ -0,0 +1,279 @@ +# Cleanup and audit + +This procedure removes an environment and checks AWS for resources that still +exist. It also covers retiring the shared backend and CI identities when the +whole reference is finished. The existing `hudson.tf` hosted zone and domain +are outside this repo's ownership and must remain. + +Each shell block starts at the repository root, enters the relevant Terraform +folder, and returns to the root when finished. + +Use this sequence before deliberately replacing the cluster as well. Keep the +currently deployed cluster settings until monitoring is destroyed and audited; +changing the cluster name first makes Helm and Kubernetes look up a different +API. CI stops when a platform plan would delete or replace EKS. + +A successful `terraform destroy` is one piece of evidence. It does not prove +that controller-created load balancers, retained disks, or manually detached +resources are gone. Use the AWS audit after destroy. It is read-only; it never +deletes anything or treats an access-denied response as proof of deletion. + +## 1. Stop deployments and choose the scope + +Disable the Terraform workflow in GitHub before cleanup. Wait for any running +apply to finish; do not cancel an apply and then launch a destroy against it. +Decide whether you are removing dev, prod, or both. If keeping an environment, +keep bootstrap and its state bucket. Work from a fresh AWS session that can +read the relevant AWS services and destroy the environment. The audit also +needs IAM reads and permission to describe snapshots and load balancer tags. + +For dev, select the workspace in both roots and capture its identity: + +```bash +export AUDIT_ENV=dev +cd environments +terraform workspace select "$AUDIT_ENV" +export AUDIT_REGION=$(terraform output -raw region) +export AUDIT_CLUSTER=$(terraform output -raw cluster_name) +terraform workspace show +cd observability +terraform workspace select "$AUDIT_ENV" +export AUDIT_ZONE=$(terraform output -raw hosted_zone_id) +terraform workspace show +cd ../.. +aws sts get-caller-identity +``` + +Set `AUDIT_ENV=prod` when removing prod. Recheck +these values before each plan. The same region must be used for every regional +AWS audit call. IAM is account-wide. + +## 2. Save the inventory before deleting anything + +```bash +umask 077 +python3 scripts/audit.py snapshot \ + --environment "$AUDIT_ENV" --region "$AUDIT_REGION" \ + --cluster-name "$AUDIT_CLUSTER" --snapshot "audit/$AUDIT_ENV-before.json" +cd environments +terraform state list > "../audit/$AUDIT_ENV-platform-state.txt" +cd observability +terraform state list > "../../audit/$AUDIT_ENV-monitoring-state.txt" +terraform show -json > "../../audit/$AUDIT_ENV-monitoring-before.json" +kubectl -n monitoring get ingress,pvc -o yaml > "../../audit/$AUDIT_ENV-kubernetes.yaml" +cd ../.. +``` + +Refresh kubeconfig for this cluster first if needed. The snapshot records its +account, region, cluster name, VPC children, EC2 instances, NAT gateways, EIPs, +EBS volumes, launch templates, load balancers, target groups, control-plane log +groups, IAM roles, IRSA providers, and certificates. It includes tagged EBS +snapshots if any were made manually. These files contain infrastructure metadata +and should remain private; `audit/` is ignored by git. + +The script uses AWS CLI pagination. It finds VPC children even without ownership +tags and independently scans resources tagged `project=eks-platform-reference` +and the selected `environment`. Captured resource IDs are checked after teardown +even if their tags were removed. A resource created later with no ownership +tags and outside the captured VPC is outside this audit's detection boundary. +Keep deployments disabled so the inventory stays useful. + +## 3. Destroy monitoring while its controllers still run + +Back up any dashboards or metrics you intend to keep. The monitoring disks +will be deleted. Inspect the destruction plan before executing it: + +```bash +cd environments/observability +terraform workspace show +terraform plan -destroy -out=cleanup.tfplan +terraform apply cleanup.tfplan +terraform state list +cd ../.. +``` + +The state list should be empty. Ingress deletion must complete before the ALB +controller is removed. The controller's finalizer deletes the ALB, listeners, +target groups, and its security group. Helm teardown removes Grafana's PVC; +namespace deletion removes Prometheus PVCs retained by the chart. The gp3 +storage class has a Delete reclaim policy, so the EBS driver deletes their disks. +Terraform deletes the ACM certificate, its validation records, and the Grafana +alias record. It does not delete the public hosted zone. + +If destroy hangs on an ingress or PVC, keep the platform running. Check: + +```bash +kubectl -n monitoring get ingress,pvc +kubectl -n kube-system logs deployment/aws-load-balancer-controller --tail=100 +kubectl -n kube-system logs deployment/ebs-csi-controller -c ebs-plugin --tail=100 +kubectl get pv +``` + +Check the controller's IAM role and AWS API errors before retrying. Do not +remove finalizers simply to make Terraform finish: that can orphan billable +resources. Restore the controller if it was removed early, or deliberately +clean up the captured AWS objects and verify each deletion before clearing a +finalizer. Never delete another application's disks or shared security groups. + +## 4. Destroy the platform + +Only proceed when monitoring state is empty and its ALB and volumes are gone. +Use the inventory to inspect their exact IDs in AWS if needed. + +```bash +cd environments +terraform workspace show +terraform plan -destroy -out=cleanup.tfplan +terraform apply cleanup.tfplan +terraform state list +cd .. +``` + +This removes EKS add-ons, the managed node group and its instances, the cluster, +IRSA providers and controller roles, the launch template, log group, NAT +resources, subnets, routing, internet gateway, and VPC. Network interfaces or +load balancers left by a controller can block VPC deletion. If the platform +destroy fails, inspect the blocking ID rather than forcing Terraform to forget it. + +## 5. Audit AWS and DNS + +```bash +python3 scripts/audit.py audit \ + --environment "$AUDIT_ENV" --region "$AUDIT_REGION" \ + --cluster-name "$AUDIT_CLUSTER" --snapshot "audit/$AUDIT_ENV-before.json" \ + --report "audit/$AUDIT_ENV-after.json" +``` + +Exit codes: `0` means no audited environment resources remain, `1` lists +remaining resources, and `2` means the audit was incomplete. An incomplete +audit is not a pass. Some APIs retain deleted NAT gateways and terminated EC2 +instances temporarily; the script treats those terminal states as deleted. +Repeat after AWS finishes asynchronous cleanup if objects are still deleting. + +To inspect individual captured objects directly: + +```bash +aws eks describe-cluster --name "$AUDIT_CLUSTER" --region "$AUDIT_REGION" +# Expected: ResourceNotFoundException. Other errors do not prove deletion. +aws ec2 describe-vpcs --region "$AUDIT_REGION" \ + --filters "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +aws ec2 describe-volumes --region "$AUDIT_REGION" \ + --filters "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +aws ec2 describe-addresses --region "$AUDIT_REGION" \ + --filters "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +aws ec2 describe-nat-gateways --region "$AUDIT_REGION" \ + --filter "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +``` + +Expect empty VPC, volume, and address collections. NAT results may contain only +`deleted` gateways. The script checks load balancers, target groups, IAM roles, +and log groups too; do not use these four commands as the entire audit. + +Check Route 53 separately. The snapshot does not capture DNS records because +they cannot carry ownership tags. The saved monitoring +state JSON contains its ACM validation record names. Keep that file private; +Terraform state can contain secrets. A state-address list alone does not contain +the DNS record values. Use the exact `AUDIT_ZONE` captured before destruction; +selecting the first zone returned by a name lookup can audit the wrong zone. + +```bash +aws route53 list-resource-record-sets --hosted-zone-id "$AUDIT_ZONE" \ + --output json > "audit/$AUDIT_ENV-dns-after.json" +jq --arg host "grafana.$AUDIT_ENV.hudson.tf." \ + '[.ResourceRecordSets[] | select(.Name == $host or (.Name | endswith("." + $host)))]' \ + "audit/$AUDIT_ENV-dns-after.json" +``` + +Expect an empty array for the Grafana alias and its `_token.grafana...` ACM +validation CNAME. Keep unrelated records, including the zone's NS and SOA. +If hostnames were changed in tfvars, use the names from the before inventory +instead of the sample hostname above. Deleting DNS alone never deletes an ALB. + +## 6. Verify state is empty, then remove the unused workspaces + +Both `terraform state list` commands must return no managed resource addresses. +A no-resource state object may remain in S3 until the workspace is removed. +Select another workspace first: + +```bash +cd environments +terraform workspace select default +terraform workspace delete "$AUDIT_ENV" +cd observability +terraform workspace select default +terraform workspace delete "$AUDIT_ENV" +cd ../.. +``` + +Use your bootstrap credentials for workspace deletion: CI roles deliberately +cannot delete state objects. The default-workspace guard blocks plans and +applies, but workspace selection for maintenance is allowed. Never use +`workspace delete -force` to hide a non-empty state. S3 versioning retains old +state versions after deletion; that is intentional recovery data and still +uses storage. Do not use `force-unlock` unless you have verified there is no +Terraform process holding the lock. + +Repeat steps 1–6 for the other environment if removing both. Keep the before +and after reports as cleanup evidence. + +## 7. Retire bootstrap only when both environments are gone + +Normal environment cleanup leaves the state bucket and the four GitHub roles. +This is expected. They are shared infrastructure, and the bucket contains the +bootstrap state. To retire the entire repo's AWS footprint: + +1. Confirm both environment audits passed and neither workspace has resources. + Disable the workflow and remove its `AWS_ROLE_ARN` variables in GitHub. +2. Save bootstrap outputs and inventory its exact bucket and role ARNs. Record + whether the GitHub OIDC provider was created here or supplied as an existing + provider. An existing shared provider must remain. +3. Move bootstrap state back to a secured local backend before deleting the + bucket. Change the local `bootstrap/backend.tf` to `terraform { backend + "local" {} }` on separate lines, then run + `terraform init -migrate-state` from the `bootstrap` folder. Confirm the + prompt and verify `terraform state list` still includes bootstrap + resources. Keep an encrypted backup until the audit passes. +4. Remove the bucket's `prevent_destroy` only for this deliberate retirement. + From that same folder, review `terraform plan -destroy`. Set + `force_destroy = true` on that bucket for this cleanup so Terraform removes **all object versions + and delete markers**, then save and apply the destroy plan. This erases state + history permanently. Keep it off during normal operation. +5. If this repo created the GitHub OIDC provider but other repositories now use + it, transfer its ownership and remove it from this destroy scope before + applying. Check those dependencies explicitly; an account-wide issuer can + be shared. Imported or externally supplied providers are not disposable. + +Verify with read APIs, using the exact IDs saved before retirement: + +```bash +aws s3api head-bucket --bucket "$AUDIT_BUCKET" +# Expected HTTP 404 / NoSuchBucket. A 403 is inconclusive. +aws iam get-role --role-name "$AUDIT_ROLE_NAME" +# Repeat for all four saved roles. Expected NoSuchEntity. +aws iam get-open-id-connect-provider --open-id-connect-provider-arn "$AUDIT_GITHUB_OIDC_ARN" +# Only for a provider owned and retired here. Expected NoSuchEntity. +cd bootstrap +terraform state list +# Expected no managed resources. +cd .. +``` + +Finally confirm the hosted zone and the domain still exist, remove temporary +local plan files and kubeconfig contexts for deleted clusters, and store or +remove the private audit files according to your own retention needs. IAM's +AWS-created service-linked roles may remain and are reused by other clusters; +they are not billable and are not owned by this repo. Do not delete them just +to make an account-wide IAM list empty. + +## 8. Check the bill after AWS has caught up + +Cost Explorer and billing data lag resource deletion. On the following day, +check EKS, EC2 compute, EC2 Other (NAT, EBS and IPv4), ELB, CloudWatch, and S3 +for continued usage. Charges for the deployment day can arrive later even when +cleanup was successful; compare usage dates, not only the posting time. +Route 53's existing hosted zone and domain renewal charges should continue. + +A resource audit proves the checked resources are gone at the audit time. +A billing review catches costs from resources outside the inventory or older +experiments. Neither should be presented as proof that every resource in a +shared AWS account has been deleted. diff --git a/docs/verification.md b/docs/verification.md new file mode 100644 index 0000000..cbb4b92 --- /dev/null +++ b/docs/verification.md @@ -0,0 +1,80 @@ +# Verification + +Verification uses Terraform 1.16.5 with AWS 6.67.0, Helm 3.3.0, and Kubernetes +2.38.0 providers. Root lockfiles include the official checksums. The configured +Kubernetes version is 1.35; monitoring chart 92.7.0 and ALB controller chart +1.14.1 are pinned. + +Run these checks from the repository root: + +```bash +bash scripts/check.sh +bash scripts/test.sh +bash -n scripts/check.sh scripts/test.sh scripts/deploy.sh +``` + +The Terraform tests run plans with synthetic credentials and replace the AWS +identity, partition, cluster and other API lookups. AWS provider schemas and +IAM policy document generation still run. Kubernetes and Helm operations are +mocked. Tests cover both environment inputs, NAT topology, service account role +separation, exact GitHub environment subjects, state-bucket protection, public +Grafana outputs, automatic settings selection by workspace, and refusal of +workspaces without configured settings. Python tests check that +audit denials fail, only recognized not-found responses count as deletion, +terminal compute resources are ignored, captured untagged disks still fail, +and a snapshot from another account cannot pass. + +A live AWS apply has not been run as part of building this repo. No claim is +made that account permissions, regional quotas, DNS delegation, controllers, or +an actual second no-change plan have been verified. For the first deployment: + +1. Apply dev platform, then dev monitoring, using the README commands. +2. Check every node is Ready, both PVCs are Bound, and controller pods are healthy. +3. Check HTTPS Grafana redirects HTTP, requires login, and displays node metrics. +4. Run both plans again in the same workspace; expect no changes. +5. Follow [cleanup](cleanup.md), save the before inventory, and require the AWS + audit and DNS checks to pass after destruction. + +The local check results are: formatting passed, three roots validated, eleven +offline plan runs passed (including missing-workspace-settings and wrong-zone +rejection runs), and seventeen Python audit and deployment-script tests passed. +Both environment roots load their settings from `terraform.tfvars`; the dev +and prod tests pass without a var-file. A separate offline test confirmed that +the default workspace fails variable validation. The script tests cover +read-only PR behavior, deployment order, working directories, workspace +selection, manual operation selection, missing-cluster deferral and errors +that must stop an apply. Cluster deletion and replacement stop before apply; +initial creation and in-place updates are allowed. Failures at each plan/apply +stage stop later deployment stages. +Deployment scripts are tested from a directory outside the repository too. Bash syntax checks passed. Both +pinned Helm charts rendered using the exact Terraform-generated dev values: 104 monitoring +objects, generated Grafana credentials, the expected gp3 PVCs, disabled anonymous +access, bounded Prometheus retention, and the controller's IRSA service account. +These are local checks, not an AWS deployment receipt. + +Chart template rendering is checked separately from mocked Terraform plans. +The exact values from the offline monitoring plan are rendered through the two +published charts; the resulting Kubernetes manifests are inspected for PVC +settings, Grafana authentication, retention and controller identity. Upgrade +checks should repeat that rendering and read upstream chart upgrade notes. + +The staged source also received local correctness, security, and cleanup review. +The security pass corrected the README's claim about bootstrap protection: +privileged role creation and policy attachment do not provide a permissions +boundary. A local synthetic S3 endpoint verified that workspace initialization +and selection use the permitted prefix and environment state key. + +The complete source received correctness, security, test, maintainability, +reliability and adversarial review before the first PR. Its two findings were +fixed: deployment refuses destructive cluster plans until monitoring cleanup, +and offline tests inspect the actual IAM assume-role policies for both plan +and apply roles. A scratch-copy mutation check confirmed that broadening the +OIDC subject to a wildcard fails the policy assertion. Workflow syntax passed +actionlint 1.7.12. These checks do not verify live AWS permissions or controllers. + +Hosted-zone selection was checked against the account through a read-only +Terraform plan. The previous name lookup reproduced the multiple-zone error. +The explicit zone ID lookup succeeded, and its four name servers matched the +domain's public delegation. No DNS records or other AWS resources were changed +by that check. Bootstrap and monitoring use the same selector; the monitoring +test also rejects a selected zone belonging to another domain. diff --git a/environments/.terraform.lock.hcl b/environments/.terraform.lock.hcl new file mode 100644 index 0000000..3622c58 --- /dev/null +++ b/environments/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.67.0" + constraints = "~> 6.67.0" + hashes = [ + "h1:EB9ixYOZrSlYD7wtJxf88qwoyyWrlKDKxhzaCLIb3t4=", + "zh:111d5686a1f4ccbc888bb5e2229308bcdd9149898c6af97969fcdfb0e7bd2aa0", + "zh:21b3b7693bd9754c039fd546f03ed09e7510c6189aa5ee37176f144cf8dd5f95", + "zh:25e03c7f025ff4851889537605aa560d2e39bd738b33a5204b8037eb164b475f", + "zh:2817a046f1060e25bf04b0eb78f4e251130bb92dc82ec1264ce156dc9bf78e67", + "zh:2d318d674c3ec9dbd97ddb10b12ad4827be4f508c43ba2ae1e0523baa9e367e3", + "zh:2f07ab356718267299e10b6072472ded29d82753883027bf43bcd687ac72feb2", + "zh:32307e67f83bc0dc94d38cfcd23d782166a09e0e975e2a5aa7d7a3e7ca5d3da8", + "zh:4ce94853264097dd7f4542b3cfd18d2b98b06d23321db45d5e384ea275a57f63", + "zh:869656c41cc7c7412f213e488f98167cff61deafcb8cf245d25d056e8dfdc263", + "zh:8fd8c814e9d8edf152552047db23bf5b5d63f22e6b0b5d47b2af851376e99349", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9cbc02ceef9bd469da497a1e7065ff984bdacfbe919ac3cce804a86c13b6e2c6", + "zh:9ea55dda2767acfc1f6337fc7d29b1d4d79d6f8f04871f8ad99a6078d2865994", + "zh:d7149df0819fb57a160489db45d33951765b8f0118daa3b4cd549a0135bc97a1", + "zh:e5819937bb7043d08c9829b32d52d9fb55a5b9a4d8d55d550d47a3d7392db546", + "zh:f93bc38dbc0ad53842303f30eec7a22c525c4d56209b54ec33a8d375cfc4e4fd", + ] +} diff --git a/environments/backend.hcl.example b/environments/backend.hcl.example new file mode 100644 index 0000000..74e411b --- /dev/null +++ b/environments/backend.hcl.example @@ -0,0 +1,6 @@ +bucket = "REPLACE-WITH-BOOTSTRAP-BUCKET" +key = "platform.tfstate" +workspace_key_prefix = "environments" +region = "us-east-1" +encrypt = true +use_lockfile = true diff --git a/environments/locals.tf b/environments/locals.tf new file mode 100644 index 0000000..2daebb0 --- /dev/null +++ b/environments/locals.tf @@ -0,0 +1,3 @@ +locals { + settings = var.environments[terraform.workspace] +} diff --git a/environments/main.tf b/environments/main.tf new file mode 100644 index 0000000..7a9a2d2 --- /dev/null +++ b/environments/main.tf @@ -0,0 +1,33 @@ +provider "aws" { + region = local.settings.region + default_tags { tags = { environment = terraform.workspace, "managed-by" = "terraform", project = "eks-platform-reference" } } +} +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} +module "network" { + source = "../modules/network" + environment = terraform.workspace + name = local.settings.cluster_name + vpc_cidr = local.settings.vpc_cidr + subnets = local.settings.subnets + single_nat_gateway = local.settings.single_nat_gateway + internet_cidr = local.settings.internet_cidr +} +module "cluster" { + source = "../modules/cluster" + environment = terraform.workspace + cluster_name = local.settings.cluster_name + kubernetes_version = local.settings.kubernetes_version + subnet_ids = module.network.private_subnet_ids + public_access_cidrs = local.settings.public_access_cidrs + instance_types = local.settings.instance_types + node_sizes = local.settings.node_sizes + node_disk_size = local.settings.node_disk_size + log_retention_days = local.settings.log_retention_days + access_entries = merge({ + deploy = { principal_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${local.settings.deploy_role_name}", policy_name = "AmazonEKSClusterAdminPolicy" } + plan = { principal_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${local.settings.plan_role_name}", policy_name = "AmazonEKSAdminViewPolicy" } + }, { for name, arn in local.settings.administrator_arns : "admin-${name}" => { principal_arn = arn, policy_name = "AmazonEKSClusterAdminPolicy" } }) + # Route tables must be ready before nodes need outbound access. + depends_on = [module.network] +} diff --git a/environments/observability/.terraform.lock.hcl b/environments/observability/.terraform.lock.hcl new file mode 100644 index 0000000..aa033f8 --- /dev/null +++ b/environments/observability/.terraform.lock.hcl @@ -0,0 +1,67 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.67.0" + constraints = "~> 6.67.0" + hashes = [ + "h1:EB9ixYOZrSlYD7wtJxf88qwoyyWrlKDKxhzaCLIb3t4=", + "zh:111d5686a1f4ccbc888bb5e2229308bcdd9149898c6af97969fcdfb0e7bd2aa0", + "zh:21b3b7693bd9754c039fd546f03ed09e7510c6189aa5ee37176f144cf8dd5f95", + "zh:25e03c7f025ff4851889537605aa560d2e39bd738b33a5204b8037eb164b475f", + "zh:2817a046f1060e25bf04b0eb78f4e251130bb92dc82ec1264ce156dc9bf78e67", + "zh:2d318d674c3ec9dbd97ddb10b12ad4827be4f508c43ba2ae1e0523baa9e367e3", + "zh:2f07ab356718267299e10b6072472ded29d82753883027bf43bcd687ac72feb2", + "zh:32307e67f83bc0dc94d38cfcd23d782166a09e0e975e2a5aa7d7a3e7ca5d3da8", + "zh:4ce94853264097dd7f4542b3cfd18d2b98b06d23321db45d5e384ea275a57f63", + "zh:869656c41cc7c7412f213e488f98167cff61deafcb8cf245d25d056e8dfdc263", + "zh:8fd8c814e9d8edf152552047db23bf5b5d63f22e6b0b5d47b2af851376e99349", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9cbc02ceef9bd469da497a1e7065ff984bdacfbe919ac3cce804a86c13b6e2c6", + "zh:9ea55dda2767acfc1f6337fc7d29b1d4d79d6f8f04871f8ad99a6078d2865994", + "zh:d7149df0819fb57a160489db45d33951765b8f0118daa3b4cd549a0135bc97a1", + "zh:e5819937bb7043d08c9829b32d52d9fb55a5b9a4d8d55d550d47a3d7392db546", + "zh:f93bc38dbc0ad53842303f30eec7a22c525c4d56209b54ec33a8d375cfc4e4fd", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "3.3.0" + constraints = "~> 3.3.0" + hashes = [ + "h1:Qbt2+dtOJty3glPZI9jMCXq82GSAKBk/T2+wBMIdu0c=", + "zh:24a942352667cf49bed467c51a951f17a6c13e0cdf1c505b01b41f75a1a60d0f", + "zh:2c057eafd1e4d8b50acdc676f9a3096e38fb7d30c3fab3d64af82fa04a2f767e", + "zh:44252c60034ca163f2f68eaeece2583411bca25ac83bc58350e24d4f1c2d60a7", + "zh:4628da7c781eebbce1b13d12b498856815bde0e174d7a0d0b85a5d89a1d36d17", + "zh:59b3f4fd8e236c5733032c289a9636ac8bb6c92bc11b540d979b92d005f6cf98", + "zh:5d5f11201b986485ca1d33096f3545c6074b3b4a592864620a8df7f7c399b59f", + "zh:5e656afadb2db3f418eb254b2da02b00851eba8ec63196d1f22b2a2854fe9fb9", + "zh:77515e5576f4ea3d8d9e404237882de62518950ccd7b312371f5c03580f22fb0", + "zh:a4e601eddf9781b5f8678ee9c2b6fb0791fbe6ab85a53707efdf80f714a3011c", + "zh:eaa58edc8bb43aeb0fe6c0f99975cb24e2e9f5441e938a389fe848795a30b825", + "zh:ec97b17b82222d2008256f9b10b88f564d30f717a43438fe425a802ac0ccf2f3", + "zh:f18613d2aa3fd3945d394911e210af7bf58968f64631b6b2e229852b60a2803e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.38.0" + hashes = [ + "h1:5CkveFo5ynsLdzKk+Kv+r7+U9rMrNjfZPT3a0N/fhgE=", + "zh:0af928d776eb269b192dc0ea0f8a3f0f5ec117224cd644bdacdc682300f84ba0", + "zh:1be998e67206f7cfc4ffe77c01a09ac91ce725de0abaec9030b22c0a832af44f", + "zh:326803fe5946023687d603f6f1bab24de7af3d426b01d20e51d4e6fbe4e7ec1b", + "zh:4a99ec8d91193af961de1abb1f824be73df07489301d62e6141a656b3ebfff12", + "zh:5136e51765d6a0b9e4dbcc3b38821e9736bd2136cf15e9aac11668f22db117d2", + "zh:63fab47349852d7802fb032e4f2b6a101ee1ce34b62557a9ad0f0f0f5b6ecfdc", + "zh:924fb0257e2d03e03e2bfe9c7b99aa73c195b1f19412ca09960001bee3c50d15", + "zh:b63a0be5e233f8f6727c56bed3b61eb9456ca7a8bb29539fba0837f1badf1396", + "zh:d39861aa21077f1bc899bc53e7233262e530ba8a3a2d737449b100daeb303e4d", + "zh:de0805e10ebe4c83ce3b728a67f6b0f9d18be32b25146aa89116634df5145ad4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:faf23e45f0090eef8ba28a8aac7ec5d4fdf11a36c40a8d286304567d71c1e7db", + ] +} diff --git a/environments/observability/backend.hcl.example b/environments/observability/backend.hcl.example new file mode 100644 index 0000000..286556a --- /dev/null +++ b/environments/observability/backend.hcl.example @@ -0,0 +1,6 @@ +bucket = "REPLACE-WITH-BOOTSTRAP-BUCKET" +key = "observability.tfstate" +workspace_key_prefix = "environments" +region = "us-east-1" +encrypt = true +use_lockfile = true diff --git a/environments/observability/locals.tf b/environments/observability/locals.tf new file mode 100644 index 0000000..2daebb0 --- /dev/null +++ b/environments/observability/locals.tf @@ -0,0 +1,3 @@ +locals { + settings = var.environments[terraform.workspace] +} diff --git a/environments/observability/main.tf b/environments/observability/main.tf new file mode 100644 index 0000000..09fd499 --- /dev/null +++ b/environments/observability/main.tf @@ -0,0 +1,67 @@ +provider "aws" { + region = local.settings.region + default_tags { tags = { environment = terraform.workspace, "managed-by" = "terraform", project = "eks-platform-reference" } } +} +data "aws_eks_cluster" "this" { name = local.settings.cluster_name } +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", local.settings.cluster_name, "--region", local.settings.region] + } + } +} +provider "kubernetes" { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", local.settings.cluster_name, "--region", local.settings.region] + } +} +data "aws_route53_zone" "public" { + zone_id = local.settings.hosted_zone_id + private_zone = false + lifecycle { + postcondition { + condition = !self.private_zone && lower(trimsuffix(self.name, ".")) == lower(trimsuffix(local.settings.domain_name, ".")) + error_message = "The selected hosted zone must be public and match domain_name." + } + } +} +data "aws_iam_role" "controller" { name = "${local.settings.cluster_name}-alb" } +data "aws_subnets" "public" { + filter { + name = "vpc-id" + values = [data.aws_eks_cluster.this.vpc_config[0].vpc_id] + } + filter { + name = "tag:kubernetes.io/role/elb" + values = ["1"] + } +} +module "observability" { + region = local.settings.region + cluster_name = local.settings.cluster_name + vpc_id = data.aws_eks_cluster.this.vpc_config[0].vpc_id + public_subnet_ids = data.aws_subnets.public.ids + controller_role_arn = data.aws_iam_role.controller.arn + controller_chart_version = local.settings.controller_chart_version + hosted_zone_id = data.aws_route53_zone.public.zone_id + grafana_hostname = local.settings.grafana_hostname + load_balancer_name = local.settings.load_balancer_name + ingress_cidrs = local.settings.ingress_cidrs + source = "../../modules/observability" + environment = terraform.workspace + chart_version = local.settings.chart_version + namespace = local.settings.namespace + prometheus_retention = local.settings.prometheus_retention + prometheus_storage_size = local.settings.prometheus_storage_size + grafana_storage_size = local.settings.grafana_storage_size + prometheus_memory_request = local.settings.prometheus_memory_request + prometheus_memory_limit = local.settings.prometheus_memory_limit +} diff --git a/environments/observability/outputs.tf b/environments/observability/outputs.tf new file mode 100644 index 0000000..b9253b5 --- /dev/null +++ b/environments/observability/outputs.tf @@ -0,0 +1,29 @@ +output "namespace" { + description = "Monitoring namespace." + value = module.observability.namespace +} + +output "grafana_service" { + description = "Grafana service for port-forward." + value = module.observability.grafana_service +} + +output "grafana_secret" { + description = "Grafana admin credential secret." + value = module.observability.grafana_secret +} + +output "grafana_url" { + description = "Public Grafana HTTPS endpoint." + value = module.observability.grafana_url +} + +output "environment" { + description = "Selected Terraform workspace." + value = terraform.workspace +} + +output "hosted_zone_id" { + description = "Public hosted zone selected for ingress and certificate validation." + value = data.aws_route53_zone.public.zone_id +} diff --git a/environments/observability/terraform.tfvars b/environments/observability/terraform.tfvars new file mode 100644 index 0000000..1b3edcb --- /dev/null +++ b/environments/observability/terraform.tfvars @@ -0,0 +1,39 @@ +environments = { + dev = { + region = "us-east-1" + cluster_name = "eks-platform-reference-dev" + chart_version = "92.7.0" + namespace = "monitoring" + prometheus_retention = "2d" + prometheus_storage_size = "10Gi" + grafana_storage_size = "5Gi" + prometheus_memory_request = "512Mi" + prometheus_memory_limit = "1Gi" + + domain_name = "hudson.tf" + grafana_hostname = "grafana.dev.hudson.tf" + load_balancer_name = "eks-reference-dev-grafana" + controller_chart_version = "1.14.1" + ingress_cidrs = ["0.0.0.0/0"] + hosted_zone_id = "Z0992412PWY8TUM0P6ZU" + } + + prod = { + region = "us-east-1" + cluster_name = "eks-platform-reference-prod" + chart_version = "92.7.0" + namespace = "monitoring" + prometheus_retention = "7d" + prometheus_storage_size = "30Gi" + grafana_storage_size = "5Gi" + prometheus_memory_request = "1Gi" + prometheus_memory_limit = "2Gi" + + domain_name = "hudson.tf" + grafana_hostname = "grafana.prod.hudson.tf" + load_balancer_name = "eks-reference-prod-grafana" + controller_chart_version = "1.14.1" + ingress_cidrs = ["0.0.0.0/0"] + hosted_zone_id = "Z0992412PWY8TUM0P6ZU" + } +} diff --git a/environments/observability/tests/monitoring.tftest.hcl b/environments/observability/tests/monitoring.tftest.hcl new file mode 100644 index 0000000..fa0f806 --- /dev/null +++ b/environments/observability/tests/monitoring.tftest.hcl @@ -0,0 +1,56 @@ +variable "test_workspace" { type = string } +variable "environments" { type = any } +provider "aws" { + region = var.environments[var.test_workspace].region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +mock_provider "helm" {} +mock_provider "kubernetes" {} +override_data { + target = data.aws_eks_cluster.this + values = { + endpoint = "https://offline.invalid" + certificate_authority = [{ data = "dGVzdA==" }] + vpc_config = [{ vpc_id = "vpc-offline", subnet_ids = ["subnet-offline-a", "subnet-offline-b"], cluster_security_group_id = "sg-offline", security_group_ids = [], endpoint_private_access = true, endpoint_public_access = true, public_access_cidrs = [] }] + } +} +override_data { + target = data.aws_route53_zone.public + values = { zone_id = "ZOFFLINETEST", name = "hudson.tf", private_zone = false } +} +override_data { + target = data.aws_iam_role.controller + values = { arn = "arn:aws:iam::111122223333:role/offline-alb" } +} +override_data { + target = data.aws_subnets.public + values = { ids = ["subnet-offline-a", "subnet-offline-b"] } +} +override_data { + target = module.observability.data.aws_lb.grafana + values = { dns_name = "offline.elb.amazonaws.com", zone_id = "ZOFFLINEALB" } +} +run "monitoring_plan" { + command = plan + assert { + condition = output.grafana_service == "monitoring-grafana" + error_message = "Ingress and the documented password lookup must address the chart's Grafana service." + } + assert { + condition = output.grafana_url == "https://${var.environments[var.test_workspace].grafana_hostname}" + error_message = "Grafana output must use the configured HTTPS hostname." + } +} + +run "reject_incorrect_hosted_zone" { + command = plan + override_data { + target = data.aws_route53_zone.public + values = { zone_id = "ZOFFLINETEST", name = "other.invalid", private_zone = false } + } + expect_failures = [data.aws_route53_zone.public] +} diff --git a/environments/observability/variables.tf b/environments/observability/variables.tf new file mode 100644 index 0000000..c3e31f3 --- /dev/null +++ b/environments/observability/variables.tf @@ -0,0 +1,40 @@ +variable "environments" { + description = "Environment settings keyed by Terraform workspace; loaded automatically from terraform.tfvars." + type = map(object({ + # Cluster AWS region. + region = string + # Existing EKS cluster name. + cluster_name = string + # Pinned kube-prometheus-stack chart. + chart_version = string + # Monitoring namespace. + namespace = string + # Prometheus retention duration. + prometheus_retention = string + # Prometheus storage in integer Gi units. + prometheus_storage_size = string + # Grafana storage capacity. + grafana_storage_size = string + # Prometheus memory request. + prometheus_memory_request = string + # Prometheus memory limit. + prometheus_memory_limit = string + # Existing public Route 53 hosted zone name. + domain_name = string + # Public HTTPS Grafana hostname. + grafana_hostname = string + # Unique ALB name, maximum 32 characters. + load_balancer_name = string + # AWS Load Balancer Controller chart pin. + controller_chart_version = string + # Client CIDRs permitted to reach Grafana. + ingress_cidrs = list(string) + # Exact existing public Route 53 zone ID; zone names are not unique. + hosted_zone_id = string + })) + + validation { + condition = terraform.workspace != "default" && contains(keys(var.environments), terraform.workspace) + error_message = "Select a named workspace with settings in terraform.tfvars, such as dev or prod." + } +} diff --git a/environments/observability/versions.tf b/environments/observability/versions.tf new file mode 100644 index 0000000..bbbc461 --- /dev/null +++ b/environments/observability/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + kubernetes = { source = "hashicorp/kubernetes", version = "~> 2.38.0" } + helm = { + source = "hashicorp/helm" + version = "~> 3.3.0" + } + } +} diff --git a/environments/outputs.tf b/environments/outputs.tf new file mode 100644 index 0000000..10440b7 --- /dev/null +++ b/environments/outputs.tf @@ -0,0 +1,49 @@ +output "region" { + description = "AWS region." + value = local.settings.region +} + +output "cluster_name" { + description = "EKS cluster name." + value = module.cluster.cluster_name +} + +output "cluster_endpoint" { + description = "Kubernetes API endpoint." + value = module.cluster.cluster_endpoint +} + +output "vpc_id" { + description = "Environment VPC." + value = module.network.vpc_id +} + +output "private_subnet_ids" { + description = "Private subnets." + value = module.network.private_subnet_ids +} + +output "nat_public_ips" { + description = "NAT egress addresses." + value = module.network.nat_public_ips +} + +output "oidc_provider_arn" { + description = "IRSA provider ARN." + value = module.cluster.oidc_provider_arn +} + +output "oidc_issuer_url" { + description = "IRSA issuer URL." + value = module.cluster.oidc_issuer_url +} + +output "irsa_role_arns" { + description = "Add-on IAM roles." + value = module.cluster.irsa_role_arns +} + +output "environment" { + description = "Selected Terraform workspace." + value = terraform.workspace +} diff --git a/environments/terraform.tfvars b/environments/terraform.tfvars new file mode 100644 index 0000000..83c0988 --- /dev/null +++ b/environments/terraform.tfvars @@ -0,0 +1,47 @@ +environments = { + dev = { + region = "us-east-1" + cluster_name = "eks-platform-reference-dev" + vpc_cidr = "10.20.0.0/16" + subnets = { + a = { availability_zone = "us-east-1a", public_cidr = "10.20.0.0/24", private_cidr = "10.20.16.0/20" } + b = { availability_zone = "us-east-1b", public_cidr = "10.20.1.0/24", private_cidr = "10.20.32.0/20" } + } + single_nat_gateway = true + internet_cidr = "0.0.0.0/0" + kubernetes_version = "1.35" + # GitHub-hosted runners have changing IPs. Use restricted CIDRs with a fixed-egress runner. + public_access_cidrs = ["0.0.0.0/0"] + instance_types = ["t3.medium"] + node_sizes = { min = 2, desired = 2, max = 3 } + node_disk_size = 30 + log_retention_days = 7 + deploy_role_name = "github-eks-platform-dev-apply" + plan_role_name = "github-eks-platform-dev-plan" + administrator_arns = { + patrick = "arn:aws:iam::626193791566:user/patrick" + } + } + + prod = { + region = "us-east-1" + cluster_name = "eks-platform-reference-prod" + vpc_cidr = "10.30.0.0/16" + subnets = { + a = { availability_zone = "us-east-1a", public_cidr = "10.30.0.0/24", private_cidr = "10.30.16.0/20" } + b = { availability_zone = "us-east-1b", public_cidr = "10.30.1.0/24", private_cidr = "10.30.32.0/20" } + c = { availability_zone = "us-east-1c", public_cidr = "10.30.2.0/24", private_cidr = "10.30.48.0/20" } + } + single_nat_gateway = false + internet_cidr = "0.0.0.0/0" + kubernetes_version = "1.35" + # GitHub-hosted runners have changing IPs. Use restricted CIDRs with a fixed-egress runner. + public_access_cidrs = ["0.0.0.0/0"] + instance_types = ["m6i.large"] + node_sizes = { min = 3, desired = 3, max = 6 } + node_disk_size = 30 + log_retention_days = 30 + deploy_role_name = "github-eks-platform-prod-apply" + plan_role_name = "github-eks-platform-prod-plan" + } +} diff --git a/environments/tests/platform.tftest.hcl b/environments/tests/platform.tftest.hcl new file mode 100644 index 0000000..74ebaf9 --- /dev/null +++ b/environments/tests/platform.tftest.hcl @@ -0,0 +1,52 @@ +variable "test_workspace" { type = string } +variable "environments" { type = any } +# Offline provider plans still run real AWS resource schemas and IAM JSON rendering. +provider "aws" { + region = var.environments[var.test_workspace].region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +override_data { + target = data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.cluster.data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.cluster.data.aws_eks_addon_version.this + values = { version = "v1.0.0-eksbuild.1" } +} +run "platform_plan" { + command = plan + assert { + condition = output.environment == var.test_workspace && output.cluster_name == var.environments[var.test_workspace].cluster_name + error_message = "Workspace selection must choose the matching environment settings." + } + assert { + condition = length(output.nat_public_ips) == (var.environments[var.test_workspace].single_nat_gateway ? 1 : length(var.environments[var.test_workspace].subnets)) + error_message = "NAT topology must match the environment's cost and availability choice." + } + assert { + condition = length(output.private_subnet_ids) == length(var.environments[var.test_workspace].subnets) + error_message = "Every configured zone needs a private node subnet." + } + assert { + condition = toset(keys(output.irsa_role_arns)) == toset(["cni", "ebs", "alb"]) + error_message = "AWS controllers need distinct service account roles." + } +} + +run "reject_missing_workspace_settings" { + command = plan + variables { environments = {} } + expect_failures = [var.environments] +} diff --git a/environments/variables.tf b/environments/variables.tf new file mode 100644 index 0000000..ad34694 --- /dev/null +++ b/environments/variables.tf @@ -0,0 +1,40 @@ +variable "environments" { + description = "Environment settings keyed by Terraform workspace; loaded automatically from terraform.tfvars." + type = map(object({ + # AWS region for the environment. + region = string + # EKS name, shared with bootstrap configuration. + cluster_name = string + # Non-overlapping VPC CIDR. + vpc_cidr = string + # Explicit AZ and subnet CIDRs. + subnets = map(object({ availability_zone = string, public_cidr = string, private_cidr = string })) + # Share one NAT gateway across zones. + single_nat_gateway = bool + # Internet route destination CIDR. + internet_cidr = string + # EKS Kubernetes minor version. + kubernetes_version = string + # Kubernetes API public ingress CIDRs. + public_access_cidrs = list(string) + # Managed node EC2 instance types. + instance_types = list(string) + # Managed node scaling settings. + node_sizes = object({ min = number, desired = number, max = number }) + # Node root volume GiB. + node_disk_size = number + # Control plane log retention. + log_retention_days = number + # Bootstrap-created deploy role name. + deploy_role_name = string + # Bootstrap-created plan role name. + plan_role_name = string + # Stable names of human administrator IAM roles, never STS session ARNs. + administrator_arns = optional(map(string), {}) + })) + + validation { + condition = terraform.workspace != "default" && contains(keys(var.environments), terraform.workspace) + error_message = "Select a named workspace with settings in terraform.tfvars, such as dev or prod." + } +} diff --git a/environments/versions.tf b/environments/versions.tf new file mode 100644 index 0000000..5775bc5 --- /dev/null +++ b/environments/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/modules/cluster/load-balancer-controller-policy.json b/modules/cluster/load-balancer-controller-policy.json new file mode 100644 index 0000000..761d0e7 --- /dev/null +++ b/modules/cluster/load-balancer-controller-policy.json @@ -0,0 +1,251 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateServiceLinkedRole" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "ec2:GetSecurityGroupsForVpc", + "ec2:DescribeIpamPools", + "ec2:DescribeRouteTables", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "StringEquals": { + "ec2:CreateAction": "CreateSecurityGroup" + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation", + "elasticloadbalancing:ModifyIpPools" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "StringEquals": { + "elasticloadbalancing:CreateAction": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule", + "elasticloadbalancing:SetRulePriorities" + ], + "Resource": "*" + } + ] +} diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf new file mode 100644 index 0000000..46e144b --- /dev/null +++ b/modules/cluster/main.tf @@ -0,0 +1,219 @@ +data "aws_partition" "current" {} +locals { + tags = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) + oidc_issuer = replace(aws_iam_openid_connect_provider.this.url, "https://", "") + partition = data.aws_partition.current.partition + irsa = { + alb = { subject = "system:serviceaccount:kube-system:aws-load-balancer-controller", policy = null } + cni = { subject = "system:serviceaccount:kube-system:aws-node", policy = "AmazonEKS_CNI_Policy" } + ebs = { subject = "system:serviceaccount:kube-system:ebs-csi-controller-sa", policy = "service-role/AmazonEBSCSIDriverPolicy" } + } +} +data "aws_iam_policy_document" "cluster_trust" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["eks.amazonaws.com"] + } + } +} +resource "aws_iam_role" "cluster" { + name = "${var.cluster_name}-cluster" + assume_role_policy = data.aws_iam_policy_document.cluster_trust.json + tags = local.tags +} +resource "aws_iam_role_policy_attachment" "cluster" { + role = aws_iam_role.cluster.name + policy_arn = "arn:${local.partition}:iam::aws:policy/AmazonEKSClusterPolicy" +} +resource "aws_cloudwatch_log_group" "cluster" { + name = "/aws/eks/${var.cluster_name}/cluster" + retention_in_days = var.log_retention_days + tags = local.tags +} +resource "aws_eks_cluster" "this" { + name = var.cluster_name + role_arn = aws_iam_role.cluster.arn + version = var.kubernetes_version + enabled_cluster_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"] + access_config { + authentication_mode = "API" + bootstrap_cluster_creator_admin_permissions = false + } + upgrade_policy { support_type = "STANDARD" } + vpc_config { + subnet_ids = var.subnet_ids + endpoint_private_access = true + endpoint_public_access = true + public_access_cidrs = var.public_access_cidrs + } + tags = local.tags + depends_on = [aws_iam_role_policy_attachment.cluster, aws_cloudwatch_log_group.cluster] +} +resource "aws_iam_openid_connect_provider" "this" { + url = aws_eks_cluster.this.identity[0].oidc[0].issuer + client_id_list = ["sts.amazonaws.com"] + tags = local.tags +} +data "aws_iam_policy_document" "irsa" { + for_each = local.irsa + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.this.arn] + } + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:sub" + values = [each.value.subject] + } + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:aud" + values = ["sts.amazonaws.com"] + } + } +} +resource "aws_iam_role" "irsa" { + for_each = local.irsa + name = "${var.cluster_name}-${each.key}" + assume_role_policy = data.aws_iam_policy_document.irsa[each.key].json + tags = local.tags +} +resource "aws_iam_role_policy_attachment" "irsa" { + for_each = { for k, v in local.irsa : k => v if v.policy != null } + role = aws_iam_role.irsa[each.key].name + policy_arn = "arn:${local.partition}:iam::aws:policy/${each.value.policy}" +} +data "aws_iam_policy_document" "node_trust" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} +resource "aws_iam_role" "node" { + name = "${var.cluster_name}-node" + assume_role_policy = data.aws_iam_policy_document.node_trust.json + tags = local.tags +} +resource "aws_iam_role_policy_attachment" "node" { + for_each = toset(["AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryPullOnly"]) + role = aws_iam_role.node.name + policy_arn = "arn:${local.partition}:iam::aws:policy/${each.key}" +} +resource "aws_launch_template" "node" { + name_prefix = "${var.cluster_name}-" + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } + block_device_mappings { + device_name = "/dev/xvda" + ebs { + volume_size = var.node_disk_size + volume_type = "gp3" + encrypted = true + delete_on_termination = true + } + } + dynamic "tag_specifications" { + for_each = toset(["instance", "volume", "network-interface"]) + content { + resource_type = tag_specifications.value + tags = local.tags + } + } + tags = local.tags +} +data "aws_eks_addon_version" "this" { + for_each = toset(["vpc-cni", "coredns", "kube-proxy", "aws-ebs-csi-driver"]) + addon_name = each.key + kubernetes_version = var.kubernetes_version + most_recent = false +} +# CNI must have its IAM role before any node tries to bring up pod networking. +resource "aws_eks_addon" "cni" { + cluster_name = aws_eks_cluster.this.name + addon_name = "vpc-cni" + addon_version = data.aws_eks_addon_version.this["vpc-cni"].version + service_account_role_arn = aws_iam_role.irsa["cni"].arn + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "PRESERVE" + tags = local.tags + depends_on = [aws_iam_role_policy_attachment.irsa] +} +resource "aws_eks_node_group" "this" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "${var.cluster_name}-workers" + node_role_arn = aws_iam_role.node.arn + subnet_ids = var.subnet_ids + instance_types = var.instance_types + ami_type = "AL2023_x86_64_STANDARD" + capacity_type = "ON_DEMAND" + launch_template { + id = aws_launch_template.node.id + version = aws_launch_template.node.latest_version + } + scaling_config { + min_size = var.node_sizes.min + desired_size = var.node_sizes.desired + max_size = var.node_sizes.max + } + update_config { max_unavailable = 1 } + labels = { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" } + tags = local.tags + depends_on = [aws_iam_role_policy_attachment.node, aws_eks_addon.cni] +} +resource "aws_eks_addon" "this" { + for_each = toset(["coredns", "kube-proxy", "aws-ebs-csi-driver"]) + cluster_name = aws_eks_cluster.this.name + addon_name = each.key + addon_version = data.aws_eks_addon_version.this[each.key].version + service_account_role_arn = each.key == "aws-ebs-csi-driver" ? aws_iam_role.irsa["ebs"].arn : null + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "PRESERVE" + tags = local.tags + depends_on = [aws_eks_node_group.this, aws_iam_role_policy_attachment.irsa] +} +resource "aws_eks_access_entry" "this" { + for_each = var.access_entries + cluster_name = aws_eks_cluster.this.name + principal_arn = each.value.principal_arn + tags = local.tags +} +resource "aws_eks_access_policy_association" "this" { + for_each = var.access_entries + cluster_name = aws_eks_cluster.this.name + principal_arn = aws_eks_access_entry.this[each.key].principal_arn + policy_arn = "arn:${local.partition}:eks::aws:cluster-access-policy/${each.value.policy_name}" + access_scope { type = "cluster" } +} + +# Upstream v2.14.1 policy, kept in sync with the pinned Helm controller chart. +resource "aws_iam_role_policy" "load_balancer_controller" { + name = "load-balancer-controller" + role = aws_iam_role.irsa["alb"].id + policy = replace(file("${path.module}/load-balancer-controller-policy.json"), "arn:aws:", "arn:${local.partition}:") +} +# EKS owns these resources, but does not propagate cluster/node-group tags to them. +resource "aws_ec2_tag" "cluster_security_group" { + for_each = local.tags + resource_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id + key = each.key + value = each.value +} +resource "aws_autoscaling_group_tag" "node" { + for_each = local.tags + autoscaling_group_name = aws_eks_node_group.this.resources[0].autoscaling_groups[0].name + tag { + key = each.key + value = each.value + propagate_at_launch = true + } +} diff --git a/modules/cluster/outputs.tf b/modules/cluster/outputs.tf new file mode 100644 index 0000000..132ea34 --- /dev/null +++ b/modules/cluster/outputs.tf @@ -0,0 +1,39 @@ +output "cluster_name" { + description = "Cluster name for CLI and Helm authentication." + value = aws_eks_cluster.this.name +} + +output "cluster_endpoint" { + description = "Kubernetes API endpoint." + value = aws_eks_cluster.this.endpoint +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded Kubernetes CA." + value = aws_eks_cluster.this.certificate_authority[0].data +} + +output "cluster_security_group_id" { + description = "EKS managed security group." + value = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id +} + +output "oidc_provider_arn" { + description = "IRSA OIDC provider ARN." + value = aws_iam_openid_connect_provider.this.arn +} + +output "oidc_issuer_url" { + description = "IRSA issuer for future service account trust policies." + value = aws_iam_openid_connect_provider.this.url +} + +output "irsa_role_arns" { + description = "Service account roles for CNI and EBS CSI." + value = { for k, role in aws_iam_role.irsa : k => role.arn } +} + +output "node_group_name" { + description = "Managed node group name." + value = aws_eks_node_group.this.node_group_name +} diff --git a/modules/cluster/variables.tf b/modules/cluster/variables.tf new file mode 100644 index 0000000..9d932f9 --- /dev/null +++ b/modules/cluster/variables.tf @@ -0,0 +1,55 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "cluster_name" { + description = "EKS cluster name and prefix for its IAM roles." + type = string +} + +variable "kubernetes_version" { + description = "EKS Kubernetes minor version in standard support." + type = string +} + +variable "subnet_ids" { + description = "Private subnet IDs for the cluster and nodes." + type = list(string) +} + +variable "public_access_cidrs" { + description = "CIDRs permitted to reach the public Kubernetes API." + type = list(string) +} + +variable "instance_types" { + description = "EC2 instance types for the managed node group." + type = list(string) +} + +variable "node_sizes" { + description = "Minimum, desired and maximum node counts." + type = object({ min = number, desired = number, max = number }) +} + +variable "node_disk_size" { + description = "Encrypted root volume size in GiB." + type = number +} + +variable "log_retention_days" { + description = "CloudWatch control plane log retention." + type = number +} + +variable "access_entries" { + description = "Stable entry names mapped to IAM principals and EKS access policy names." + type = map(object({ principal_arn = string, policy_name = string })) +} diff --git a/modules/cluster/versions.tf b/modules/cluster/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/modules/cluster/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/modules/network/main.tf b/modules/network/main.tf new file mode 100644 index 0000000..1b2da9e --- /dev/null +++ b/modules/network/main.tf @@ -0,0 +1,72 @@ +locals { + tags = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) + primary_nat_key = sort(keys(var.subnets))[0] + nat_subnets = var.single_nat_gateway ? { for k, v in var.subnets : k => v if k == local.primary_nat_key } : var.subnets +} +resource "aws_vpc" "this" { + cidr_block = var.vpc_cidr + enable_dns_support = true + enable_dns_hostnames = true + tags = merge(local.tags, { Name = var.name }) +} +resource "aws_internet_gateway" "this" { + vpc_id = aws_vpc.this.id + tags = merge(local.tags, { Name = var.name }) +} +resource "aws_subnet" "public" { + for_each = var.subnets + vpc_id = aws_vpc.this.id + availability_zone = each.value.availability_zone + cidr_block = each.value.public_cidr + map_public_ip_on_launch = false + tags = merge(local.tags, { Name = "${var.name}-public-${each.key}", "kubernetes.io/role/elb" = "1" }) +} +resource "aws_subnet" "private" { + for_each = var.subnets + vpc_id = aws_vpc.this.id + availability_zone = each.value.availability_zone + cidr_block = each.value.private_cidr + tags = merge(local.tags, { Name = "${var.name}-private-${each.key}", "kubernetes.io/role/internal-elb" = "1" }) +} +resource "aws_eip" "nat" { + for_each = local.nat_subnets + domain = "vpc" + tags = merge(local.tags, { Name = "${var.name}-nat-${each.key}" }) +} +resource "aws_nat_gateway" "this" { + for_each = local.nat_subnets + allocation_id = aws_eip.nat[each.key].id + subnet_id = aws_subnet.public[each.key].id + tags = merge(local.tags, { Name = "${var.name}-${each.key}" }) + depends_on = [aws_internet_gateway.this] +} +resource "aws_route_table" "public" { + vpc_id = aws_vpc.this.id + tags = merge(local.tags, { Name = "${var.name}-public" }) +} +resource "aws_route" "public" { + route_table_id = aws_route_table.public.id + destination_cidr_block = var.internet_cidr + gateway_id = aws_internet_gateway.this.id +} +resource "aws_route_table_association" "public" { + for_each = var.subnets + subnet_id = aws_subnet.public[each.key].id + route_table_id = aws_route_table.public.id +} +resource "aws_route_table" "private" { + for_each = var.subnets + vpc_id = aws_vpc.this.id + tags = merge(local.tags, { Name = "${var.name}-private-${each.key}" }) +} +resource "aws_route" "private" { + for_each = var.subnets + route_table_id = aws_route_table.private[each.key].id + destination_cidr_block = var.internet_cidr + nat_gateway_id = aws_nat_gateway.this[var.single_nat_gateway ? local.primary_nat_key : each.key].id +} +resource "aws_route_table_association" "private" { + for_each = var.subnets + subnet_id = aws_subnet.private[each.key].id + route_table_id = aws_route_table.private[each.key].id +} diff --git a/modules/network/outputs.tf b/modules/network/outputs.tf new file mode 100644 index 0000000..f59891a --- /dev/null +++ b/modules/network/outputs.tf @@ -0,0 +1,19 @@ +output "vpc_id" { + description = "VPC identifier." + value = aws_vpc.this.id +} + +output "public_subnet_ids" { + description = "Public subnets in stable key order." + value = [for s in aws_subnet.public : s.id] +} + +output "private_subnet_ids" { + description = "Private subnets in stable key order." + value = [for s in aws_subnet.private : s.id] +} + +output "nat_public_ips" { + description = "Outbound IP addresses by availability zone." + value = { for k, e in aws_eip.nat : k => e.public_ip } +} diff --git a/modules/network/variables.tf b/modules/network/variables.tf new file mode 100644 index 0000000..5eec17e --- /dev/null +++ b/modules/network/variables.tf @@ -0,0 +1,35 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "name" { + description = "Prefix for network resource names." + type = string +} + +variable "vpc_cidr" { + description = "IPv4 CIDR for the VPC." + type = string +} + +variable "subnets" { + description = "One public and one private subnet per named availability zone." + type = map(object({ availability_zone = string, public_cidr = string, private_cidr = string })) +} + +variable "single_nat_gateway" { + description = "Use one shared NAT gateway instead of one in each availability zone." + type = bool +} + +variable "internet_cidr" { + description = "Destination CIDR for internet routes." + type = string +} diff --git a/modules/network/versions.tf b/modules/network/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/modules/network/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/modules/observability/ingress.tf b/modules/observability/ingress.tf new file mode 100644 index 0000000..b3a9cd1 --- /dev/null +++ b/modules/observability/ingress.tf @@ -0,0 +1,102 @@ + +resource "helm_release" "load_balancer_controller" { + name = "aws-load-balancer-controller" + namespace = "kube-system" + repository = "https://aws.github.io/eks-charts" + chart = "aws-load-balancer-controller" + version = var.controller_chart_version + atomic = true + wait = true + timeout = 600 + values = [yamlencode({ + clusterName = var.cluster_name + region = var.region + vpcId = var.vpc_id + replicaCount = 2 + defaultTags = local.labels + serviceAccount = { + create = true + name = "aws-load-balancer-controller" + annotations = { "eks.amazonaws.com/role-arn" = var.controller_role_arn } + } + })] +} +resource "aws_acm_certificate" "grafana" { + domain_name = var.grafana_hostname + validation_method = "DNS" + tags = local.labels + lifecycle { create_before_destroy = true } +} +resource "aws_route53_record" "validation" { + for_each = { + for option in aws_acm_certificate.grafana.domain_validation_options : option.domain_name => { + name = option.resource_record_name + type = option.resource_record_type + value = option.resource_record_value + } + } + zone_id = var.hosted_zone_id + name = each.value.name + type = each.value.type + ttl = 60 + records = [each.value.value] +} +resource "aws_acm_certificate_validation" "grafana" { + certificate_arn = aws_acm_certificate.grafana.arn + validation_record_fqdns = [for record in aws_route53_record.validation : record.fqdn] +} +resource "kubernetes_ingress_v1" "grafana" { + wait_for_load_balancer = true + metadata { + name = "grafana" + namespace = kubernetes_namespace_v1.monitoring.metadata[0].name + labels = local.labels + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/load-balancer-name" = var.load_balancer_name + "alb.ingress.kubernetes.io/subnets" = join(",", var.public_subnet_ids) + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.ingress_cidrs) + "alb.ingress.kubernetes.io/listen-ports" = jsonencode([{ HTTP = 80 }, { HTTPS = 443 }]) + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/ssl-policy" = "ELBSecurityPolicy-TLS13-1-2-2021-06" + "alb.ingress.kubernetes.io/certificate-arn" = aws_acm_certificate_validation.grafana.certificate_arn + "alb.ingress.kubernetes.io/healthcheck-path" = "/api/health" + "alb.ingress.kubernetes.io/tags" = join(",", [for key, value in local.labels : "${key}=${value}"]) + } + } + spec { + ingress_class_name = "alb" + rule { + host = var.grafana_hostname + http { + path { + path = "/" + path_type = "Prefix" + backend { + service { + name = "${helm_release.monitoring.name}-grafana" + port { number = 80 } + } + } + } + } + } + } + depends_on = [helm_release.load_balancer_controller] +} +# The controller owns the ALB; Terraform reads it after the ingress receives its hostname. +data "aws_lb" "grafana" { + name = var.load_balancer_name + depends_on = [kubernetes_ingress_v1.grafana] +} +resource "aws_route53_record" "grafana" { + zone_id = var.hosted_zone_id + name = var.grafana_hostname + type = "A" + alias { + name = data.aws_lb.grafana.dns_name + zone_id = data.aws_lb.grafana.zone_id + evaluate_target_health = true + } +} diff --git a/modules/observability/main.tf b/modules/observability/main.tf new file mode 100644 index 0000000..b01864b --- /dev/null +++ b/modules/observability/main.tf @@ -0,0 +1,75 @@ +locals { + labels = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) +} +resource "helm_release" "monitoring" { + name = "monitoring" + namespace = var.namespace + create_namespace = false + repository = "https://prometheus-community.github.io/helm-charts" + chart = "kube-prometheus-stack" + version = var.chart_version + atomic = true + cleanup_on_fail = true + timeout = 900 + wait = true + depends_on = [kubernetes_namespace_v1.monitoring] + values = [yamlencode({ + commonLabels = local.labels + # EKS does not expose these managed control-plane scrape targets. + kubeEtcd = { enabled = false } + kubeControllerManager = { enabled = false } + kubeScheduler = { enabled = false } + kubeProxy = { enabled = false } + defaultRules = { rules = { etcd = false, kubeControllerManager = false, kubeSchedulerAlerting = false, kubeSchedulerRecording = false, kubeProxy = false } } + alertmanager = { enabled = false } + grafana = { + # Grafana's chart generates its own password; no password in tfvars or Helm values. + adminPassword = null + "grafana.ini" = { + server = { domain = var.grafana_hostname, root_url = "https://${var.grafana_hostname}" } + users = { allow_sign_up = false } + "auth.anonymous" = { enabled = false } + security = { cookie_secure = true, cookie_samesite = "lax" } + } + service = { type = "ClusterIP" } + persistence = { enabled = true, storageClassName = kubernetes_storage_class_v1.monitoring.metadata[0].name, size = var.grafana_storage_size } + resources = { requests = { cpu = "100m", memory = "128Mi" }, limits = { memory = "512Mi" } } + podLabels = local.labels + } + prometheus = { + service = { type = "ClusterIP" } + prometheusSpec = { + retention = var.prometheus_retention + retentionSize = "${floor(tonumber(trimsuffix(var.prometheus_storage_size, "Gi")) * 0.8)}GB" + externalLabels = local.labels + podMetadata = { labels = local.labels } + resources = { requests = { cpu = "200m", memory = var.prometheus_memory_request }, limits = { memory = var.prometheus_memory_limit } } + storageSpec = { volumeClaimTemplate = { spec = { + storageClassName = kubernetes_storage_class_v1.monitoring.metadata[0].name + accessModes = ["ReadWriteOnce"] + resources = { requests = { storage = var.prometheus_storage_size } } + } } } + } + } + })] +} + +resource "kubernetes_namespace_v1" "monitoring" { + metadata { + name = var.namespace + labels = local.labels + } +} +resource "kubernetes_storage_class_v1" "monitoring" { + metadata { + name = "monitoring-gp3" + labels = local.labels + } + storage_provisioner = "ebs.csi.aws.com" + reclaim_policy = "Delete" + volume_binding_mode = "WaitForFirstConsumer" + allow_volume_expansion = true + parameters = merge({ type = "gp3", encrypted = "true" }, { + for i, k in sort(keys(local.labels)) : "tagSpecification_${i + 1}" => "${k}=${local.labels[k]}" + }) +} diff --git a/modules/observability/outputs.tf b/modules/observability/outputs.tf new file mode 100644 index 0000000..4d55ed5 --- /dev/null +++ b/modules/observability/outputs.tf @@ -0,0 +1,28 @@ +output "namespace" { + description = "Monitoring namespace." + value = helm_release.monitoring.namespace +} + +output "release_name" { + description = "Helm release name." + value = helm_release.monitoring.name +} + +output "grafana_service" { + description = "Grafana ClusterIP service for port forwarding." + value = "${helm_release.monitoring.name}-grafana" +} + +output "grafana_secret" { + description = "Secret holding the generated Grafana admin password." + value = "${helm_release.monitoring.name}-grafana" +} + +output "grafana_url" { + description = "Public Grafana HTTPS URL, with authentication required." + value = "https://${aws_route53_record.grafana.name}" +} +output "load_balancer_dns_name" { + description = "Controller-managed ALB DNS name." + value = data.aws_lb.grafana.dns_name +} diff --git a/modules/observability/variables.tf b/modules/observability/variables.tf new file mode 100644 index 0000000..ae5faa6 --- /dev/null +++ b/modules/observability/variables.tf @@ -0,0 +1,95 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "chart_version" { + description = "Pinned kube-prometheus-stack chart version." + type = string +} + +variable "namespace" { + description = "Kubernetes namespace for monitoring." + type = string +} + +variable "prometheus_retention" { + description = "Prometheus local retention duration." + type = string +} + +variable "prometheus_storage_size" { + description = "Prometheus gp3 PVC capacity." + type = string +} + +variable "grafana_storage_size" { + description = "Grafana gp3 PVC capacity." + type = string +} + +variable "prometheus_memory_request" { + description = "Prometheus memory request in Kubernetes units." + type = string +} + +variable "prometheus_memory_limit" { + description = "Prometheus memory limit in Kubernetes units." + type = string +} + +variable "region" { + description = "AWS region, supplied explicitly to the ALB controller." + type = string +} + +variable "cluster_name" { + description = "EKS cluster name for the ALB controller." + type = string +} + +variable "vpc_id" { + description = "Cluster VPC for ALB reconciliation." + type = string +} + +variable "public_subnet_ids" { + description = "Public subnets for an internet-facing ALB in at least two zones." + type = list(string) +} + +variable "controller_role_arn" { + description = "IRSA role for the AWS Load Balancer Controller." + type = string +} + +variable "controller_chart_version" { + description = "Pinned AWS Load Balancer Controller chart version." + type = string +} + +variable "hosted_zone_id" { + description = "Existing public Route 53 hosted zone ID." + type = string +} + +variable "grafana_hostname" { + description = "Public Grafana hostname covered by the ACM certificate." + type = string +} + +variable "load_balancer_name" { + description = "Unique ALB name, at most 32 characters." + type = string +} + +variable "ingress_cidrs" { + description = "Client IPv4 CIDRs allowed to connect to Grafana." + type = list(string) +} diff --git a/modules/observability/versions.tf b/modules/observability/versions.tf new file mode 100644 index 0000000..fc563ab --- /dev/null +++ b/modules/observability/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { source = "hashicorp/aws", version = "~> 6.67.0" } + kubernetes = { source = "hashicorp/kubernetes", version = "~> 2.38.0" } + helm = { source = "hashicorp/helm", version = "~> 3.3.0" } + } +} diff --git a/modules/pipeline/main.tf b/modules/pipeline/main.tf new file mode 100644 index 0000000..bbdd650 --- /dev/null +++ b/modules/pipeline/main.tf @@ -0,0 +1,133 @@ +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} +locals { + tags = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) + cluster_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-*" +} +data "aws_iam_policy_document" "trust" { + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [var.oidc_provider_arn] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = ["repo:${var.github_repository}:environment:${var.github_environment}"] + } + } +} +resource "aws_iam_role" "this" { + name = var.role_name + assume_role_policy = data.aws_iam_policy_document.trust.json + max_session_duration = 3600 + tags = local.tags +} +data "aws_iam_policy_document" "permissions" { + statement { + sid = "StateBucket" + actions = ["s3:ListBucket"] + resources = [var.state_bucket_arn] + condition { + test = "StringLike" + variable = "s3:prefix" + values = ["environments/", "${var.state_prefix}*"] + } + } + statement { + sid = "StateObjects" + actions = var.read_only ? ["s3:GetObject"] : ["s3:GetObject", "s3:PutObject"] + resources = ["${var.state_bucket_arn}/${var.state_prefix}*"] + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "StateLockFiles" + actions = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"] + resources = ["${var.state_bucket_arn}/${var.state_prefix}*.tflock"] + } + } + statement { + sid = "RegionalServices" + actions = var.read_only ? ["ec2:Describe*", "eks:Describe*", "eks:List*", "logs:Describe*", "logs:ListTagsForResource", "autoscaling:Describe*", "elasticloadbalancing:Describe*", "acm:DescribeCertificate", "acm:ListCertificates", "acm:ListTagsForCertificate"] : ["ec2:*", "eks:*", "logs:CreateLogGroup", "logs:DeleteLogGroup", "logs:PutRetentionPolicy", "logs:DeleteRetentionPolicy", "logs:Describe*", "logs:ListTagsForResource", "logs:TagResource", "logs:UntagResource", "autoscaling:Describe*", "autoscaling:CreateOrUpdateTags", "autoscaling:DeleteTags", "elasticloadbalancing:Describe*", "acm:RequestCertificate", "acm:DeleteCertificate", "acm:DescribeCertificate", "acm:ListCertificates", "acm:ListTagsForCertificate", "acm:AddTagsToCertificate", "acm:RemoveTagsFromCertificate"] + resources = ["*"] + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [var.region] + } + } + statement { + sid = "DNSRead" + actions = ["route53:ListHostedZones", "route53:ListHostedZonesByName"] + resources = ["*"] + } + statement { + sid = "DNSZone" + actions = var.read_only ? ["route53:GetHostedZone", "route53:ListResourceRecordSets", "route53:ListTagsForResource"] : ["route53:GetHostedZone", "route53:ListResourceRecordSets", "route53:ListTagsForResource", "route53:ChangeResourceRecordSets"] + resources = ["arn:${data.aws_partition.current.partition}:route53:::hostedzone/${var.hosted_zone_id}"] + } + statement { + sid = "DNSChangeStatus" + actions = ["route53:GetChange"] + resources = ["arn:${data.aws_partition.current.partition}:route53:::change/*"] + } + statement { + sid = "IAMRead" + actions = ["iam:Get*", "iam:List*"] + resources = ["*"] + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "ClusterRoles" + actions = ["iam:CreateRole", "iam:DeleteRole", "iam:UpdateAssumeRolePolicy", "iam:AttachRolePolicy", "iam:DetachRolePolicy", "iam:TagRole", "iam:UntagRole", "iam:PutRolePolicy", "iam:DeleteRolePolicy"] + resources = [local.cluster_role_arn] + } + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "PassClusterRoles" + actions = ["iam:PassRole"] + resources = [local.cluster_role_arn] + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["eks.amazonaws.com", "ec2.amazonaws.com"] + } + } + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "ClusterOIDC" + actions = ["iam:CreateOpenIDConnectProvider", "iam:DeleteOpenIDConnectProvider", "iam:TagOpenIDConnectProvider", "iam:UntagOpenIDConnectProvider", "iam:UpdateOpenIDConnectProviderThumbprint"] + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${var.region}.amazonaws.com/id/*"] + } + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "EKSServiceLinkedRoles" + actions = ["iam:CreateServiceLinkedRole"] + resources = ["*"] + condition { + test = "StringEquals" + variable = "iam:AWSServiceName" + values = ["eks.amazonaws.com", "eks-nodegroup.amazonaws.com", "autoscaling.amazonaws.com"] + } + } + } +} +resource "aws_iam_role_policy" "this" { + name = "terraform" + role = aws_iam_role.this.id + policy = data.aws_iam_policy_document.permissions.json +} diff --git a/modules/pipeline/outputs.tf b/modules/pipeline/outputs.tf new file mode 100644 index 0000000..4a8a992 --- /dev/null +++ b/modules/pipeline/outputs.tf @@ -0,0 +1,9 @@ +output "role_arn" { + description = "Role ARN for the GitHub environment AWS_ROLE_ARN variable." + value = aws_iam_role.this.arn +} + +output "oidc_subject" { + description = "Exact accepted GitHub OIDC subject." + value = "repo:${var.github_repository}:environment:${var.github_environment}" +} diff --git a/modules/pipeline/variables.tf b/modules/pipeline/variables.tf new file mode 100644 index 0000000..984c8a5 --- /dev/null +++ b/modules/pipeline/variables.tf @@ -0,0 +1,60 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "github_repository" { + description = "GitHub owner/repository allowed to obtain credentials." + type = string +} + +variable "github_environment" { + description = "Exact GitHub environment in the OIDC subject." + type = string +} + +variable "oidc_provider_arn" { + description = "Shared account-level GitHub OIDC provider ARN." + type = string +} + +variable "role_name" { + description = "IAM role name for this environment and permission level." + type = string +} + +variable "cluster_name" { + description = "Cluster whose IAM role prefix the deploy role may manage." + type = string +} + +variable "region" { + description = "AWS region in which service operations are permitted." + type = string +} + +variable "state_bucket_arn" { + description = "State bucket ARN." + type = string +} + +variable "state_prefix" { + description = "Environment-specific state key prefix including trailing slash." + type = string +} + +variable "read_only" { + description = "Create a plan role without AWS write permissions." + type = bool +} + +variable "hosted_zone_id" { + description = "Existing Route 53 zone in which certificates and ingress records are managed." + type = string +} diff --git a/modules/pipeline/versions.tf b/modules/pipeline/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/modules/pipeline/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100755 index 0000000..f5ad676 --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Capture and audit this reference's AWS resources. All AWS calls are read-only.""" +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +PROJECT = "eks-platform-reference" + + +def aws(region, service, operation, missing_ok=False, **options): + command = ["aws", service, operation, "--region", region, "--output", "json", "--no-cli-pager"] + for key, value in options.items(): + command += ["--" + key.replace("_", "-"), json.dumps(value) if isinstance(value, (list, dict)) else str(value)] + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode: + # Missing resources are expected during audit; denied or failed calls are not proof of deletion. + missing_errors = { + ("eks", "describe-cluster"): "ResourceNotFoundException", + ("iam", "get-role"): "NoSuchEntity", + ("acm", "describe-certificate"): "ResourceNotFoundException", + ("elbv2", "describe-load-balancers"): "LoadBalancerNotFound", + ("elbv2", "describe-target-groups"): "TargetGroupNotFound", + } + missing_error = missing_errors.get((service, operation)) + if missing_ok and missing_error and f"({missing_error})" in result.stderr: + return {} + raise RuntimeError(f"{service} {operation}: {result.stderr.strip()}") + return json.loads(result.stdout) + + +def owned(tags, environment): + values = {tag["Key"]: tag["Value"] for tag in tags or []} + return values.get("project") == PROJECT and values.get("environment") == environment + + +def collect(region, environment, cluster_name): + resources = set() + + def add(kind, identifier): + resources.add((kind, identifier)) + + filters = [{"Name": "tag:project", "Values": [PROJECT]}, {"Name": "tag:environment", "Values": [environment]}] + cluster = aws(region, "eks", "describe-cluster", missing_ok=True, name=cluster_name).get("cluster") + if cluster: + add("eks-cluster", cluster["name"]) + for name in aws(region, "eks", "list-nodegroups", cluster_name=cluster_name)["nodegroups"]: + add("eks-nodegroup", name) + for name in aws(region, "eks", "list-addons", cluster_name=cluster_name)["addons"]: + add("eks-addon", name) + + vpcs = aws(region, "ec2", "describe-vpcs", filters=filters)["Vpcs"] + vpc_ids = {vpc["VpcId"] for vpc in vpcs} + if cluster: + vpc_ids.add(cluster["resourcesVpcConfig"]["vpcId"]) + subnet_ids = set() + for vpc_id in vpc_ids: + add("vpc", vpc_id) + scoped = [{"Name": "vpc-id", "Values": [vpc_id]}] + for operation, response_key, id_key, kind in [ + ("describe-subnets", "Subnets", "SubnetId", "subnet"), + ("describe-route-tables", "RouteTables", "RouteTableId", "route-table"), + ("describe-security-groups", "SecurityGroups", "GroupId", "security-group"), + ("describe-network-interfaces", "NetworkInterfaces", "NetworkInterfaceId", "network-interface"), + ("describe-vpc-endpoints", "VpcEndpoints", "VpcEndpointId", "vpc-endpoint"), + ]: + for item in aws(region, "ec2", operation, filters=scoped)[response_key]: + add(kind, item[id_key]) + if kind == "subnet": + subnet_ids.add(item[id_key]) + for gateway in aws(region, "ec2", "describe-nat-gateways", filter=scoped)["NatGateways"]: + if gateway["State"] != "deleted": + add("nat-gateway", gateway["NatGatewayId"]) + for address in gateway["NatGatewayAddresses"]: + if address.get("AllocationId"): + add("elastic-ip", address["AllocationId"]) + for gateway in aws(region, "ec2", "describe-internet-gateways", filters=[{"Name": "attachment.vpc-id", "Values": [vpc_id]}])["InternetGateways"]: + add("internet-gateway", gateway["InternetGatewayId"]) + for reservation in aws(region, "ec2", "describe-instances", filters=scoped)["Reservations"]: + for instance in reservation["Instances"]: + if instance["State"]["Name"] != "terminated": + add("instance", instance["InstanceId"]) + for mapping in instance.get("BlockDeviceMappings", []): + if mapping.get("Ebs"): + add("ebs-volume", mapping["Ebs"]["VolumeId"]) + + for group in aws(region, "autoscaling", "describe-auto-scaling-groups")["AutoScalingGroups"]: + group_subnets = set(group.get("VPCZoneIdentifier", "").split(",")) + if group_subnets & subnet_ids or owned(group.get("Tags"), environment): + add("auto-scaling-group", group["AutoScalingGroupName"]) + + # These objects can outlive their VPC or cluster. Scan tags independently. + for operation, response_key, id_key, kind in [ + ("describe-addresses", "Addresses", "AllocationId", "elastic-ip"), + ("describe-volumes", "Volumes", "VolumeId", "ebs-volume"), + ("describe-launch-templates", "LaunchTemplates", "LaunchTemplateId", "launch-template"), + ]: + for item in aws(region, "ec2", operation, filters=filters)[response_key]: + add(kind, item[id_key]) + for snapshot in aws(region, "ec2", "describe-snapshots", owner_ids=["self"], filters=filters)["Snapshots"]: + add("ebs-snapshot", snapshot["SnapshotId"]) + for operation, response_key, id_key, kind in [ + ("describe-load-balancers", "LoadBalancers", "LoadBalancerArn", "load-balancer"), + ("describe-target-groups", "TargetGroups", "TargetGroupArn", "target-group"), + ]: + items = aws(region, "elbv2", operation)[response_key] + arns = [item[id_key] for item in items] + tags_by_arn = {} + for start in range(0, len(arns), 20): + response = aws(region, "elbv2", "describe-tags", resource_arns=arns[start:start + 20]) + tags_by_arn.update({entry["ResourceArn"]: entry["Tags"] for entry in response["TagDescriptions"]}) + for item in items: + identifier = item[id_key] + if item.get("VpcId") in vpc_ids or owned(tags_by_arn[identifier], environment): + add(kind, identifier) + for group in aws(region, "logs", "describe-log-groups", log_group_name_prefix=f"/aws/eks/{cluster_name}/")["logGroups"]: + add("log-group", group["logGroupName"]) + for role in aws(region, "iam", "list-roles")["Roles"]: + if role["RoleName"].startswith(cluster_name + "-"): + add("iam-role", role["RoleName"]) + for provider in aws(region, "iam", "list-open-id-connect-providers")["OpenIDConnectProviderList"]: + info = aws(region, "iam", "get-open-id-connect-provider", open_id_connect_provider_arn=provider["Arn"]) + if info["Url"].startswith(f"oidc.eks.{region}.") and owned(info.get("Tags"), environment): + add("irsa-provider", provider["Arn"]) + for certificate in aws(region, "acm", "list-certificates")["CertificateSummaryList"]: + arn = certificate["CertificateArn"] + tags = aws(region, "acm", "list-tags-for-certificate", certificate_arn=arn)["Tags"] + if owned(tags, environment): + add("acm-certificate", arn) + return resources + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["snapshot", "audit"]) + parser.add_argument("--region", required=True) + parser.add_argument("--environment", required=True, choices=["dev", "prod"]) + parser.add_argument("--cluster-name", required=True) + parser.add_argument("--snapshot", required=True, type=Path) + parser.add_argument("--report", type=Path) + args = parser.parse_args() + account = aws(args.region, "sts", "get-caller-identity")["Account"] + metadata = {"account_id": account, "region": args.region, "environment": args.environment, "cluster_name": args.cluster_name} + if args.mode == "snapshot": + current = collect(args.region, args.environment, args.cluster_name) + payload = dict(metadata, captured_at=datetime.now(timezone.utc).isoformat(), resources=[{"type": kind, "id": identifier} for kind, identifier in sorted(current)]) + args.snapshot.parent.mkdir(parents=True, exist_ok=True) + args.snapshot.write_text(json.dumps(payload, indent=2) + "\n") + print(f"Captured {len(current)} resources in {args.snapshot}") + return 0 + previous = json.loads(args.snapshot.read_text()) + if any(previous[key] != value for key, value in metadata.items()): + raise RuntimeError("Snapshot account, region, environment or cluster does not match this audit") + current = collect(args.region, args.environment, args.cluster_name) + captured = {(item["type"], item["id"]) for item in previous["resources"]} + # Check exact captured objects with their describe APIs even after all ownership tags are removed. + remaining = set(current) + for kind, identifier in captured - current: + if exists(args.region, kind, identifier): + remaining.add((kind, identifier)) + report = dict(metadata, audited_at=datetime.now(timezone.utc).isoformat(), remaining=[{"type": kind, "id": identifier} for kind, identifier in sorted(remaining)]) + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n") + for kind, identifier in sorted(remaining): + print(f"REMAINS {kind}: {identifier}") + if remaining: + print(f"Audit failed: {len(remaining)} resources remain.") + return 1 + print("Environment resource audit passed. Check DNS, Terraform state, and shared resources using docs/cleanup.md.") + return 0 + + +def exists(region, kind, identifier): + # Filter-based EC2 describes return an empty collection for absent objects. + ec2 = { + "vpc": ("describe-vpcs", "Vpcs", "vpc-id"), + "subnet": ("describe-subnets", "Subnets", "subnet-id"), + "route-table": ("describe-route-tables", "RouteTables", "route-table-id"), + "security-group": ("describe-security-groups", "SecurityGroups", "group-id"), + "network-interface": ("describe-network-interfaces", "NetworkInterfaces", "network-interface-id"), + "vpc-endpoint": ("describe-vpc-endpoints", "VpcEndpoints", "vpc-endpoint-id"), + "internet-gateway": ("describe-internet-gateways", "InternetGateways", "internet-gateway-id"), + "elastic-ip": ("describe-addresses", "Addresses", "allocation-id"), + "ebs-volume": ("describe-volumes", "Volumes", "volume-id"), + "ebs-snapshot": ("describe-snapshots", "Snapshots", "snapshot-id"), + } + if kind in ec2: + operation, key, filter_name = ec2[kind] + return bool(aws(region, "ec2", operation, filters=[{"Name": filter_name, "Values": [identifier]}])[key]) + if kind == "auto-scaling-group": + return bool(aws(region, "autoscaling", "describe-auto-scaling-groups", auto_scaling_group_names=[identifier])["AutoScalingGroups"]) + if kind == "launch-template": + return any(template["LaunchTemplateId"] == identifier for template in aws(region, "ec2", "describe-launch-templates")["LaunchTemplates"]) + if kind == "instance": + response = aws(region, "ec2", "describe-instances", filters=[{"Name": "instance-id", "Values": [identifier]}]) + return any(instance["State"]["Name"] != "terminated" for reservation in response["Reservations"] for instance in reservation["Instances"]) + if kind == "nat-gateway": + return any(gateway["State"] != "deleted" for gateway in aws(region, "ec2", "describe-nat-gateways", filter=[{"Name": "nat-gateway-id", "Values": [identifier]}])["NatGateways"]) + if kind in {"eks-nodegroup", "eks-addon"}: + # Absence of the cluster already proves its managed nodegroups and add-ons are gone. + # Instances, volumes and ENIs are checked independently. + return False + if kind == "eks-cluster": + return bool(aws(region, "eks", "describe-cluster", missing_ok=True, name=identifier)) + if kind == "log-group": + return any(group["logGroupName"] == identifier for group in aws(region, "logs", "describe-log-groups", log_group_name_prefix=identifier)["logGroups"]) + if kind == "iam-role": + return bool(aws(region, "iam", "get-role", missing_ok=True, role_name=identifier)) + if kind == "irsa-provider": + return any(provider["Arn"] == identifier for provider in aws(region, "iam", "list-open-id-connect-providers")["OpenIDConnectProviderList"]) + if kind == "acm-certificate": + return bool(aws(region, "acm", "describe-certificate", missing_ok=True, certificate_arn=identifier)) + if kind in {"load-balancer", "target-group"}: + if kind == "load-balancer": + return bool(aws(region, "elbv2", "describe-load-balancers", missing_ok=True, load_balancer_arns=[identifier]).get("LoadBalancers")) + return bool(aws(region, "elbv2", "describe-target-groups", missing_ok=True, target_group_arns=[identifier]).get("TargetGroups")) + raise RuntimeError(f"No deletion check for resource type {kind}") + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (RuntimeError, OSError, KeyError, ValueError) as error: + print(f"Audit incomplete: {error}", file=sys.stderr) + sys.exit(2) diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100755 index 0000000..f2a4a25 --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +terraform fmt -check -recursive +for root in bootstrap environments environments/observability; do + ( + cd "$root" + terraform init -backend=false -input=false -lockfile=readonly + terraform validate + ) +done diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..2e9713d --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +: "${TARGET:?}" "${EVENT:?}" "${OPERATION:?}" "${AWS_REGION:?}" "${STATE_BUCKET:?}" +case "$TARGET" in dev|prod) ;; *) echo "Invalid environment" >&2; exit 1 ;; esac +case "$EVENT:$OPERATION" in + pull_request:plan|push:apply|workflow_dispatch:plan|workflow_dispatch:apply) ;; + *) echo "Invalid event and operation combination" >&2; exit 1 ;; +esac +export TF_WORKSPACE="$TARGET" +cd environments +# Read cluster details from the reviewed plan rather than a second naming convention. +init_root() { + terraform init -input=false -lockfile=readonly \ + -backend-config="bucket=$STATE_BUCKET" \ + -backend-config="region=$AWS_REGION" \ + -backend-config="key=$1.tfstate" \ + -backend-config="workspace_key_prefix=environments" \ + -backend-config="encrypt=true" \ + -backend-config="use_lockfile=true" + if [[ "$OPERATION" == plan ]]; then + env -u TF_WORKSPACE terraform workspace select "$TARGET" + else + env -u TF_WORKSPACE terraform workspace select -or-create "$TARGET" + fi +} +plan_root() { + if [[ "$OPERATION" == plan ]]; then + # The plan role cannot create locks or mutate infrastructure. Apply replans under a lock. + terraform plan -input=false -lock=false -no-color -out=deploy.tfplan + else + terraform plan -input=false -lock-timeout=5m -out=deploy.tfplan + fi +} +init_root platform +plan_root +plan_json=$(terraform show -json deploy.tfplan) +if [[ "$OPERATION" == apply ]] && jq -e \ + '.resource_changes // [] | any(.type == "aws_eks_cluster" and (.change.actions | index("delete") != null))' \ + <<< "$plan_json" >/dev/null; then + echo "Cluster deletion or replacement requires monitoring-first cleanup and audit; see docs/cleanup.md before applying." >&2 + exit 1 +fi +cluster_name=$(jq -r '.planned_values.outputs.cluster_name.value' <<< "$plan_json") +cluster_region=$(jq -r '.planned_values.outputs.region.value' <<< "$plan_json") +if [[ "$AWS_REGION" != "$cluster_region" ]]; then + echo "GitHub AWS_REGION must match the Terraform region input" >&2 + exit 1 +fi +if [[ "$OPERATION" == apply ]]; then + terraform apply -input=false -lock-timeout=5m deploy.tfplan +fi +if ! status=$(aws eks describe-cluster --name "$cluster_name" --region "$cluster_region" --query cluster.status --output text 2>&1); then + if [[ "$OPERATION" == plan && "$status" == *ResourceNotFoundException* ]]; then + printf 'Monitoring plan deferred: this cluster does not exist yet. Merge applies platform first.\n' + exit 0 + fi + printf '%s\n' "$status" >&2 + exit 1 +fi +if [[ "$status" != ACTIVE ]]; then + printf 'Cluster is %s; monitoring requires an ACTIVE cluster.\n' "$status" >&2 + exit 1 +fi +cd observability +init_root observability +plan_root +if [[ "$OPERATION" == apply ]]; then + terraform apply -input=false -lock-timeout=5m deploy.tfplan +fi diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..a3581bf --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +# Synthetic fixtures and overridden AWS data sources make all plans credential-free. +( + cd bootstrap + terraform test -var-file=tests/fixtures.tfvars +) +for target in dev prod; do + ( + cd environments + TF_WORKSPACE="$target" terraform test -var="test_workspace=$target" + cd observability + TF_WORKSPACE="$target" terraform test -var="test_workspace=$target" + ) +done +python3 -m unittest discover -s tests -v diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..f967fec --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,65 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("audit", Path(__file__).parents[1] / "scripts/audit.py") +audit = importlib.util.module_from_spec(spec) +spec.loader.exec_module(audit) + + +class AuditTests(unittest.TestCase): + def test_missing_cluster_is_absent_but_denial_is_incomplete(self): + with patch.object(audit.subprocess, "run", return_value=SimpleNamespace(returncode=255, stderr="An error occurred (ResourceNotFoundException)", stdout="")): + self.assertEqual(audit.aws("test-region", "eks", "describe-cluster", missing_ok=True, name="test"), {}) + with patch.object(audit.subprocess, "run", return_value=SimpleNamespace(returncode=255, stderr="An error occurred (AccessDeniedException)", stdout="")): + with self.assertRaises(RuntimeError): + audit.aws("test-region", "eks", "describe-cluster", missing_ok=True, name="test") + + def test_unexpected_missing_error_is_not_a_pass(self): + with patch.object(audit.subprocess, "run", return_value=SimpleNamespace(returncode=255, stderr="An error occurred (ResourceNotFoundException)", stdout="")): + with self.assertRaises(RuntimeError): + audit.aws("test-region", "ec2", "describe-vpcs", missing_ok=True) + + def test_terminal_instances_and_nat_are_not_leftovers(self): + with patch.object(audit, "aws", return_value={"Reservations": [{"Instances": [{"State": {"Name": "terminated"}}]}]}): + self.assertFalse(audit.exists("test-region", "instance", "i-test")) + with patch.object(audit, "aws", return_value={"NatGateways": [{"State": "deleted"}]}): + self.assertFalse(audit.exists("test-region", "nat-gateway", "nat-test")) + with patch.object(audit, "aws", return_value={"NatGateways": [{"State": "deleting"}]}): + self.assertTrue(audit.exists("test-region", "nat-gateway", "nat-test")) + + def run_audit(self, directory, remaining, captured_exists, account="test-account"): + snapshot = Path(directory) / "before.json" + snapshot.write_text(json.dumps({"account_id": "test-account", "region": "test-region", "environment": "dev", "cluster_name": "test-cluster", "resources": [{"type": "ebs-volume", "id": "vol-test"}]})) + argv = ["audit.py", "audit", "--environment", "dev", "--region", "test-region", "--cluster-name", "test-cluster", "--snapshot", str(snapshot)] + with patch.object(audit.sys, "argv", argv), patch.object(audit, "aws", return_value={"Account": account}), patch.object(audit, "collect", return_value=remaining) as collect, patch.object(audit, "exists", return_value=captured_exists), patch("builtins.print"): + result = audit.main() + return result, collect + + def test_untagged_captured_volume_still_fails_audit(self): + with tempfile.TemporaryDirectory() as directory: + result, _ = self.run_audit(directory, set(), True) + self.assertEqual(result, 1) + + def test_no_resources_passes(self): + with tempfile.TemporaryDirectory() as directory: + result, _ = self.run_audit(directory, set(), False) + self.assertEqual(result, 0) + + def test_wrong_account_is_rejected_before_resource_inventory(self): + with tempfile.TemporaryDirectory() as directory: + snapshot = Path(directory) / "before.json" + snapshot.write_text(json.dumps({"account_id": "right-account", "region": "test-region", "environment": "dev", "cluster_name": "test-cluster", "resources": []})) + argv = ["audit.py", "audit", "--environment", "dev", "--region", "test-region", "--cluster-name", "test-cluster", "--snapshot", str(snapshot)] + with patch.object(audit.sys, "argv", argv), patch.object(audit, "aws", return_value={"Account": "wrong-account"}), patch.object(audit, "collect") as collect: + with self.assertRaises(RuntimeError): + audit.main() + collect.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_deploy.py b/tests/test_deploy.py new file mode 100644 index 0000000..ec37027 --- /dev/null +++ b/tests/test_deploy.py @@ -0,0 +1,135 @@ +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).parents[1] + +TERRAFORM = '''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +with open(os.environ['DEPLOY_TEST_LOG'], 'a') as log: + log.write(json.dumps({'args': args, 'workspace': os.environ.get('TF_WORKSPACE'), 'directory': str(Path.cwd().relative_to(os.environ['DEPLOY_TEST_REPO']))}) + '\\n') +operation = next(arg for arg in args if not arg.startswith('-')) +if operation == 'workspace' and os.environ.get('TF_WORKSPACE'): + print('Workspace selection blocked by TF_WORKSPACE', file=sys.stderr) + sys.exit(1) +stage = str(Path.cwd().relative_to(os.environ['DEPLOY_TEST_REPO'])) + ':' + operation +if stage == os.environ.get('DEPLOY_TEST_FAIL_STAGE'): + print('Synthetic Terraform failure', file=sys.stderr) + sys.exit(1) +if operation == 'show': + print(json.dumps({'planned_values': {'outputs': {'cluster_name': {'value': 'test-cluster'}, 'region': {'value': 'test-region'}}}, 'resource_changes': [{'type': 'aws_eks_cluster', 'change': {'actions': json.loads(os.environ['DEPLOY_TEST_CLUSTER_ACTIONS'])}}]})) +''' +AWS = '''#!/usr/bin/env python3 +import os, sys +result = os.environ.get('DEPLOY_TEST_AWS_RESULT', 'ACTIVE') +if result == 'ACTIVE': + print('ACTIVE') +else: + print('An error occurred (' + result + ')', file=sys.stderr) + sys.exit(255) +''' + + +class DeployTests(unittest.TestCase): + def deploy(self, event, aws_result="ACTIVE", region="test-region", operation=None, target="dev", cluster_actions=None, fail_stage=""): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + log = directory / "commands.jsonl" + for name, content in [("terraform", TERRAFORM), ("aws", AWS)]: + executable = directory / name + executable.write_text(content) + executable.chmod(0o755) + if operation is None: + operation = "plan" if event == "pull_request" else "apply" + environment = dict(os.environ, PATH=str(directory) + os.pathsep + os.environ["PATH"], DEPLOY_TEST_LOG=str(log), DEPLOY_TEST_REPO=str(REPO), DEPLOY_TEST_AWS_RESULT=aws_result, DEPLOY_TEST_CLUSTER_ACTIONS=json.dumps(cluster_actions or ["create"]), DEPLOY_TEST_FAIL_STAGE=fail_stage, TARGET=target, EVENT=event, OPERATION=operation, AWS_REGION=region, STATE_BUCKET="test-state-bucket") + result = subprocess.run(["bash", str(REPO / "scripts/deploy.sh")], cwd=directory, env=environment, capture_output=True, text=True) + commands = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + return result, commands + + def test_pr_plans_both_roots_without_apply_or_remote_locks(self): + result, commands = self.deploy("pull_request") + self.assertEqual(result.returncode, 0, result.stderr) + plans = [command for command in commands if command["args"][0] == "plan"] + self.assertEqual(len(plans), 2) + self.assertTrue(all("-lock=false" in command["args"] for command in plans)) + self.assertFalse(any(arg.startswith("-var-file") for command in plans for arg in command["args"])) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + self.assertTrue(all(command["workspace"] is None for command in commands if command["args"][0] == "workspace")) + + def test_apply_plans_platform_before_monitoring(self): + result, commands = self.deploy("push") + self.assertEqual(result.returncode, 0, result.stderr) + changes = [(command["directory"], command["args"][0]) for command in commands if command["args"][0] in {"plan", "apply"}] + self.assertEqual(changes, [("environments", "plan"), ("environments", "apply"), ("environments/observability", "plan"), ("environments/observability", "apply")]) + self.assertTrue(all(command["workspace"] == "dev" for command in commands if command["args"][0] in {"plan", "apply"})) + + def test_only_missing_cluster_defers_monitoring(self): + result, commands = self.deploy("pull_request", "ResourceNotFoundException") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Monitoring plan deferred", result.stdout) + self.assertEqual(sum(command["args"][0] == "plan" for command in commands), 1) + denied, _ = self.deploy("pull_request", "AccessDeniedException") + self.assertNotEqual(denied.returncode, 0) + self.assertNotIn("Monitoring plan deferred", denied.stdout) + + def test_region_mismatch_prevents_apply(self): + result, commands = self.deploy("push", region="wrong-region") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + + def test_manual_prod_plan_is_read_only(self): + result, commands = self.deploy("workflow_dispatch", operation="plan", target="prod", cluster_actions=["delete", "create"]) + self.assertEqual(result.returncode, 0, result.stderr) + plans = [command for command in commands if command["args"][0] == "plan"] + self.assertEqual(len(plans), 2) + self.assertTrue(all("-lock=false" in command["args"] for command in plans)) + self.assertTrue(all(command["workspace"] == "prod" for command in plans)) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + + def test_manual_apply_retains_state_locking(self): + result, commands = self.deploy("workflow_dispatch", operation="apply") + self.assertEqual(result.returncode, 0, result.stderr) + changes = [command for command in commands if command["args"][0] in {"plan", "apply"}] + self.assertEqual([command["args"][0] for command in changes], ["plan", "apply", "plan", "apply"]) + self.assertTrue(all("-lock-timeout=5m" in command["args"] for command in changes)) + + def test_manual_plan_defers_missing_cluster(self): + result, commands = self.deploy("workflow_dispatch", "ResourceNotFoundException", operation="plan") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sum(command["args"][0] == "plan" for command in commands), 1) + + def test_pr_cannot_select_apply(self): + result, commands = self.deploy("pull_request", operation="apply") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(commands) + + def test_cluster_deletion_or_replacement_prevents_apply(self): + for actions in (["delete"], ["delete", "create"], ["create", "delete"]): + with self.subTest(actions=actions): + result, commands = self.deploy("push", cluster_actions=actions) + self.assertNotEqual(result.returncode, 0) + self.assertIn("monitoring", result.stderr) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + + def test_in_place_cluster_update_is_allowed(self): + result, commands = self.deploy("push", cluster_actions=["update"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sum(command["args"][0] == "apply" for command in commands), 2) + + def test_failed_terraform_stage_stops_later_stages(self): + stages = [("environments", "plan"), ("environments", "apply"), ("environments/observability", "plan"), ("environments/observability", "apply")] + for index, (directory, operation) in enumerate(stages): + with self.subTest(directory=directory, operation=operation): + result, commands = self.deploy("push", fail_stage=directory + ":" + operation) + self.assertNotEqual(result.returncode, 0) + observed = [(command["directory"], command["args"][0]) for command in commands if command["args"][0] in {"plan", "apply"}] + self.assertEqual(observed, stages[:index + 1]) + + +if __name__ == "__main__": + unittest.main() From c8d4d51df5721744af135ae2a6b65893a6aa0f0e Mon Sep 17 00:00:00 2001 From: Patrick Hudson <5728403+patrick-hudson@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:19:57 -0500 Subject: [PATCH 2/4] fix: validate workspaces and make deployments explicit Validate dev and prod from a fresh runner without selecting remote state. Keep main pushes validation-only and require manual apply runs. Update one PR plan comment per environment with safe change counts. --- .github/workflows/terraform.yml | 18 +++++++-- README.md | 14 +++++-- docs/verification.md | 12 +++++- scripts/check.sh | 8 +++- scripts/comment-plan.sh | 34 ++++++++++++++++ scripts/deploy.sh | 43 ++++++++++++++++---- tests/test_deploy.py | 43 +++++++++++++++++--- tests/test_plan_comment.py | 71 +++++++++++++++++++++++++++++++++ 8 files changed, 221 insertions(+), 22 deletions(-) create mode 100755 scripts/comment-plan.sh create mode 100644 tests/test_plan_comment.py diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 3be50c8..05d3964 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -36,8 +36,8 @@ jobs: needs: validate # Fork code never receives an OIDC session or access to state. Its validate job still runs. if: >- - (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && - (github.event_name != 'workflow_dispatch' || inputs.operation == 'plan' || github.ref == 'refs/heads/main') + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'workflow_dispatch' && (inputs.operation == 'plan' || github.ref == 'refs/heads/main')) runs-on: ubuntu-latest strategy: fail-fast: false @@ -47,6 +47,7 @@ jobs: permissions: contents: read id-token: write + pull-requests: write concurrency: group: terraform-${{ matrix.target }} cancel-in-progress: false @@ -55,7 +56,7 @@ jobs: TF_INPUT: 'false' TARGET: ${{ matrix.target }} EVENT: ${{ github.event_name }} - OPERATION: ${{ github.event_name == 'pull_request' && 'plan' || github.event_name == 'workflow_dispatch' && inputs.operation || 'apply' }} + OPERATION: ${{ github.event_name == 'pull_request' && 'plan' || inputs.operation }} AWS_REGION: ${{ vars.AWS_REGION }} STATE_BUCKET: ${{ vars.TF_STATE_BUCKET }} steps: @@ -73,4 +74,15 @@ jobs: allowed-account-ids: ${{ vars.AWS_ACCOUNT_ID }} - name: Plan or apply platform, then monitoring shell: bash + env: + PLAN_REPORT: ${{ runner.temp }}/terraform-plan-${{ matrix.target }}.md run: bash scripts/deploy.sh + - name: Update PR plan comment + if: always() && github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PLAN_JOB_STATUS: ${{ job.status }} + PLAN_REPORT: ${{ runner.temp }}/terraform-plan-${{ matrix.target }}.md + run: bash scripts/comment-plan.sh diff --git a/README.md b/README.md index b77143b..d2a3e0b 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ environments/ shared platform root terraform.tfvars platform settings keyed by dev/prod workspace observability/ shared monitoring root terraform.tfvars monitoring settings keyed by dev/prod workspace -.github/workflows/terraform.yml validation, PR plans and main applies +.github/workflows/terraform.yml validation, PR plans and manual applies scripts/ local checks and workflow deployment ``` @@ -258,11 +258,19 @@ credential-free checks only. No `pull_request_target` execution is used. For an environment without a cluster yet, the platform plan runs and the live monitoring plan is explicitly deferred. Static validation and offline monitoring plans still run. Once a cluster exists, both live plans run. PR plans are unlocked -because their roles cannot write lock files; they can be stale. Main applies +because their roles cannot write lock files; they can be stale. Manual applies create new saved plans under S3 locks and apply those exact files in the same job. PR plan artifacts are never reused for deployment or published as artifacts. -A merge to `main` applies platform then monitoring for both environments. +Each PR plan updates one bot comment per environment with platform and monitoring +resource counts: add, change, destroy and replace. The comment includes the +commit, job result and workflow link. Failed plans and deferred monitoring are +marked explicitly. Comments contain counts only; raw plan values and JSON are +not posted. Retries update the existing comment for that environment. + +A merge or push to `main` runs validation only. It does not request AWS +credentials or run a live plan or apply. This lets the demo be merged and +maintained without starting billable infrastructure. Manual runs select one environment and either `plan` or `apply`; they default to a dev plan. Manual applies require `main`. Deployment jobs serialize per environment and are never canceled mid-apply. The prod environment approval sits before AWS diff --git a/docs/verification.md b/docs/verification.md index cbb4b92..c40cde0 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -10,7 +10,7 @@ Run these checks from the repository root: ```bash bash scripts/check.sh bash scripts/test.sh -bash -n scripts/check.sh scripts/test.sh scripts/deploy.sh +bash -n scripts/check.sh scripts/test.sh scripts/deploy.sh scripts/comment-plan.sh ``` The Terraform tests run plans with synthetic credentials and replace the AWS @@ -37,7 +37,7 @@ an actual second no-change plan have been verified. For the first deployment: The local check results are: formatting passed, three roots validated, eleven offline plan runs passed (including missing-workspace-settings and wrong-zone -rejection runs), and seventeen Python audit and deployment-script tests passed. +rejection runs), and twenty-four Python audit, deployment and PR-comment tests passed. Both environment roots load their settings from `terraform.tfvars`; the dev and prod tests pass without a var-file. A separate offline test confirmed that the default workspace fails variable validation. The script tests cover @@ -46,6 +46,14 @@ selection, manual operation selection, missing-cluster deferral and errors that must stop an apply. Cluster deletion and replacement stop before apply; initial creation and in-place updates are allowed. Failures at each plan/apply stage stop later deployment stages. +Push-triggered deployments are rejected; merges run validation only and live +applies require an explicit manual workflow run. Validation selects dev and +prod through `TF_WORKSPACE` without changing the local workspace selection. +The checks pass with the fresh runner's default-workspace context. +PR-comment tests cover counts without resource values, replacements, no-ops, +updates, deferred monitoring, failed plans, paginated comment lookup, and +updating only the matching environment's bot comment. GitHub API calls are +mocked; live comment publication requires an approved PR plan job. Deployment scripts are tested from a directory outside the repository too. Bash syntax checks passed. Both pinned Helm charts rendered using the exact Terraform-generated dev values: 104 monitoring objects, generated Grafana credentials, the expected gp3 PVCs, disabled anonymous diff --git a/scripts/check.sh b/scripts/check.sh index f2a4a25..1546c34 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -6,6 +6,12 @@ for root in bootstrap environments environments/observability; do ( cd "$root" terraform init -backend=false -input=false -lockfile=readonly - terraform validate + if [[ "$root" == bootstrap ]]; then + terraform validate + else + for workspace in dev prod; do + TF_WORKSPACE="$workspace" terraform validate + done + fi ) done diff --git a/scripts/comment-plan.sh b/scripts/comment-plan.sh new file mode 100755 index 0000000..8df9232 --- /dev/null +++ b/scripts/comment-plan.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${TARGET:?}" "${GITHUB_REPOSITORY:?}" "${PR_NUMBER:?}" "${HEAD_SHA:?}" \ + "${GITHUB_SERVER_URL:?}" "${GITHUB_RUN_ID:?}" "${PLAN_JOB_STATUS:?}" "${PLAN_REPORT:?}" "${GH_TOKEN:?}" +case "$TARGET" in dev|prod) ;; *) echo "Invalid environment" >&2; exit 1 ;; esac +directory=$(mktemp -d) +trap 'rm -rf "$directory"' EXIT +marker="" +{ + printf '%s\n### Terraform plan: %s\n\n' "$marker" "$TARGET" + printf 'Job result: **%s**. Commit: `%s`. [Workflow run](%s/%s/actions/runs/%s).\n\n' \ + "$PLAN_JOB_STATUS" "$HEAD_SHA" "$GITHUB_SERVER_URL" "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" + if [[ -s "$PLAN_REPORT" ]]; then + cat "$PLAN_REPORT" + printf '\nReplacements also count toward Add and Destroy. Counts describe managed resources; output-only changes are not included.\n' + printf '\nA deferred monitoring plan requires an existing cluster. Check the workflow logs for any failure or incomplete results.\n' + else + printf 'No plan results were produced. Check the workflow logs.\n' + fi +} > "$directory/comment.md" +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + cat "$directory/comment.md" >> "$GITHUB_STEP_SUMMARY" +fi +jq -n --rawfile body "$directory/comment.md" '{body: $body}' > "$directory/payload.json" +comments=$(gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments") +comment_id=$(jq -r --arg marker "$marker" ' + [.[][] | select(.user.login == "github-actions[bot]" and ((.body // "") | startswith($marker)))] | + sort_by(.id) | last | .id // empty +' <<< "$comments") +if [[ -n "$comment_id" ]]; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$comment_id" --input "$directory/payload.json" --silent +else + gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --input "$directory/payload.json" --silent +fi diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 2e9713d..b7e2fd0 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -4,10 +4,18 @@ cd "$(dirname "$0")/.." : "${TARGET:?}" "${EVENT:?}" "${OPERATION:?}" "${AWS_REGION:?}" "${STATE_BUCKET:?}" case "$TARGET" in dev|prod) ;; *) echo "Invalid environment" >&2; exit 1 ;; esac case "$EVENT:$OPERATION" in - pull_request:plan|push:apply|workflow_dispatch:plan|workflow_dispatch:apply) ;; + pull_request:plan|workflow_dispatch:plan|workflow_dispatch:apply) ;; *) echo "Invalid event and operation combination" >&2; exit 1 ;; esac export TF_WORKSPACE="$TARGET" +if [[ -n "${PLAN_REPORT:-}" ]]; then + printf '| Root | Result | Add | Change | Destroy | Replace |\n| --- | --- | ---: | ---: | ---: | ---: |\n' > "$PLAN_REPORT" +fi +report_status() { + if [[ -n "${PLAN_REPORT:-}" ]]; then + printf '| %s | %s | — | — | — | — |\n' "$1" "$2" >> "$PLAN_REPORT" + fi +} cd environments # Read cluster details from the reviewed plan rather than a second naming convention. init_root() { @@ -25,16 +33,36 @@ init_root() { fi } plan_root() { + local root="$1" + local -a options=(-input=false -no-color -out=deploy.tfplan) if [[ "$OPERATION" == plan ]]; then # The plan role cannot create locks or mutate infrastructure. Apply replans under a lock. - terraform plan -input=false -lock=false -no-color -out=deploy.tfplan + options+=(-lock=false) else - terraform plan -input=false -lock-timeout=5m -out=deploy.tfplan + options+=(-lock-timeout=5m) + fi + if ! terraform plan "${options[@]}"; then + report_status "$root" Failed + return 1 + fi + if ! plan_json=$(terraform show -json deploy.tfplan); then + report_status "$root" Failed + return 1 + fi + if [[ -n "${PLAN_REPORT:-}" ]]; then + jq -r --arg root "$root" ' + [.resource_changes[]?.change.actions] as $actions | + [$root, "Planned", + ([$actions[] | select(index("create"))] | length), + ([$actions[] | select(index("update"))] | length), + ([$actions[] | select(index("delete"))] | length), + ([$actions[] | select(index("create") and index("delete"))] | length)] | + "| " + (map(tostring) | join(" | ")) + " |" + ' <<< "$plan_json" >> "$PLAN_REPORT" fi } init_root platform -plan_root -plan_json=$(terraform show -json deploy.tfplan) +plan_root Platform if [[ "$OPERATION" == apply ]] && jq -e \ '.resource_changes // [] | any(.type == "aws_eks_cluster" and (.change.actions | index("delete") != null))' \ <<< "$plan_json" >/dev/null; then @@ -52,7 +80,8 @@ if [[ "$OPERATION" == apply ]]; then fi if ! status=$(aws eks describe-cluster --name "$cluster_name" --region "$cluster_region" --query cluster.status --output text 2>&1); then if [[ "$OPERATION" == plan && "$status" == *ResourceNotFoundException* ]]; then - printf 'Monitoring plan deferred: this cluster does not exist yet. Merge applies platform first.\n' + report_status Monitoring 'Deferred: cluster does not exist' + printf 'Monitoring plan deferred: this cluster does not exist yet. Run a manual apply to create the platform first.\n' exit 0 fi printf '%s\n' "$status" >&2 @@ -64,7 +93,7 @@ if [[ "$status" != ACTIVE ]]; then fi cd observability init_root observability -plan_root +plan_root Monitoring if [[ "$OPERATION" == apply ]]; then terraform apply -input=false -lock-timeout=5m deploy.tfplan fi diff --git a/tests/test_deploy.py b/tests/test_deploy.py index ec37027..4407713 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -22,7 +22,7 @@ print('Synthetic Terraform failure', file=sys.stderr) sys.exit(1) if operation == 'show': - print(json.dumps({'planned_values': {'outputs': {'cluster_name': {'value': 'test-cluster'}, 'region': {'value': 'test-region'}}}, 'resource_changes': [{'type': 'aws_eks_cluster', 'change': {'actions': json.loads(os.environ['DEPLOY_TEST_CLUSTER_ACTIONS'])}}]})) + print(json.dumps({'planned_values': {'outputs': {'cluster_name': {'value': 'test-cluster'}, 'region': {'value': 'test-region'}}}, 'resource_changes': [{'type': 'aws_eks_cluster', 'change': {'actions': json.loads(os.environ['DEPLOY_TEST_CLUSTER_ACTIONS']), 'after': {'password': 'sensitive-test-value'}}}]})) ''' AWS = '''#!/usr/bin/env python3 import os, sys @@ -47,7 +47,10 @@ def deploy(self, event, aws_result="ACTIVE", region="test-region", operation=Non if operation is None: operation = "plan" if event == "pull_request" else "apply" environment = dict(os.environ, PATH=str(directory) + os.pathsep + os.environ["PATH"], DEPLOY_TEST_LOG=str(log), DEPLOY_TEST_REPO=str(REPO), DEPLOY_TEST_AWS_RESULT=aws_result, DEPLOY_TEST_CLUSTER_ACTIONS=json.dumps(cluster_actions or ["create"]), DEPLOY_TEST_FAIL_STAGE=fail_stage, TARGET=target, EVENT=event, OPERATION=operation, AWS_REGION=region, STATE_BUCKET="test-state-bucket") + report = directory / "plan.md" + environment["PLAN_REPORT"] = str(report) result = subprocess.run(["bash", str(REPO / "scripts/deploy.sh")], cwd=directory, env=environment, capture_output=True, text=True) + result.plan_report = report.read_text() if report.exists() else "" commands = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] return result, commands @@ -62,7 +65,7 @@ def test_pr_plans_both_roots_without_apply_or_remote_locks(self): self.assertTrue(all(command["workspace"] is None for command in commands if command["args"][0] == "workspace")) def test_apply_plans_platform_before_monitoring(self): - result, commands = self.deploy("push") + result, commands = self.deploy("workflow_dispatch", operation="apply") self.assertEqual(result.returncode, 0, result.stderr) changes = [(command["directory"], command["args"][0]) for command in commands if command["args"][0] in {"plan", "apply"}] self.assertEqual(changes, [("environments", "plan"), ("environments", "apply"), ("environments/observability", "plan"), ("environments/observability", "apply")]) @@ -78,7 +81,7 @@ def test_only_missing_cluster_defers_monitoring(self): self.assertNotIn("Monitoring plan deferred", denied.stdout) def test_region_mismatch_prevents_apply(self): - result, commands = self.deploy("push", region="wrong-region") + result, commands = self.deploy("workflow_dispatch", operation="apply", region="wrong-region") self.assertNotEqual(result.returncode, 0) self.assertFalse(any(command["args"][0] == "apply" for command in commands)) @@ -111,13 +114,13 @@ def test_pr_cannot_select_apply(self): def test_cluster_deletion_or_replacement_prevents_apply(self): for actions in (["delete"], ["delete", "create"], ["create", "delete"]): with self.subTest(actions=actions): - result, commands = self.deploy("push", cluster_actions=actions) + result, commands = self.deploy("workflow_dispatch", operation="apply", cluster_actions=actions) self.assertNotEqual(result.returncode, 0) self.assertIn("monitoring", result.stderr) self.assertFalse(any(command["args"][0] == "apply" for command in commands)) def test_in_place_cluster_update_is_allowed(self): - result, commands = self.deploy("push", cluster_actions=["update"]) + result, commands = self.deploy("workflow_dispatch", operation="apply", cluster_actions=["update"]) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(sum(command["args"][0] == "apply" for command in commands), 2) @@ -125,11 +128,39 @@ def test_failed_terraform_stage_stops_later_stages(self): stages = [("environments", "plan"), ("environments", "apply"), ("environments/observability", "plan"), ("environments/observability", "apply")] for index, (directory, operation) in enumerate(stages): with self.subTest(directory=directory, operation=operation): - result, commands = self.deploy("push", fail_stage=directory + ":" + operation) + result, commands = self.deploy("workflow_dispatch", operation="apply", fail_stage=directory + ":" + operation) self.assertNotEqual(result.returncode, 0) observed = [(command["directory"], command["args"][0]) for command in commands if command["args"][0] in {"plan", "apply"}] self.assertEqual(observed, stages[:index + 1]) + def test_push_cannot_deploy(self): + result, commands = self.deploy("push", operation="apply") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(commands) + + def test_plan_report_counts_replacements_without_values(self): + result, _ = self.deploy("pull_request", cluster_actions=["delete", "create"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("| Platform | Planned | 1 | 0 | 1 | 1 |", result.plan_report) + self.assertIn("| Monitoring | Planned | 1 | 0 | 1 | 1 |", result.plan_report) + self.assertNotIn("sensitive-test-value", result.plan_report) + self.assertNotIn("test-cluster", result.plan_report) + + def test_plan_report_marks_deferred_and_failed_results(self): + deferred, _ = self.deploy("pull_request", "ResourceNotFoundException") + self.assertIn("| Monitoring | Deferred: cluster does not exist |", deferred.plan_report) + failed, _ = self.deploy("pull_request", fail_stage="environments/observability:plan") + self.assertNotEqual(failed.returncode, 0) + self.assertIn("| Platform | Planned | 1 | 0 | 0 | 0 |", failed.plan_report) + self.assertIn("| Monitoring | Failed |", failed.plan_report) + + def test_noop_and_update_counts(self): + for actions, counts in [(["no-op"], "0 | 0 | 0 | 0"), (["update"], "0 | 1 | 0 | 0")]: + with self.subTest(actions=actions): + result, _ = self.deploy("pull_request", cluster_actions=actions) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("| Platform | Planned | " + counts + " |", result.plan_report) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_plan_comment.py b/tests/test_plan_comment.py new file mode 100644 index 0000000..0c6d891 --- /dev/null +++ b/tests/test_plan_comment.py @@ -0,0 +1,71 @@ +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).parents[1] +GH = '''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +record = {'args': args} +if '--input' in args: + record['payload'] = json.loads(Path(args[args.index('--input') + 1]).read_text()) +with open(os.environ['COMMENT_TEST_LOG'], 'a') as log: + log.write(json.dumps(record) + '\\n') +if '--method' not in args: + print(os.environ['COMMENT_TEST_COMMENTS']) +''' + + +class PlanCommentTests(unittest.TestCase): + def comment(self, comments=None, report="| Platform | Planned | 1 | 0 | 0 | 0 |", status="success"): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + executable = directory / "gh" + executable.write_text(GH) + executable.chmod(0o755) + report_path = directory / "plan.md" + if report is not None: + report_path.write_text(report) + log = directory / "commands.jsonl" + environment = dict(os.environ, PATH=str(directory) + os.pathsep + os.environ["PATH"], + COMMENT_TEST_LOG=str(log), COMMENT_TEST_COMMENTS=json.dumps(comments or [[]]), + TARGET="dev", GITHUB_REPOSITORY="owner/repo", PR_NUMBER="1", HEAD_SHA="abc123", + GITHUB_SERVER_URL="https://github.com", GITHUB_RUN_ID="42", GH_TOKEN="test-token", + PLAN_REPORT=str(report_path), PLAN_JOB_STATUS=status, GITHUB_STEP_SUMMARY=str(directory / "summary.md")) + result = subprocess.run(["bash", str(REPO / "scripts/comment-plan.sh")], env=environment, capture_output=True, text=True) + calls = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual(result.returncode, 0, result.stderr) + return calls + + def test_creates_comment_with_counts_commit_and_run_link(self): + calls = self.comment() + self.assertIn("--paginate", calls[0]["args"]) + self.assertIn("POST", calls[1]["args"]) + self.assertIn("repos/owner/repo/issues/1/comments", calls[1]["args"]) + body = calls[1]["payload"]["body"] + self.assertTrue(body.startswith("")) + self.assertIn("abc123", body) + self.assertIn("https://github.com/owner/repo/actions/runs/42", body) + self.assertIn("| Platform | Planned | 1 | 0 | 0 | 0 |", body) + + def test_updates_only_matching_bot_comment_across_pages(self): + comments = [[{"id": 3, "user": {"login": "github-actions[bot]"}, "body": " old"}], + [{"id": 4, "user": {"login": "human"}, "body": ""}, + {"id": 5, "user": {"login": "github-actions[bot]"}, "body": ""}]] + calls = self.comment(comments) + self.assertIn("PATCH", calls[1]["args"]) + self.assertIn("repos/owner/repo/issues/comments/3", calls[1]["args"]) + + def test_failure_before_plan_produces_clear_comment(self): + calls = self.comment(report=None, status="failure") + body = calls[1]["payload"]["body"] + self.assertIn("**failure**", body) + self.assertIn("No plan results were produced", body) + + +if __name__ == "__main__": + unittest.main() From 45d2cee1c4e72f957466377518e1236d90b4ad42 Mon Sep 17 00:00:00 2001 From: Patrick Hudson <5728403+patrick-hudson@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:26:31 -0500 Subject: [PATCH 3/4] fix: use credentials action with account validation --- .github/workflows/terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 05d3964..86f9bc5 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -65,7 +65,7 @@ jobs: with: terraform_version: 1.16.5 terraform_wrapper: false - - uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} From ec54c10e9b8a322d82769ccd6773e6b91e61b367 Mon Sep 17 00:00:00 2001 From: Patrick Hudson <5728403+patrick-hudson@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:29:55 -0500 Subject: [PATCH 4/4] fix: document and test immutable GitHub OIDC subjects --- README.md | 21 ++++++++++++++++++--- bootstrap/terraform.tfvars.example | 3 ++- bootstrap/tests/pipeline.tftest.hcl | 6 +++--- bootstrap/variables.tf | 2 +- docs/verification.md | 4 +++- modules/pipeline/variables.tf | 2 +- 6 files changed, 28 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d2a3e0b..499e1a7 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,24 @@ aws sts get-caller-identity cp bootstrap/terraform.tfvars.example bootstrap/terraform.tfvars ``` -Edit that local tfvars file: choose a globally unique state bucket, set your -GitHub `owner/repository`, and confirm the cluster and role names match the -environment inputs. Confirm that `hosted_zone_id` identifies the zone whose +Read the repository value GitHub actually uses in its OIDC subject: + +```bash +gh api repos/OWNER/REPOSITORY/actions/oidc/customization/sub \ + --jq '.sub_claim_prefix | sub("^repo:"; "")' +``` + +Set `github_repository` in the local tfvars file to that exact result. New +repositories use `OWNER@OWNER-ID/REPOSITORY@REPOSITORY-ID`; older repositories +may use `OWNER/REPOSITORY`. Do not infer the value from the repository URL. +The module adds `repo:` and the exact environment to build the trust condition. +An incorrect value causes `Not authorized to perform sts:AssumeRoleWithWebIdentity`. +After correcting it, plan and apply bootstrap again to update the role trusts, +then retry the workflow. This does not require reapplying the EKS platform. +See [GitHub's subject formats](https://docs.github.com/en/actions/reference/security/oidc#immutable-subject-claims). + +Choose a globally unique state bucket and confirm the cluster and role names +match the environment inputs. Confirm that `hosted_zone_id` identifies the zone whose name servers are delegated for your domain; duplicate hosted-zone names are allowed in AWS. If this account already has GitHub's OIDC provider, set `github_oidc_provider_arn` to its ARN. There is only one provider per issuer per diff --git a/bootstrap/terraform.tfvars.example b/bootstrap/terraform.tfvars.example index bd52463..d471d7d 100644 --- a/bootstrap/terraform.tfvars.example +++ b/bootstrap/terraform.tfvars.example @@ -1,6 +1,7 @@ region = "us-east-1" state_bucket_name = "REPLACE-WITH-A-GLOBALLY-UNIQUE-BUCKET" -github_repository = "REPLACE-OWNER/eks-platform-reference" +# Copy sub_claim_prefix from GitHub's OIDC API, removing only the leading repo:. +github_repository = "REPLACE-OWNER@OWNER-ID/eks-platform-reference@REPOSITORY-ID" environments = { dev = { cluster_name = "eks-platform-reference-dev", deploy_role_name = "github-eks-platform-dev-apply", plan_role_name = "github-eks-platform-dev-plan" } prod = { cluster_name = "eks-platform-reference-prod", deploy_role_name = "github-eks-platform-prod-apply", plan_role_name = "github-eks-platform-prod-plan" } diff --git a/bootstrap/tests/pipeline.tftest.hcl b/bootstrap/tests/pipeline.tftest.hcl index f958fc2..58c5816 100644 --- a/bootstrap/tests/pipeline.tftest.hcl +++ b/bootstrap/tests/pipeline.tftest.hcl @@ -17,7 +17,7 @@ override_data { } variables { environment = "dev" - github_repository = "offline/eks-platform-reference" + github_repository = "offline@123456/eks-platform-reference@789012" github_environment = "dev-plan" oidc_provider_arn = "arn:aws:iam::111122223333:oidc-provider/token.actions.githubusercontent.com" role_name = "offline-dev-plan" @@ -39,7 +39,7 @@ run "plan_role_trust" { jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Condition == { StringEquals = { "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" - "token.actions.githubusercontent.com:sub" = "repo:offline/eks-platform-reference:environment:dev-plan" + "token.actions.githubusercontent.com:sub" = "repo:offline@123456/eks-platform-reference@789012:environment:dev-plan" } } ) @@ -63,7 +63,7 @@ run "apply_role_trust" { jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Condition == { StringEquals = { "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" - "token.actions.githubusercontent.com:sub" = "repo:offline/eks-platform-reference:environment:dev" + "token.actions.githubusercontent.com:sub" = "repo:offline@123456/eks-platform-reference@789012:environment:dev" } } ) diff --git a/bootstrap/variables.tf b/bootstrap/variables.tf index 6b70a4f..166141f 100644 --- a/bootstrap/variables.tf +++ b/bootstrap/variables.tf @@ -9,7 +9,7 @@ variable "state_bucket_name" { } variable "github_repository" { - description = "GitHub owner/repository that will run the workflows." + description = "Repository portion of GitHub's OIDC sub_claim_prefix, without repo:. Include immutable owner/repository IDs when GitHub reports them." type = string } diff --git a/docs/verification.md b/docs/verification.md index c40cde0..953389a 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -76,7 +76,9 @@ The complete source received correctness, security, test, maintainability, reliability and adversarial review before the first PR. Its two findings were fixed: deployment refuses destructive cluster plans until monitoring cleanup, and offline tests inspect the actual IAM assume-role policies for both plan -and apply roles. A scratch-copy mutation check confirmed that broadening the +and apply roles. The policy tests use immutable GitHub owner and repository IDs; +the bootstrap wiring test also covers the older name-only subject format. +A scratch-copy mutation check confirmed that broadening the OIDC subject to a wildcard fails the policy assertion. Workflow syntax passed actionlint 1.7.12. These checks do not verify live AWS permissions or controllers. diff --git a/modules/pipeline/variables.tf b/modules/pipeline/variables.tf index 984c8a5..77c335b 100644 --- a/modules/pipeline/variables.tf +++ b/modules/pipeline/variables.tf @@ -10,7 +10,7 @@ variable "tags" { } variable "github_repository" { - description = "GitHub owner/repository allowed to obtain credentials." + description = "Repository portion of GitHub's OIDC sub_claim_prefix, without repo:. Include immutable owner/repository IDs when GitHub reports them." type = string }