diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..86f9bc5 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,88 @@ +name: Terraform +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + inputs: + environment: + description: Environment to run + required: true + type: choice + default: dev + options: [dev, prod] + operation: + description: Plan only, or plan and apply + required: true + type: choice + default: plan + options: [plan, apply] +permissions: + contents: read +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.16.5 + terraform_wrapper: false + - run: bash scripts/check.sh + - run: bash scripts/test.sh + deploy: + name: Terraform (${{ matrix.target }}) + needs: validate + # Fork code never receives an OIDC session or access to state. Its validate job still runs. + if: >- + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'workflow_dispatch' && (inputs.operation == 'plan' || github.ref == 'refs/heads/main')) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: ${{ github.event_name == 'workflow_dispatch' && fromJSON(format('["{0}"]', inputs.environment)) || fromJSON('["dev", "prod"]') }} + environment: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.operation == 'plan')) && format('{0}-plan', matrix.target) || matrix.target }} + permissions: + contents: read + id-token: write + pull-requests: write + concurrency: + group: terraform-${{ matrix.target }} + cancel-in-progress: false + env: + TF_IN_AUTOMATION: 'true' + TF_INPUT: 'false' + TARGET: ${{ matrix.target }} + EVENT: ${{ github.event_name }} + OPERATION: ${{ github.event_name == 'pull_request' && 'plan' || inputs.operation }} + AWS_REGION: ${{ vars.AWS_REGION }} + STATE_BUCKET: ${{ vars.TF_STATE_BUCKET }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.16.5 + terraform_wrapper: false + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-session-name: terraform-${{ github.run_id }} + role-duration-seconds: 3600 + allowed-account-ids: ${{ vars.AWS_ACCOUNT_ID }} + - name: Plan or apply platform, then monitoring + shell: bash + env: + PLAN_REPORT: ${{ runner.temp }}/terraform-plan-${{ matrix.target }}.md + run: bash scripts/deploy.sh + - name: Update PR plan comment + if: always() && github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PLAN_JOB_STATUS: ${{ job.status }} + PLAN_REPORT: ${{ runner.temp }}/terraform-plan-${{ matrix.target }}.md + run: bash scripts/comment-plan.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fdf3ba9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +**/.terraform/* +*.tfstate +*.tfstate.* +*.tfplan +crash.log +crash.*.log +**/backend.hcl +**/backend.bootstrap.hcl +bootstrap/terraform.tfvars +*.auto.tfvars +*.auto.tfvars.json +*.tfvars.local +*.tfvars.local.json +.terraformrc +terraform.rc +bootstrap/backend.tf +audit/ +__pycache__/ +*.pyc diff --git a/.terraform-version b/.terraform-version new file mode 100644 index 0000000..0d92a10 --- /dev/null +++ b/.terraform-version @@ -0,0 +1 @@ +1.16.5 diff --git a/README.md b/README.md new file mode 100644 index 0000000..499e1a7 --- /dev/null +++ b/README.md @@ -0,0 +1,500 @@ +Terraform for an AWS EKS platform with private worker nodes, Prometheus, public HTTPS Grafana, and GitHub OIDC deployments. + +## What gets built + +```text +GitHub Actions -- OIDC --> plan / apply IAM roles + | + S3 state + lock files + | + Terraform platform root + | + VPC across availability zones + | | + public subnets private subnets + NAT gateways EKS managed nodes + Grafana ALB -- HTTP ----> Grafana + Prometheus + ^ | + | gp3 EBS + HTTPS / ACM certificate + | + Route 53: grafana..hudson.tf + +Terraform monitoring root --> Helm / Kubernetes API +EKS control-plane logs --> CloudWatch +CNI / EBS / ALB controller --> individual IRSA roles +``` + +Dev uses two zones, two small nodes, and one NAT gateway. Prod uses three +zones, three larger nodes, and a NAT gateway in each zone. Neither cluster +places worker nodes on the public internet. The public ALB forwards only +Grafana traffic; Prometheus stays inside the cluster. + +## Repository layout + +```text +bootstrap/ state bucket and GitHub OIDC identities +modules/network/ VPC, subnets, routing and NAT +modules/cluster/ EKS, managed nodes, add-ons and IRSA +modules/observability/ monitoring, ALB controller, TLS and DNS +modules/pipeline/ scoped state access and GitHub IAM roles +environments/ shared platform root + terraform.tfvars platform settings keyed by dev/prod workspace + observability/ shared monitoring root + terraform.tfvars monitoring settings keyed by dev/prod workspace +.github/workflows/terraform.yml validation, PR plans and manual applies +scripts/ local checks and workflow deployment +``` + +Both roots use Terraform CLI workspaces named `dev` and `prod`. All environment +differences live in each root's `terraform.tfvars`, which Terraform loads +automatically. Each file contains an `environments` map; the root selects +`environments[terraform.workspace]`. Changing workspace selects both state and +settings, so plan and destroy need no `-var-file`. Variable validation rejects +`default` and any workspace without configured settings. + +The states are separate because Helm needs an existing Kubernetes API, valid +credentials, and ready nodes. A fresh deployment applies the platform before +monitoring. Destruction runs in the opposite order. No targeted apply is needed. + +## Before you run it + +You need Terraform 1.16.5, AWS CLI v2, kubectl compatible with the configured +Kubernetes version, and an AWS account with room for the nodes, NAT gateways, +and load balancers. Bash, jq, and Python 3 are used by the scripts; the Python code uses only the +standard library. Helm is only +needed if you want to inspect charts yourself; Terraform installs the releases. + +Use an AWS profile or SSO session with permission to bootstrap IAM and S3 and +to provision the platform. Configure a persistent IAM administrator role in +`administrator_arns` before applying; use its IAM ARN, not an STS session ARN. +The cluster does not grant implicit administrator access to its creator. + +The public `hudson.tf` hosted zone must already exist in the same account, with +its nameservers delegated correctly. Set its exact `hosted_zone_id` in bootstrap +and both monitoring tfvars files. +Terraform selects that ID and checks the zone name and public-zone setting +before adding Grafana alias records and ACM validation records. It does not create or replace +the zone or register the domain. Use unused Grafana hostnames; existing records +need to be imported or moved deliberately before this configuration owns them. + +Review both sets of tfvars. The shipped values use one AWS region; if you change +it, update the AZs, backend region, bootstrap inputs, and GitHub variables as +well. Verify that the configured Kubernetes version is available and in +[standard EKS support](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html). +The control plane uses the STANDARD upgrade policy, so AWS can upgrade it at +the end of standard support; schedule explicit upgrades before that happens. + +## Run dev first + +Start each shell block below at the repository root. The commands enter the +relevant Terraform folder and return to the root when finished. They provision +billable AWS resources. Read the [cost breakdown](#cost) before leaving a +cluster running. + +### Bootstrap state and CI roles + +Authenticate your AWS profile first using your configured login method +(`aws login` or `aws sso login`), then select it for this shell: + +```bash +export AWS_PROFILE=YOUR_PROFILE +aws sts get-caller-identity +cp bootstrap/terraform.tfvars.example bootstrap/terraform.tfvars +``` + +Read the repository value GitHub actually uses in its OIDC subject: + +```bash +gh api repos/OWNER/REPOSITORY/actions/oidc/customization/sub \ + --jq '.sub_claim_prefix | sub("^repo:"; "")' +``` + +Set `github_repository` in the local tfvars file to that exact result. New +repositories use `OWNER@OWNER-ID/REPOSITORY@REPOSITORY-ID`; older repositories +may use `OWNER/REPOSITORY`. Do not infer the value from the repository URL. +The module adds `repo:` and the exact environment to build the trust condition. +An incorrect value causes `Not authorized to perform sts:AssumeRoleWithWebIdentity`. +After correcting it, plan and apply bootstrap again to update the role trusts, +then retry the workflow. This does not require reapplying the EKS platform. +See [GitHub's subject formats](https://docs.github.com/en/actions/reference/security/oidc#immutable-subject-claims). + +Choose a globally unique state bucket and confirm the cluster and role names +match the environment inputs. Confirm that `hosted_zone_id` identifies the zone whose +name servers are delegated for your domain; duplicate hosted-zone names are +allowed in AWS. If this account already has GitHub's OIDC provider, set +`github_oidc_provider_arn` to its ARN. There is only one provider per issuer per +account; this configuration can reuse it. + +```bash +cd bootstrap +terraform init +terraform plan -out=bootstrap.tfplan +terraform apply bootstrap.tfplan +terraform output +cd .. +``` + +Bootstrap starts with local state because the bucket does not exist yet. +Move that state into the bucket once it has been created: + +```bash +cp bootstrap/backend.tf.example bootstrap/backend.tf +cp bootstrap/backend.hcl.example bootstrap/backend.hcl +# Edit bootstrap/backend.hcl with the bucket output and region. +cd bootstrap +terraform init -migrate-state -backend-config=backend.hcl +cd .. +``` + +Confirm the migration prompt. Secure any local backup state left by the +migration. Bootstrap state lives at `bootstrap/terraform.tfstate`; environment +roles cannot read or modify it. Bucket deletion is blocked with `prevent_destroy`. +S3 versioning, encryption, TLS-only access, and public-access blocking are enabled. + +### Initialize the two roots + +```bash +cp environments/backend.hcl.example environments/backend.hcl +cp environments/observability/backend.hcl.example environments/observability/backend.hcl +# Set the same bucket and region in both files. Keep their distinct keys. +cd environments +terraform init -backend-config=backend.hcl +terraform workspace select -or-create dev +terraform workspace select -or-create prod +terraform workspace select dev +cd observability +terraform init -backend-config=backend.hcl +terraform workspace select -or-create dev +terraform workspace select -or-create prod +terraform workspace select dev +cd ../.. +``` + +Create both workspaces using your bootstrap credentials before enabling PR +plans. A read-only plan role cannot initialize an empty workspace state. The +backend uses native S3 locking with `use_lockfile = true`, without DynamoDB. +The [S3 backend documentation](https://developer.hashicorp.com/terraform/language/backend/s3) +explains lock permissions and workspace paths. + +| Workspace | Platform state | Monitoring state | +| --- | --- | --- | +| dev | `environments/dev/platform.tfstate` | `environments/dev/observability.tfstate` | +| prod | `environments/prod/platform.tfstate` | `environments/prod/observability.tfstate` | + +### Plan and apply dev + +In `environments/terraform.tfvars`, set `administrator_arns` inside the +`environments.dev` object to your human administrator's persistent IAM ARN: + +```hcl +administrator_arns = { + operator = "REPLACE_WITH_YOUR_PERSISTENT_IAM_ROLE_ARN" +} +``` + +Then apply using that role's AWS session: + +```bash +cd environments +terraform workspace select dev +terraform plan -out=dev.tfplan +terraform apply dev.tfplan +cd observability +terraform workspace select dev +terraform plan -out=dev.tfplan +terraform apply dev.tfplan +cd ../.. +``` + +The second apply waits for Helm, certificate validation, and ALB reconciliation. +An access-denied error is a reason to check your IAM principal and its EKS access +entry. Do not rerun monitoring with an unrelated administrator identity. + +### Open Grafana and check the cluster + +```bash +cd environments +cluster_region=$(terraform output -raw region) +cluster_name=$(terraform output -raw cluster_name) +aws eks update-kubeconfig --region "$cluster_region" --name "$cluster_name" +kubectl get nodes +kubectl -n monitoring get pods,pvc,ingress +cd observability +terraform output -raw grafana_url +kubectl -n monitoring get secret monitoring-grafana \ + -o jsonpath='{.data.admin-password}' | base64 --decode +cd ../.. +``` + +Log in as `admin` with that generated password. The dev URL is +`https://grafana.dev.hudson.tf`; prod uses `https://grafana.prod.hudson.tf`. +HTTP redirects to HTTPS. Anonymous access and account sign-up are disabled. +Keep the password private and change it after first login. Prometheus provides +node, pod, and Kubernetes object metrics; Grafana ships the chart's dashboards. + +The API endpoint is public with IAM authentication, and private access is also +enabled. The sample API CIDRs allow GitHub-hosted runners with changing egress +addresses. For a fixed-egress runner, restrict `public_access_cidrs` in platform +tfvars. `ingress_cidrs` separately controls who can reach Grafana's ALB. +For a long-lived shared Grafana deployment, add SSO before giving more people access. + +### Run prod + +Select `prod` in both roots and configure `environments.prod.administrator_arns` +in the platform tfvars. Use the same plan and apply commands; the selected +workspace supplies the prod settings. Name the saved plans `prod.tfplan`. +Do not copy Terraform code or change modules to deploy prod. + +## GitHub Actions + +Create GitHub environments `dev-plan`, `prod-plan`, `dev`, and `prod`. Each needs +these environment variables; they are configuration, not stored AWS secrets: + +| Variable | Value | +| --- | --- | +| `AWS_ROLE_ARN` | Corresponding role from bootstrap's `github_role_arns` output | +| `AWS_ACCOUNT_ID` | Account allowed by the AWS credentials action | +| `AWS_REGION` | Region matching tfvars and the state bucket | +| `TF_STATE_BUCKET` | Bootstrap state bucket output | + +Restrict deployment branches on `dev` and `prod` to `main`. Require reviewers +for `prod`. Restrict who can approve same-repository PRs against `dev-plan` and +`prod-plan`: plan roles can read state and Kubernetes Secrets to refresh Helm +releases. The plan environments must allow PR refs, not just `main`. Protect +`main`, require the validate job, and require review for changes to workflows, +Terraform, and IAM policies. Environment branch restrictions are essential: +the OIDC environment subject does not itself prove which branch requested it. + +Every PR runs formatting, validation, and offline plan tests. Same-repository +PRs also plan both environments using the read-only roles. Fork PRs run the +credential-free checks only. No `pull_request_target` execution is used. + +For an environment without a cluster yet, the platform plan runs and the live +monitoring plan is explicitly deferred. Static validation and offline monitoring +plans still run. Once a cluster exists, both live plans run. PR plans are unlocked +because their roles cannot write lock files; they can be stale. Manual applies +create new saved plans under S3 locks and apply those exact files in the same job. +PR plan artifacts are never reused for deployment or published as artifacts. + +Each PR plan updates one bot comment per environment with platform and monitoring +resource counts: add, change, destroy and replace. The comment includes the +commit, job result and workflow link. Failed plans and deferred monitoring are +marked explicitly. Comments contain counts only; raw plan values and JSON are +not posted. Retries update the existing comment for that environment. + +A merge or push to `main` runs validation only. It does not request AWS +credentials or run a live plan or apply. This lets the demo be merged and +maintained without starting billable infrastructure. +Manual runs select one environment and either `plan` or `apply`; they default +to a dev plan. Manual applies require `main`. Deployment jobs serialize per environment and +are never canceled mid-apply. The prod environment approval sits before AWS +credentials are requested. This is not a sequential dev-to-prod promotion gate. + +After the workflow is merged into `main`, use Actions > Terraform > Run workflow +to select the environment and operation, or run: + +```bash +gh workflow run terraform.yml --ref main -f environment=dev -f operation=plan +gh workflow run terraform.yml --ref main -f environment=prod -f operation=apply +``` + +Manual plans use the corresponding `dev-plan` or `prod-plan` environment and +read-only role. Manual applies use `dev` or `prod`, including its approval rules. +To retry a failed PR job before merging, use GitHub's Re-run failed jobs button +or `gh run rerun RUN_ID --failed`. To retry just one environment job, find its +`databaseId` with `gh run view RUN_ID --json jobs --jq '.jobs[] | {name, databaseId}'`, +then run `gh run rerun RUN_ID --job JOB_ID` using that ID. +Plans do not take a state lock, so an active local apply does not block them; +wait until the apply finishes for a reliable result. Applies wait up to five +minutes for the S3 lock. Retry after the other apply finishes if that wait expires. + +CI refuses plans that delete or replace the EKS cluster. Those changes need +monitoring-first cleanup and an AWS audit while its controllers still run; +follow [the cleanup procedure](docs/cleanup.md) before changing the cluster's +identity or other settings that require replacement. Initial creation and +in-place updates run normally. + +The apply roles can manage EC2 and EKS resources in the configured region, +cluster-prefixed IAM roles, and records in the chosen hosted zone. Direct IAM grants target cluster-prefixed roles, and state-object deletion is +not granted; lock files can be deleted. This is not a permissions boundary: +creating roles and attaching policies can allow privilege escalation beyond +those direct grants, including access to bootstrap identities. Treat apply +workflows and their approvers as fully trusted within this AWS account. These +are privileged infrastructure roles, not least-privilege application roles. Workspaces separate state, not accounts or failure domains. At larger +scale, use separate accounts and backends and narrow IAM permissions further. + +## Design decisions + +**Managed node groups.** EKS manages node registration, replacement, and rolling +updates. That removes work that would otherwise live in custom launch scripts +and ASG lifecycle hooks. On-demand capacity keeps interruption handling out of +this baseline. Dev uses burstable nodes; sustained CPU loads need different +instance types or incur CPU credit charges. Node counts are explicit: increasing +`max` does not install an autoscaler or automatically grow the cluster. + +**OIDC rather than stored AWS keys.** GitHub gets a short-lived session tied to +this repository and an exact GitHub environment. There is no IAM user access key +to rotate or leak in repository secrets. Plan and apply roles are separate, and +IRSA gives the CNI, EBS CSI driver, and ALB controller separate AWS identities. +IMDSv2 is required, with a hop limit that keeps normal pods from relying on node +credentials. The ALB controller receives its region and VPC explicitly. + +**Public and private subnets.** Nodes need outbound access for images and AWS +APIs but do not need public IPs. NAT provides that path. Dev shares one NAT to +save money, accepting an availability dependency and cross-zone traffic charges. +Prod routes each private subnet to its local NAT. Public subnets also host the +Grafana ALB. The controller is the standard AWS controller, not a custom operator. + +**S3 locking and workspaces.** S3 now supports Terraform lock files directly. +This avoids a separate lock database. Workspaces give the same code distinct +states; auto-loaded tfvars maps supply settings for the selected workspace. +Platform and monitoring are +separate roots because their provider dependencies and teardown order differ. +They look up the existing cluster through AWS, rather than granting monitoring +access to the entire platform state. + +**Monitoring storage and ingress.** Prometheus and Grafana use encrypted gp3 +volumes with delayed binding to a node's zone. Retention is bounded by time and +size. Neither deployment is replicated; a node failure can interrupt monitoring +while a replacement attaches the zonal volume. Grafana has a public TLS endpoint +because this reference includes internet access. The ALB terminates TLS and +forwards HTTP inside the VPC. Prometheus is not exposed, and alert routing is +not configured. Managed control-plane targets that EKS does not expose are +disabled in the chart instead of leaving permanently failing scrape targets. + +**What changes at larger scale.** Separate AWS accounts for prod, private API +access from a VPC runner, SSO for Grafana, a permissions boundary on deployment +roles, and stronger DNS ownership controls come first. Add autoscaling when the +workload requires it. Move metrics to durable remote storage when retention or +availability requirements exceed a single Prometheus volume. Add alert receivers, +backup and restore tests, application ingress policies, and budget alerts with +an actual owner. Measure traffic before adding paid VPC endpoints; an S3 gateway +endpoint is a useful first step to reduce NAT traffic. + +Provider versions and Helm charts are pinned. EKS add-ons use AWS's compatible +default for the configured minor version. That default can change; inspect its +version in every plan. Chart upgrades need their upstream upgrade notes, +particularly Prometheus CRD changes. The checked-in controller IAM policy is +from [AWS Load Balancer Controller v2.14.1](https://github.com/kubernetes-sigs/aws-load-balancer-controller/blob/v2.14.1/docs/install/iam_policy.json) +and must be reviewed alongside the controller chart pin. + +## Cost + +Estimate checked on October 14, 2026, for Linux on-demand instances in the sample +region, 730 hours per month, and the desired node counts. No free-tier credits, +Savings Plans, taxes, or support plan are included. Monitoring runs on the listed +nodes; there are no extra EC2 monitoring servers. + +| Component | Dev / month | Prod / month | Basis | +| --- | ---: | ---: | --- | +| EKS control plane | $73.00 | $73.00 | One cluster at $0.10/hour | +| Worker nodes | $60.74 | $210.24 | Two at $0.0416/hour; three at $0.096/hour | +| Node root disks | $4.80 | $7.20 | 30 GiB gp3 per node at $0.08/GiB-month | +| Monitoring disks | $1.20 | $2.80 | 15 / 35 GiB total gp3 | +| NAT gateways | $32.85 | $98.55 | One / three at $0.045/hour | +| NAT public IPv4 | $3.65 | $10.95 | One / three at $0.005/IP-hour | +| Grafana ALB | $16.43 | $16.43 | One at $0.0225/hour | +| ALB public IPv4 | $7.30 | $10.95 | At least one address per enabled zone: two / three | +| ALB capacity allowance | $5.84 | $5.84 | Budget assumption: average one LCU at $0.008/LCU-hour | +| ACM certificate | $0 | $0 | Non-exportable public certificate used by ALB | +| Route 53 alias record | $0 additional | $0 additional | Existing zone; alias queries to ELB are free | +| **Estimated subtotal** | **$205.80** | **$435.96** | Before usage costs below | + +The existing `hudson.tf` hosted zone costs $0.50/month once, not per environment; +it is already being paid for. Domain registration and renewal are separate. +State storage and S3 requests are usually well under $1/month for this project. +CloudWatch control-plane log ingestion and storage depend on traffic; allow a +few dollars initially and measure it. A reasonable light-use budget is roughly +**$210–225/month for dev** and **$440–465/month for prod**, or **$650–690/month +for both**. These are budget estimates, not limits or a bill guarantee. + +For a short verification run, use these subtotal estimates: + +| Runtime | Dev | Prod | Both | +| --- | ---: | ---: | ---: | +| 3 hours | $0.85 | $1.79 | $2.64 | +| 6 hours | $1.69 | $3.58 | $5.27 | +| 24 hours | $6.77 | $14.33 | $21.10 | + +Allow **$2–3 for a short dev test** or **$8–10 for a day of dev**, including some +startup downloads and logs. Those allowances assume light traffic and successful +cleanup; they are not caps. Timing starts when AWS resources are provisioned, +including time spent waiting for a deployment or teardown. + +Public Grafana adds about **$29.57/month in dev** or **$33.22/month in prod**, +including the one-LCU allowance and minimum public IPs. Capacity and IP counts +can grow under load. Without the LCU allowance, the fixed ingress floor is about +$23.73 / $27.38 per month. A 24-hour dev run is about **$6.77** at the subtotal +rate; prod is about **$14.33**, before variable charges. There is no separate +one-time EKS provisioning fee. Deployment, idle time, and teardown all count. + +Add $0.045 for each GB processed by NAT, plus applicable internet and cross-zone +data transfer. Image pulls during first startup use NAT too. Dev's shared NAT +makes cross-zone charges more likely. Burstable CPU credit use can add EC2 +charges. Extra nodes cost their instance rate and root disk storage. Retained +PVCs, snapshots, or ALBs left after a failed teardown continue to cost money. +If a cluster enters extended Kubernetes support, EKS's price rises to +$0.60/hour, adding $365/month; this configuration opts into STANDARD support. + +Sources: [EKS](https://aws.amazon.com/eks/pricing/), +[EC2 Linux regional rate data](https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/ec2/USD/current/ec2-ondemand-without-sec-sel/US%20East%20(N.%20Virginia)/Linux/index.json), +[EBS](https://aws.amazon.com/ebs/pricing/), +[NAT and public IPv4](https://aws.amazon.com/vpc/pricing/), +[ALB](https://aws.amazon.com/elasticloadbalancing/pricing/), +[ACM](https://aws.amazon.com/certificate-manager/pricing/), and +[Route 53](https://aws.amazon.com/route53/pricing/). + +## Checks + +```bash +bash scripts/check.sh +bash scripts/test.sh +``` + +Checks install the locked providers, format-check the repo, validate every root, +and run offline Terraform plans with AWS lookups replaced by fixtures. They do +not contact AWS or create resources. Helm chart rendering is a separate local +check described in [verification](docs/verification.md). Live permissions, +quotas, DNS delegation, cluster readiness, and a second no-change AWS plan must +be checked during the first real deployment. Static validation cannot prove those. + +## Tear it down + +Use the [full cleanup and API audit runbook](docs/cleanup.md) to capture an +inventory before teardown and verify that billable resources are gone afterward. +The brief commands below are only the normal destruction sequence. + +Select the same workspace in both roots. Save anything you want to keep from +Grafana and Prometheus, then destroy monitoring before the platform: + +```bash +cd environments/observability +terraform workspace select dev +terraform destroy +cd .. +terraform workspace select dev +terraform destroy +cd .. +``` + +The ingress is deleted before its controller. Namespace deletion removes any +Prometheus PVCs retained by Helm; the storage class uses a Delete reclaim policy. +Before removing the cluster, confirm that the ALB and monitoring EBS volumes +are gone. A failed controller cleanup can leave AWS resources behind. If that +happens, fix it while the cluster and controller still exist. Do not delete the +cluster first and assume the finalizers will finish later. + +The state bucket, bootstrap roles, and existing hosted zone remain. Keep state +versions until you no longer need recovery. Removing bootstrap protection is an +explicit cleanup task, not part of destroying dev or prod. + +## What this is not + +This is a small platform reference, not a production service or a claim that +workspaces isolate tenants. It has no application deployment, service mesh, +GitOps controller, custom operators, autoscaler, remote metrics storage, +monitoring high availability, disaster recovery, or staffed alert response. +The prod inputs improve node and NAT capacity; they do not create those missing +operating practices. Grafana's password login is a starting point, not a complete +identity system for a team. diff --git a/bootstrap/.terraform.lock.hcl b/bootstrap/.terraform.lock.hcl new file mode 100644 index 0000000..3622c58 --- /dev/null +++ b/bootstrap/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.67.0" + constraints = "~> 6.67.0" + hashes = [ + "h1:EB9ixYOZrSlYD7wtJxf88qwoyyWrlKDKxhzaCLIb3t4=", + "zh:111d5686a1f4ccbc888bb5e2229308bcdd9149898c6af97969fcdfb0e7bd2aa0", + "zh:21b3b7693bd9754c039fd546f03ed09e7510c6189aa5ee37176f144cf8dd5f95", + "zh:25e03c7f025ff4851889537605aa560d2e39bd738b33a5204b8037eb164b475f", + "zh:2817a046f1060e25bf04b0eb78f4e251130bb92dc82ec1264ce156dc9bf78e67", + "zh:2d318d674c3ec9dbd97ddb10b12ad4827be4f508c43ba2ae1e0523baa9e367e3", + "zh:2f07ab356718267299e10b6072472ded29d82753883027bf43bcd687ac72feb2", + "zh:32307e67f83bc0dc94d38cfcd23d782166a09e0e975e2a5aa7d7a3e7ca5d3da8", + "zh:4ce94853264097dd7f4542b3cfd18d2b98b06d23321db45d5e384ea275a57f63", + "zh:869656c41cc7c7412f213e488f98167cff61deafcb8cf245d25d056e8dfdc263", + "zh:8fd8c814e9d8edf152552047db23bf5b5d63f22e6b0b5d47b2af851376e99349", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9cbc02ceef9bd469da497a1e7065ff984bdacfbe919ac3cce804a86c13b6e2c6", + "zh:9ea55dda2767acfc1f6337fc7d29b1d4d79d6f8f04871f8ad99a6078d2865994", + "zh:d7149df0819fb57a160489db45d33951765b8f0118daa3b4cd549a0135bc97a1", + "zh:e5819937bb7043d08c9829b32d52d9fb55a5b9a4d8d55d550d47a3d7392db546", + "zh:f93bc38dbc0ad53842303f30eec7a22c525c4d56209b54ec33a8d375cfc4e4fd", + ] +} diff --git a/bootstrap/backend.hcl.example b/bootstrap/backend.hcl.example new file mode 100644 index 0000000..123c8b8 --- /dev/null +++ b/bootstrap/backend.hcl.example @@ -0,0 +1,5 @@ +bucket = "REPLACE-WITH-BOOTSTRAP-BUCKET" +key = "bootstrap/terraform.tfstate" +region = "us-east-1" +encrypt = true +use_lockfile = true diff --git a/bootstrap/backend.tf.example b/bootstrap/backend.tf.example new file mode 100644 index 0000000..12c0dbe --- /dev/null +++ b/bootstrap/backend.tf.example @@ -0,0 +1,3 @@ +terraform { + backend "s3" {} +} diff --git a/bootstrap/main.tf b/bootstrap/main.tf new file mode 100644 index 0000000..a872673 --- /dev/null +++ b/bootstrap/main.tf @@ -0,0 +1,84 @@ +provider "aws" { + region = var.region + default_tags { tags = { environment = "bootstrap", "managed-by" = "terraform", project = "eks-platform-reference" } } +} +resource "aws_s3_bucket" "state" { + bucket = var.state_bucket_name + lifecycle { prevent_destroy = true } +} +resource "aws_s3_bucket_versioning" "state" { + bucket = aws_s3_bucket.state.id + versioning_configuration { status = "Enabled" } +} +resource "aws_s3_bucket_server_side_encryption_configuration" "state" { + bucket = aws_s3_bucket.state.id + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} +resource "aws_s3_bucket_public_access_block" "state" { + bucket = aws_s3_bucket.state.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} +data "aws_iam_policy_document" "state" { + statement { + effect = "Deny" + actions = ["s3:*"] + resources = [aws_s3_bucket.state.arn, "${aws_s3_bucket.state.arn}/*"] + principals { + type = "*" + identifiers = ["*"] + } + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} +resource "aws_s3_bucket_policy" "state" { + bucket = aws_s3_bucket.state.id + policy = data.aws_iam_policy_document.state.json +} +resource "aws_iam_openid_connect_provider" "github" { + count = var.github_oidc_provider_arn == null ? 1 : 0 + url = "https://token.actions.githubusercontent.com" + client_id_list = ["sts.amazonaws.com"] +} +data "aws_route53_zone" "public" { + zone_id = var.hosted_zone_id + private_zone = false + lifecycle { + postcondition { + condition = !self.private_zone && lower(trimsuffix(self.name, ".")) == lower(trimsuffix(var.domain_name, ".")) + error_message = "The selected hosted zone must be public and match domain_name." + } + } +} +locals { + github_oidc_provider_arn = var.github_oidc_provider_arn != null ? var.github_oidc_provider_arn : aws_iam_openid_connect_provider.github[0].arn + roles = merge( + { for env, config in var.environments : "${env}-apply" => { environment = env, github_environment = env, role_name = config.deploy_role_name, cluster_name = config.cluster_name, read_only = false } }, + { for env, config in var.environments : "${env}-plan" => { environment = env, github_environment = "${env}-plan", role_name = config.plan_role_name, cluster_name = config.cluster_name, read_only = true } } + ) +} +module "pipeline" { + for_each = local.roles + source = "../modules/pipeline" + environment = each.value.environment + github_repository = var.github_repository + github_environment = each.value.github_environment + oidc_provider_arn = local.github_oidc_provider_arn + role_name = each.value.role_name + cluster_name = each.value.cluster_name + region = var.region + state_bucket_arn = aws_s3_bucket.state.arn + state_prefix = "environments/${each.value.environment}/" + hosted_zone_id = data.aws_route53_zone.public.zone_id + read_only = each.value.read_only +} diff --git a/bootstrap/outputs.tf b/bootstrap/outputs.tf new file mode 100644 index 0000000..e03526e --- /dev/null +++ b/bootstrap/outputs.tf @@ -0,0 +1,19 @@ +output "state_bucket_name" { + description = "S3 backend bucket." + value = aws_s3_bucket.state.id +} + +output "github_role_arns" { + description = "Apply and plan role ARNs by environment." + value = { for k, role in module.pipeline : k => role.role_arn } +} + +output "github_oidc_provider_arn" { + description = "Shared GitHub OIDC provider ARN." + value = local.github_oidc_provider_arn +} + +output "hosted_zone_id" { + description = "Public hosted zone selected for ingress and certificate validation." + value = data.aws_route53_zone.public.zone_id +} diff --git a/bootstrap/terraform.tfvars.example b/bootstrap/terraform.tfvars.example new file mode 100644 index 0000000..d471d7d --- /dev/null +++ b/bootstrap/terraform.tfvars.example @@ -0,0 +1,10 @@ +region = "us-east-1" +state_bucket_name = "REPLACE-WITH-A-GLOBALLY-UNIQUE-BUCKET" +# Copy sub_claim_prefix from GitHub's OIDC API, removing only the leading repo:. +github_repository = "REPLACE-OWNER@OWNER-ID/eks-platform-reference@REPOSITORY-ID" +environments = { + dev = { cluster_name = "eks-platform-reference-dev", deploy_role_name = "github-eks-platform-dev-apply", plan_role_name = "github-eks-platform-dev-plan" } + prod = { cluster_name = "eks-platform-reference-prod", deploy_role_name = "github-eks-platform-prod-apply", plan_role_name = "github-eks-platform-prod-plan" } +} +domain_name = "hudson.tf" +hosted_zone_id = "Z0992412PWY8TUM0P6ZU" diff --git a/bootstrap/tests/bootstrap.tftest.hcl b/bootstrap/tests/bootstrap.tftest.hcl new file mode 100644 index 0000000..62de681 --- /dev/null +++ b/bootstrap/tests/bootstrap.tftest.hcl @@ -0,0 +1,67 @@ +variable "region" { type = string } +provider "aws" { + region = var.region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +override_data { + target = data.aws_route53_zone.public + values = { zone_id = "ZOFFLINETEST", name = "offline.invalid", private_zone = false } +} +override_data { + target = module.pipeline["dev-plan"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["dev-plan"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.pipeline["dev-apply"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["dev-apply"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.pipeline["prod-plan"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["prod-plan"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.pipeline["prod-apply"].data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = module.pipeline["prod-apply"].data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +run "bootstrap_plan" { + command = plan + variables { + state_bucket_name = "offline-eks-reference-state" + github_repository = "offline/eks-platform-reference" + github_oidc_provider_arn = "arn:aws:iam::111122223333:oidc-provider/token.actions.githubusercontent.com" + domain_name = "offline.invalid" + hosted_zone_id = "ZOFFLINETEST" + environments = { + dev = { cluster_name = "offline-dev", deploy_role_name = "offline-dev-apply", plan_role_name = "offline-dev-plan" } + prod = { cluster_name = "offline-prod", deploy_role_name = "offline-prod-apply", plan_role_name = "offline-prod-plan" } + } + } + assert { + condition = aws_s3_bucket_versioning.state.versioning_configuration[0].status == "Enabled" && aws_s3_bucket_public_access_block.state.block_public_policy && aws_s3_bucket_public_access_block.state.block_public_acls + error_message = "State must have versioning and public access blocking." + } + assert { + condition = alltrue([for name, role in module.pipeline : role.oidc_subject == "repo:offline/eks-platform-reference:environment:${replace(name, "-apply", "")}"]) + error_message = "Bootstrap must wire each CI role to its matching GitHub environment." + } +} diff --git a/bootstrap/tests/fixtures.tfvars b/bootstrap/tests/fixtures.tfvars new file mode 100644 index 0000000..0d253dd --- /dev/null +++ b/bootstrap/tests/fixtures.tfvars @@ -0,0 +1 @@ +region = "us-east-1" diff --git a/bootstrap/tests/pipeline.tftest.hcl b/bootstrap/tests/pipeline.tftest.hcl new file mode 100644 index 0000000..58c5816 --- /dev/null +++ b/bootstrap/tests/pipeline.tftest.hcl @@ -0,0 +1,72 @@ +variable "region" { type = string } +provider "aws" { + region = var.region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +override_data { + target = data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +variables { + environment = "dev" + github_repository = "offline@123456/eks-platform-reference@789012" + github_environment = "dev-plan" + oidc_provider_arn = "arn:aws:iam::111122223333:oidc-provider/token.actions.githubusercontent.com" + role_name = "offline-dev-plan" + cluster_name = "offline-dev" + state_bucket_arn = "arn:aws:s3:::offline-eks-reference-state" + state_prefix = "environments/dev/" + read_only = true + hosted_zone_id = "ZOFFLINETEST" +} +run "plan_role_trust" { + command = plan + module { source = "../modules/pipeline" } + assert { + condition = ( + length(jsondecode(aws_iam_role.this.assume_role_policy).Statement) == 1 && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Effect == "Allow" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Action == "sts:AssumeRoleWithWebIdentity" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Principal.Federated == var.oidc_provider_arn && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Condition == { + StringEquals = { + "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" + "token.actions.githubusercontent.com:sub" = "repo:offline@123456/eks-platform-reference@789012:environment:dev-plan" + } + } + ) + error_message = "The actual plan role must trust only the exact GitHub environment, audience and provider." + } +} +run "apply_role_trust" { + command = plan + module { source = "../modules/pipeline" } + variables { + github_environment = "dev" + role_name = "offline-dev-apply" + read_only = false + } + assert { + condition = ( + length(jsondecode(aws_iam_role.this.assume_role_policy).Statement) == 1 && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Effect == "Allow" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Action == "sts:AssumeRoleWithWebIdentity" && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Principal.Federated == var.oidc_provider_arn && + jsondecode(aws_iam_role.this.assume_role_policy).Statement[0].Condition == { + StringEquals = { + "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" + "token.actions.githubusercontent.com:sub" = "repo:offline@123456/eks-platform-reference@789012:environment:dev" + } + } + ) + error_message = "The actual apply role must trust only the exact GitHub environment, audience and provider." + } +} diff --git a/bootstrap/variables.tf b/bootstrap/variables.tf new file mode 100644 index 0000000..166141f --- /dev/null +++ b/bootstrap/variables.tf @@ -0,0 +1,35 @@ +variable "region" { + description = "AWS region for state and locking." + type = string +} + +variable "state_bucket_name" { + description = "Globally unique S3 state bucket name." + type = string +} + +variable "github_repository" { + description = "Repository portion of GitHub's OIDC sub_claim_prefix, without repo:. Include immutable owner/repository IDs when GitHub reports them." + type = string +} + +variable "github_oidc_provider_arn" { + description = "Existing account GitHub provider ARN; null creates it here." + type = string + default = null +} + +variable "environments" { + description = "Environment names and the cluster and CI role names for each." + type = map(object({ cluster_name = string, deploy_role_name = string, plan_role_name = string })) +} + +variable "domain_name" { + description = "Existing public Route 53 hosted zone name." + type = string +} + +variable "hosted_zone_id" { + description = "Exact existing public Route 53 zone ID; zone names are not unique." + type = string +} diff --git a/bootstrap/versions.tf b/bootstrap/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/bootstrap/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/docs/cleanup.md b/docs/cleanup.md new file mode 100644 index 0000000..a9fd532 --- /dev/null +++ b/docs/cleanup.md @@ -0,0 +1,279 @@ +# Cleanup and audit + +This procedure removes an environment and checks AWS for resources that still +exist. It also covers retiring the shared backend and CI identities when the +whole reference is finished. The existing `hudson.tf` hosted zone and domain +are outside this repo's ownership and must remain. + +Each shell block starts at the repository root, enters the relevant Terraform +folder, and returns to the root when finished. + +Use this sequence before deliberately replacing the cluster as well. Keep the +currently deployed cluster settings until monitoring is destroyed and audited; +changing the cluster name first makes Helm and Kubernetes look up a different +API. CI stops when a platform plan would delete or replace EKS. + +A successful `terraform destroy` is one piece of evidence. It does not prove +that controller-created load balancers, retained disks, or manually detached +resources are gone. Use the AWS audit after destroy. It is read-only; it never +deletes anything or treats an access-denied response as proof of deletion. + +## 1. Stop deployments and choose the scope + +Disable the Terraform workflow in GitHub before cleanup. Wait for any running +apply to finish; do not cancel an apply and then launch a destroy against it. +Decide whether you are removing dev, prod, or both. If keeping an environment, +keep bootstrap and its state bucket. Work from a fresh AWS session that can +read the relevant AWS services and destroy the environment. The audit also +needs IAM reads and permission to describe snapshots and load balancer tags. + +For dev, select the workspace in both roots and capture its identity: + +```bash +export AUDIT_ENV=dev +cd environments +terraform workspace select "$AUDIT_ENV" +export AUDIT_REGION=$(terraform output -raw region) +export AUDIT_CLUSTER=$(terraform output -raw cluster_name) +terraform workspace show +cd observability +terraform workspace select "$AUDIT_ENV" +export AUDIT_ZONE=$(terraform output -raw hosted_zone_id) +terraform workspace show +cd ../.. +aws sts get-caller-identity +``` + +Set `AUDIT_ENV=prod` when removing prod. Recheck +these values before each plan. The same region must be used for every regional +AWS audit call. IAM is account-wide. + +## 2. Save the inventory before deleting anything + +```bash +umask 077 +python3 scripts/audit.py snapshot \ + --environment "$AUDIT_ENV" --region "$AUDIT_REGION" \ + --cluster-name "$AUDIT_CLUSTER" --snapshot "audit/$AUDIT_ENV-before.json" +cd environments +terraform state list > "../audit/$AUDIT_ENV-platform-state.txt" +cd observability +terraform state list > "../../audit/$AUDIT_ENV-monitoring-state.txt" +terraform show -json > "../../audit/$AUDIT_ENV-monitoring-before.json" +kubectl -n monitoring get ingress,pvc -o yaml > "../../audit/$AUDIT_ENV-kubernetes.yaml" +cd ../.. +``` + +Refresh kubeconfig for this cluster first if needed. The snapshot records its +account, region, cluster name, VPC children, EC2 instances, NAT gateways, EIPs, +EBS volumes, launch templates, load balancers, target groups, control-plane log +groups, IAM roles, IRSA providers, and certificates. It includes tagged EBS +snapshots if any were made manually. These files contain infrastructure metadata +and should remain private; `audit/` is ignored by git. + +The script uses AWS CLI pagination. It finds VPC children even without ownership +tags and independently scans resources tagged `project=eks-platform-reference` +and the selected `environment`. Captured resource IDs are checked after teardown +even if their tags were removed. A resource created later with no ownership +tags and outside the captured VPC is outside this audit's detection boundary. +Keep deployments disabled so the inventory stays useful. + +## 3. Destroy monitoring while its controllers still run + +Back up any dashboards or metrics you intend to keep. The monitoring disks +will be deleted. Inspect the destruction plan before executing it: + +```bash +cd environments/observability +terraform workspace show +terraform plan -destroy -out=cleanup.tfplan +terraform apply cleanup.tfplan +terraform state list +cd ../.. +``` + +The state list should be empty. Ingress deletion must complete before the ALB +controller is removed. The controller's finalizer deletes the ALB, listeners, +target groups, and its security group. Helm teardown removes Grafana's PVC; +namespace deletion removes Prometheus PVCs retained by the chart. The gp3 +storage class has a Delete reclaim policy, so the EBS driver deletes their disks. +Terraform deletes the ACM certificate, its validation records, and the Grafana +alias record. It does not delete the public hosted zone. + +If destroy hangs on an ingress or PVC, keep the platform running. Check: + +```bash +kubectl -n monitoring get ingress,pvc +kubectl -n kube-system logs deployment/aws-load-balancer-controller --tail=100 +kubectl -n kube-system logs deployment/ebs-csi-controller -c ebs-plugin --tail=100 +kubectl get pv +``` + +Check the controller's IAM role and AWS API errors before retrying. Do not +remove finalizers simply to make Terraform finish: that can orphan billable +resources. Restore the controller if it was removed early, or deliberately +clean up the captured AWS objects and verify each deletion before clearing a +finalizer. Never delete another application's disks or shared security groups. + +## 4. Destroy the platform + +Only proceed when monitoring state is empty and its ALB and volumes are gone. +Use the inventory to inspect their exact IDs in AWS if needed. + +```bash +cd environments +terraform workspace show +terraform plan -destroy -out=cleanup.tfplan +terraform apply cleanup.tfplan +terraform state list +cd .. +``` + +This removes EKS add-ons, the managed node group and its instances, the cluster, +IRSA providers and controller roles, the launch template, log group, NAT +resources, subnets, routing, internet gateway, and VPC. Network interfaces or +load balancers left by a controller can block VPC deletion. If the platform +destroy fails, inspect the blocking ID rather than forcing Terraform to forget it. + +## 5. Audit AWS and DNS + +```bash +python3 scripts/audit.py audit \ + --environment "$AUDIT_ENV" --region "$AUDIT_REGION" \ + --cluster-name "$AUDIT_CLUSTER" --snapshot "audit/$AUDIT_ENV-before.json" \ + --report "audit/$AUDIT_ENV-after.json" +``` + +Exit codes: `0` means no audited environment resources remain, `1` lists +remaining resources, and `2` means the audit was incomplete. An incomplete +audit is not a pass. Some APIs retain deleted NAT gateways and terminated EC2 +instances temporarily; the script treats those terminal states as deleted. +Repeat after AWS finishes asynchronous cleanup if objects are still deleting. + +To inspect individual captured objects directly: + +```bash +aws eks describe-cluster --name "$AUDIT_CLUSTER" --region "$AUDIT_REGION" +# Expected: ResourceNotFoundException. Other errors do not prove deletion. +aws ec2 describe-vpcs --region "$AUDIT_REGION" \ + --filters "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +aws ec2 describe-volumes --region "$AUDIT_REGION" \ + --filters "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +aws ec2 describe-addresses --region "$AUDIT_REGION" \ + --filters "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +aws ec2 describe-nat-gateways --region "$AUDIT_REGION" \ + --filter "Name=tag:project,Values=eks-platform-reference" "Name=tag:environment,Values=$AUDIT_ENV" +``` + +Expect empty VPC, volume, and address collections. NAT results may contain only +`deleted` gateways. The script checks load balancers, target groups, IAM roles, +and log groups too; do not use these four commands as the entire audit. + +Check Route 53 separately. The snapshot does not capture DNS records because +they cannot carry ownership tags. The saved monitoring +state JSON contains its ACM validation record names. Keep that file private; +Terraform state can contain secrets. A state-address list alone does not contain +the DNS record values. Use the exact `AUDIT_ZONE` captured before destruction; +selecting the first zone returned by a name lookup can audit the wrong zone. + +```bash +aws route53 list-resource-record-sets --hosted-zone-id "$AUDIT_ZONE" \ + --output json > "audit/$AUDIT_ENV-dns-after.json" +jq --arg host "grafana.$AUDIT_ENV.hudson.tf." \ + '[.ResourceRecordSets[] | select(.Name == $host or (.Name | endswith("." + $host)))]' \ + "audit/$AUDIT_ENV-dns-after.json" +``` + +Expect an empty array for the Grafana alias and its `_token.grafana...` ACM +validation CNAME. Keep unrelated records, including the zone's NS and SOA. +If hostnames were changed in tfvars, use the names from the before inventory +instead of the sample hostname above. Deleting DNS alone never deletes an ALB. + +## 6. Verify state is empty, then remove the unused workspaces + +Both `terraform state list` commands must return no managed resource addresses. +A no-resource state object may remain in S3 until the workspace is removed. +Select another workspace first: + +```bash +cd environments +terraform workspace select default +terraform workspace delete "$AUDIT_ENV" +cd observability +terraform workspace select default +terraform workspace delete "$AUDIT_ENV" +cd ../.. +``` + +Use your bootstrap credentials for workspace deletion: CI roles deliberately +cannot delete state objects. The default-workspace guard blocks plans and +applies, but workspace selection for maintenance is allowed. Never use +`workspace delete -force` to hide a non-empty state. S3 versioning retains old +state versions after deletion; that is intentional recovery data and still +uses storage. Do not use `force-unlock` unless you have verified there is no +Terraform process holding the lock. + +Repeat steps 1–6 for the other environment if removing both. Keep the before +and after reports as cleanup evidence. + +## 7. Retire bootstrap only when both environments are gone + +Normal environment cleanup leaves the state bucket and the four GitHub roles. +This is expected. They are shared infrastructure, and the bucket contains the +bootstrap state. To retire the entire repo's AWS footprint: + +1. Confirm both environment audits passed and neither workspace has resources. + Disable the workflow and remove its `AWS_ROLE_ARN` variables in GitHub. +2. Save bootstrap outputs and inventory its exact bucket and role ARNs. Record + whether the GitHub OIDC provider was created here or supplied as an existing + provider. An existing shared provider must remain. +3. Move bootstrap state back to a secured local backend before deleting the + bucket. Change the local `bootstrap/backend.tf` to `terraform { backend + "local" {} }` on separate lines, then run + `terraform init -migrate-state` from the `bootstrap` folder. Confirm the + prompt and verify `terraform state list` still includes bootstrap + resources. Keep an encrypted backup until the audit passes. +4. Remove the bucket's `prevent_destroy` only for this deliberate retirement. + From that same folder, review `terraform plan -destroy`. Set + `force_destroy = true` on that bucket for this cleanup so Terraform removes **all object versions + and delete markers**, then save and apply the destroy plan. This erases state + history permanently. Keep it off during normal operation. +5. If this repo created the GitHub OIDC provider but other repositories now use + it, transfer its ownership and remove it from this destroy scope before + applying. Check those dependencies explicitly; an account-wide issuer can + be shared. Imported or externally supplied providers are not disposable. + +Verify with read APIs, using the exact IDs saved before retirement: + +```bash +aws s3api head-bucket --bucket "$AUDIT_BUCKET" +# Expected HTTP 404 / NoSuchBucket. A 403 is inconclusive. +aws iam get-role --role-name "$AUDIT_ROLE_NAME" +# Repeat for all four saved roles. Expected NoSuchEntity. +aws iam get-open-id-connect-provider --open-id-connect-provider-arn "$AUDIT_GITHUB_OIDC_ARN" +# Only for a provider owned and retired here. Expected NoSuchEntity. +cd bootstrap +terraform state list +# Expected no managed resources. +cd .. +``` + +Finally confirm the hosted zone and the domain still exist, remove temporary +local plan files and kubeconfig contexts for deleted clusters, and store or +remove the private audit files according to your own retention needs. IAM's +AWS-created service-linked roles may remain and are reused by other clusters; +they are not billable and are not owned by this repo. Do not delete them just +to make an account-wide IAM list empty. + +## 8. Check the bill after AWS has caught up + +Cost Explorer and billing data lag resource deletion. On the following day, +check EKS, EC2 compute, EC2 Other (NAT, EBS and IPv4), ELB, CloudWatch, and S3 +for continued usage. Charges for the deployment day can arrive later even when +cleanup was successful; compare usage dates, not only the posting time. +Route 53's existing hosted zone and domain renewal charges should continue. + +A resource audit proves the checked resources are gone at the audit time. +A billing review catches costs from resources outside the inventory or older +experiments. Neither should be presented as proof that every resource in a +shared AWS account has been deleted. diff --git a/docs/verification.md b/docs/verification.md new file mode 100644 index 0000000..953389a --- /dev/null +++ b/docs/verification.md @@ -0,0 +1,90 @@ +# Verification + +Verification uses Terraform 1.16.5 with AWS 6.67.0, Helm 3.3.0, and Kubernetes +2.38.0 providers. Root lockfiles include the official checksums. The configured +Kubernetes version is 1.35; monitoring chart 92.7.0 and ALB controller chart +1.14.1 are pinned. + +Run these checks from the repository root: + +```bash +bash scripts/check.sh +bash scripts/test.sh +bash -n scripts/check.sh scripts/test.sh scripts/deploy.sh scripts/comment-plan.sh +``` + +The Terraform tests run plans with synthetic credentials and replace the AWS +identity, partition, cluster and other API lookups. AWS provider schemas and +IAM policy document generation still run. Kubernetes and Helm operations are +mocked. Tests cover both environment inputs, NAT topology, service account role +separation, exact GitHub environment subjects, state-bucket protection, public +Grafana outputs, automatic settings selection by workspace, and refusal of +workspaces without configured settings. Python tests check that +audit denials fail, only recognized not-found responses count as deletion, +terminal compute resources are ignored, captured untagged disks still fail, +and a snapshot from another account cannot pass. + +A live AWS apply has not been run as part of building this repo. No claim is +made that account permissions, regional quotas, DNS delegation, controllers, or +an actual second no-change plan have been verified. For the first deployment: + +1. Apply dev platform, then dev monitoring, using the README commands. +2. Check every node is Ready, both PVCs are Bound, and controller pods are healthy. +3. Check HTTPS Grafana redirects HTTP, requires login, and displays node metrics. +4. Run both plans again in the same workspace; expect no changes. +5. Follow [cleanup](cleanup.md), save the before inventory, and require the AWS + audit and DNS checks to pass after destruction. + +The local check results are: formatting passed, three roots validated, eleven +offline plan runs passed (including missing-workspace-settings and wrong-zone +rejection runs), and twenty-four Python audit, deployment and PR-comment tests passed. +Both environment roots load their settings from `terraform.tfvars`; the dev +and prod tests pass without a var-file. A separate offline test confirmed that +the default workspace fails variable validation. The script tests cover +read-only PR behavior, deployment order, working directories, workspace +selection, manual operation selection, missing-cluster deferral and errors +that must stop an apply. Cluster deletion and replacement stop before apply; +initial creation and in-place updates are allowed. Failures at each plan/apply +stage stop later deployment stages. +Push-triggered deployments are rejected; merges run validation only and live +applies require an explicit manual workflow run. Validation selects dev and +prod through `TF_WORKSPACE` without changing the local workspace selection. +The checks pass with the fresh runner's default-workspace context. +PR-comment tests cover counts without resource values, replacements, no-ops, +updates, deferred monitoring, failed plans, paginated comment lookup, and +updating only the matching environment's bot comment. GitHub API calls are +mocked; live comment publication requires an approved PR plan job. +Deployment scripts are tested from a directory outside the repository too. Bash syntax checks passed. Both +pinned Helm charts rendered using the exact Terraform-generated dev values: 104 monitoring +objects, generated Grafana credentials, the expected gp3 PVCs, disabled anonymous +access, bounded Prometheus retention, and the controller's IRSA service account. +These are local checks, not an AWS deployment receipt. + +Chart template rendering is checked separately from mocked Terraform plans. +The exact values from the offline monitoring plan are rendered through the two +published charts; the resulting Kubernetes manifests are inspected for PVC +settings, Grafana authentication, retention and controller identity. Upgrade +checks should repeat that rendering and read upstream chart upgrade notes. + +The staged source also received local correctness, security, and cleanup review. +The security pass corrected the README's claim about bootstrap protection: +privileged role creation and policy attachment do not provide a permissions +boundary. A local synthetic S3 endpoint verified that workspace initialization +and selection use the permitted prefix and environment state key. + +The complete source received correctness, security, test, maintainability, +reliability and adversarial review before the first PR. Its two findings were +fixed: deployment refuses destructive cluster plans until monitoring cleanup, +and offline tests inspect the actual IAM assume-role policies for both plan +and apply roles. The policy tests use immutable GitHub owner and repository IDs; +the bootstrap wiring test also covers the older name-only subject format. +A scratch-copy mutation check confirmed that broadening the +OIDC subject to a wildcard fails the policy assertion. Workflow syntax passed +actionlint 1.7.12. These checks do not verify live AWS permissions or controllers. + +Hosted-zone selection was checked against the account through a read-only +Terraform plan. The previous name lookup reproduced the multiple-zone error. +The explicit zone ID lookup succeeded, and its four name servers matched the +domain's public delegation. No DNS records or other AWS resources were changed +by that check. Bootstrap and monitoring use the same selector; the monitoring +test also rejects a selected zone belonging to another domain. diff --git a/environments/.terraform.lock.hcl b/environments/.terraform.lock.hcl new file mode 100644 index 0000000..3622c58 --- /dev/null +++ b/environments/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.67.0" + constraints = "~> 6.67.0" + hashes = [ + "h1:EB9ixYOZrSlYD7wtJxf88qwoyyWrlKDKxhzaCLIb3t4=", + "zh:111d5686a1f4ccbc888bb5e2229308bcdd9149898c6af97969fcdfb0e7bd2aa0", + "zh:21b3b7693bd9754c039fd546f03ed09e7510c6189aa5ee37176f144cf8dd5f95", + "zh:25e03c7f025ff4851889537605aa560d2e39bd738b33a5204b8037eb164b475f", + "zh:2817a046f1060e25bf04b0eb78f4e251130bb92dc82ec1264ce156dc9bf78e67", + "zh:2d318d674c3ec9dbd97ddb10b12ad4827be4f508c43ba2ae1e0523baa9e367e3", + "zh:2f07ab356718267299e10b6072472ded29d82753883027bf43bcd687ac72feb2", + "zh:32307e67f83bc0dc94d38cfcd23d782166a09e0e975e2a5aa7d7a3e7ca5d3da8", + "zh:4ce94853264097dd7f4542b3cfd18d2b98b06d23321db45d5e384ea275a57f63", + "zh:869656c41cc7c7412f213e488f98167cff61deafcb8cf245d25d056e8dfdc263", + "zh:8fd8c814e9d8edf152552047db23bf5b5d63f22e6b0b5d47b2af851376e99349", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9cbc02ceef9bd469da497a1e7065ff984bdacfbe919ac3cce804a86c13b6e2c6", + "zh:9ea55dda2767acfc1f6337fc7d29b1d4d79d6f8f04871f8ad99a6078d2865994", + "zh:d7149df0819fb57a160489db45d33951765b8f0118daa3b4cd549a0135bc97a1", + "zh:e5819937bb7043d08c9829b32d52d9fb55a5b9a4d8d55d550d47a3d7392db546", + "zh:f93bc38dbc0ad53842303f30eec7a22c525c4d56209b54ec33a8d375cfc4e4fd", + ] +} diff --git a/environments/backend.hcl.example b/environments/backend.hcl.example new file mode 100644 index 0000000..74e411b --- /dev/null +++ b/environments/backend.hcl.example @@ -0,0 +1,6 @@ +bucket = "REPLACE-WITH-BOOTSTRAP-BUCKET" +key = "platform.tfstate" +workspace_key_prefix = "environments" +region = "us-east-1" +encrypt = true +use_lockfile = true diff --git a/environments/locals.tf b/environments/locals.tf new file mode 100644 index 0000000..2daebb0 --- /dev/null +++ b/environments/locals.tf @@ -0,0 +1,3 @@ +locals { + settings = var.environments[terraform.workspace] +} diff --git a/environments/main.tf b/environments/main.tf new file mode 100644 index 0000000..7a9a2d2 --- /dev/null +++ b/environments/main.tf @@ -0,0 +1,33 @@ +provider "aws" { + region = local.settings.region + default_tags { tags = { environment = terraform.workspace, "managed-by" = "terraform", project = "eks-platform-reference" } } +} +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} +module "network" { + source = "../modules/network" + environment = terraform.workspace + name = local.settings.cluster_name + vpc_cidr = local.settings.vpc_cidr + subnets = local.settings.subnets + single_nat_gateway = local.settings.single_nat_gateway + internet_cidr = local.settings.internet_cidr +} +module "cluster" { + source = "../modules/cluster" + environment = terraform.workspace + cluster_name = local.settings.cluster_name + kubernetes_version = local.settings.kubernetes_version + subnet_ids = module.network.private_subnet_ids + public_access_cidrs = local.settings.public_access_cidrs + instance_types = local.settings.instance_types + node_sizes = local.settings.node_sizes + node_disk_size = local.settings.node_disk_size + log_retention_days = local.settings.log_retention_days + access_entries = merge({ + deploy = { principal_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${local.settings.deploy_role_name}", policy_name = "AmazonEKSClusterAdminPolicy" } + plan = { principal_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${local.settings.plan_role_name}", policy_name = "AmazonEKSAdminViewPolicy" } + }, { for name, arn in local.settings.administrator_arns : "admin-${name}" => { principal_arn = arn, policy_name = "AmazonEKSClusterAdminPolicy" } }) + # Route tables must be ready before nodes need outbound access. + depends_on = [module.network] +} diff --git a/environments/observability/.terraform.lock.hcl b/environments/observability/.terraform.lock.hcl new file mode 100644 index 0000000..aa033f8 --- /dev/null +++ b/environments/observability/.terraform.lock.hcl @@ -0,0 +1,67 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.67.0" + constraints = "~> 6.67.0" + hashes = [ + "h1:EB9ixYOZrSlYD7wtJxf88qwoyyWrlKDKxhzaCLIb3t4=", + "zh:111d5686a1f4ccbc888bb5e2229308bcdd9149898c6af97969fcdfb0e7bd2aa0", + "zh:21b3b7693bd9754c039fd546f03ed09e7510c6189aa5ee37176f144cf8dd5f95", + "zh:25e03c7f025ff4851889537605aa560d2e39bd738b33a5204b8037eb164b475f", + "zh:2817a046f1060e25bf04b0eb78f4e251130bb92dc82ec1264ce156dc9bf78e67", + "zh:2d318d674c3ec9dbd97ddb10b12ad4827be4f508c43ba2ae1e0523baa9e367e3", + "zh:2f07ab356718267299e10b6072472ded29d82753883027bf43bcd687ac72feb2", + "zh:32307e67f83bc0dc94d38cfcd23d782166a09e0e975e2a5aa7d7a3e7ca5d3da8", + "zh:4ce94853264097dd7f4542b3cfd18d2b98b06d23321db45d5e384ea275a57f63", + "zh:869656c41cc7c7412f213e488f98167cff61deafcb8cf245d25d056e8dfdc263", + "zh:8fd8c814e9d8edf152552047db23bf5b5d63f22e6b0b5d47b2af851376e99349", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9cbc02ceef9bd469da497a1e7065ff984bdacfbe919ac3cce804a86c13b6e2c6", + "zh:9ea55dda2767acfc1f6337fc7d29b1d4d79d6f8f04871f8ad99a6078d2865994", + "zh:d7149df0819fb57a160489db45d33951765b8f0118daa3b4cd549a0135bc97a1", + "zh:e5819937bb7043d08c9829b32d52d9fb55a5b9a4d8d55d550d47a3d7392db546", + "zh:f93bc38dbc0ad53842303f30eec7a22c525c4d56209b54ec33a8d375cfc4e4fd", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "3.3.0" + constraints = "~> 3.3.0" + hashes = [ + "h1:Qbt2+dtOJty3glPZI9jMCXq82GSAKBk/T2+wBMIdu0c=", + "zh:24a942352667cf49bed467c51a951f17a6c13e0cdf1c505b01b41f75a1a60d0f", + "zh:2c057eafd1e4d8b50acdc676f9a3096e38fb7d30c3fab3d64af82fa04a2f767e", + "zh:44252c60034ca163f2f68eaeece2583411bca25ac83bc58350e24d4f1c2d60a7", + "zh:4628da7c781eebbce1b13d12b498856815bde0e174d7a0d0b85a5d89a1d36d17", + "zh:59b3f4fd8e236c5733032c289a9636ac8bb6c92bc11b540d979b92d005f6cf98", + "zh:5d5f11201b986485ca1d33096f3545c6074b3b4a592864620a8df7f7c399b59f", + "zh:5e656afadb2db3f418eb254b2da02b00851eba8ec63196d1f22b2a2854fe9fb9", + "zh:77515e5576f4ea3d8d9e404237882de62518950ccd7b312371f5c03580f22fb0", + "zh:a4e601eddf9781b5f8678ee9c2b6fb0791fbe6ab85a53707efdf80f714a3011c", + "zh:eaa58edc8bb43aeb0fe6c0f99975cb24e2e9f5441e938a389fe848795a30b825", + "zh:ec97b17b82222d2008256f9b10b88f564d30f717a43438fe425a802ac0ccf2f3", + "zh:f18613d2aa3fd3945d394911e210af7bf58968f64631b6b2e229852b60a2803e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.38.0" + hashes = [ + "h1:5CkveFo5ynsLdzKk+Kv+r7+U9rMrNjfZPT3a0N/fhgE=", + "zh:0af928d776eb269b192dc0ea0f8a3f0f5ec117224cd644bdacdc682300f84ba0", + "zh:1be998e67206f7cfc4ffe77c01a09ac91ce725de0abaec9030b22c0a832af44f", + "zh:326803fe5946023687d603f6f1bab24de7af3d426b01d20e51d4e6fbe4e7ec1b", + "zh:4a99ec8d91193af961de1abb1f824be73df07489301d62e6141a656b3ebfff12", + "zh:5136e51765d6a0b9e4dbcc3b38821e9736bd2136cf15e9aac11668f22db117d2", + "zh:63fab47349852d7802fb032e4f2b6a101ee1ce34b62557a9ad0f0f0f5b6ecfdc", + "zh:924fb0257e2d03e03e2bfe9c7b99aa73c195b1f19412ca09960001bee3c50d15", + "zh:b63a0be5e233f8f6727c56bed3b61eb9456ca7a8bb29539fba0837f1badf1396", + "zh:d39861aa21077f1bc899bc53e7233262e530ba8a3a2d737449b100daeb303e4d", + "zh:de0805e10ebe4c83ce3b728a67f6b0f9d18be32b25146aa89116634df5145ad4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:faf23e45f0090eef8ba28a8aac7ec5d4fdf11a36c40a8d286304567d71c1e7db", + ] +} diff --git a/environments/observability/backend.hcl.example b/environments/observability/backend.hcl.example new file mode 100644 index 0000000..286556a --- /dev/null +++ b/environments/observability/backend.hcl.example @@ -0,0 +1,6 @@ +bucket = "REPLACE-WITH-BOOTSTRAP-BUCKET" +key = "observability.tfstate" +workspace_key_prefix = "environments" +region = "us-east-1" +encrypt = true +use_lockfile = true diff --git a/environments/observability/locals.tf b/environments/observability/locals.tf new file mode 100644 index 0000000..2daebb0 --- /dev/null +++ b/environments/observability/locals.tf @@ -0,0 +1,3 @@ +locals { + settings = var.environments[terraform.workspace] +} diff --git a/environments/observability/main.tf b/environments/observability/main.tf new file mode 100644 index 0000000..09fd499 --- /dev/null +++ b/environments/observability/main.tf @@ -0,0 +1,67 @@ +provider "aws" { + region = local.settings.region + default_tags { tags = { environment = terraform.workspace, "managed-by" = "terraform", project = "eks-platform-reference" } } +} +data "aws_eks_cluster" "this" { name = local.settings.cluster_name } +provider "helm" { + kubernetes = { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", local.settings.cluster_name, "--region", local.settings.region] + } + } +} +provider "kubernetes" { + host = data.aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.this.certificate_authority[0].data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", local.settings.cluster_name, "--region", local.settings.region] + } +} +data "aws_route53_zone" "public" { + zone_id = local.settings.hosted_zone_id + private_zone = false + lifecycle { + postcondition { + condition = !self.private_zone && lower(trimsuffix(self.name, ".")) == lower(trimsuffix(local.settings.domain_name, ".")) + error_message = "The selected hosted zone must be public and match domain_name." + } + } +} +data "aws_iam_role" "controller" { name = "${local.settings.cluster_name}-alb" } +data "aws_subnets" "public" { + filter { + name = "vpc-id" + values = [data.aws_eks_cluster.this.vpc_config[0].vpc_id] + } + filter { + name = "tag:kubernetes.io/role/elb" + values = ["1"] + } +} +module "observability" { + region = local.settings.region + cluster_name = local.settings.cluster_name + vpc_id = data.aws_eks_cluster.this.vpc_config[0].vpc_id + public_subnet_ids = data.aws_subnets.public.ids + controller_role_arn = data.aws_iam_role.controller.arn + controller_chart_version = local.settings.controller_chart_version + hosted_zone_id = data.aws_route53_zone.public.zone_id + grafana_hostname = local.settings.grafana_hostname + load_balancer_name = local.settings.load_balancer_name + ingress_cidrs = local.settings.ingress_cidrs + source = "../../modules/observability" + environment = terraform.workspace + chart_version = local.settings.chart_version + namespace = local.settings.namespace + prometheus_retention = local.settings.prometheus_retention + prometheus_storage_size = local.settings.prometheus_storage_size + grafana_storage_size = local.settings.grafana_storage_size + prometheus_memory_request = local.settings.prometheus_memory_request + prometheus_memory_limit = local.settings.prometheus_memory_limit +} diff --git a/environments/observability/outputs.tf b/environments/observability/outputs.tf new file mode 100644 index 0000000..b9253b5 --- /dev/null +++ b/environments/observability/outputs.tf @@ -0,0 +1,29 @@ +output "namespace" { + description = "Monitoring namespace." + value = module.observability.namespace +} + +output "grafana_service" { + description = "Grafana service for port-forward." + value = module.observability.grafana_service +} + +output "grafana_secret" { + description = "Grafana admin credential secret." + value = module.observability.grafana_secret +} + +output "grafana_url" { + description = "Public Grafana HTTPS endpoint." + value = module.observability.grafana_url +} + +output "environment" { + description = "Selected Terraform workspace." + value = terraform.workspace +} + +output "hosted_zone_id" { + description = "Public hosted zone selected for ingress and certificate validation." + value = data.aws_route53_zone.public.zone_id +} diff --git a/environments/observability/terraform.tfvars b/environments/observability/terraform.tfvars new file mode 100644 index 0000000..1b3edcb --- /dev/null +++ b/environments/observability/terraform.tfvars @@ -0,0 +1,39 @@ +environments = { + dev = { + region = "us-east-1" + cluster_name = "eks-platform-reference-dev" + chart_version = "92.7.0" + namespace = "monitoring" + prometheus_retention = "2d" + prometheus_storage_size = "10Gi" + grafana_storage_size = "5Gi" + prometheus_memory_request = "512Mi" + prometheus_memory_limit = "1Gi" + + domain_name = "hudson.tf" + grafana_hostname = "grafana.dev.hudson.tf" + load_balancer_name = "eks-reference-dev-grafana" + controller_chart_version = "1.14.1" + ingress_cidrs = ["0.0.0.0/0"] + hosted_zone_id = "Z0992412PWY8TUM0P6ZU" + } + + prod = { + region = "us-east-1" + cluster_name = "eks-platform-reference-prod" + chart_version = "92.7.0" + namespace = "monitoring" + prometheus_retention = "7d" + prometheus_storage_size = "30Gi" + grafana_storage_size = "5Gi" + prometheus_memory_request = "1Gi" + prometheus_memory_limit = "2Gi" + + domain_name = "hudson.tf" + grafana_hostname = "grafana.prod.hudson.tf" + load_balancer_name = "eks-reference-prod-grafana" + controller_chart_version = "1.14.1" + ingress_cidrs = ["0.0.0.0/0"] + hosted_zone_id = "Z0992412PWY8TUM0P6ZU" + } +} diff --git a/environments/observability/tests/monitoring.tftest.hcl b/environments/observability/tests/monitoring.tftest.hcl new file mode 100644 index 0000000..fa0f806 --- /dev/null +++ b/environments/observability/tests/monitoring.tftest.hcl @@ -0,0 +1,56 @@ +variable "test_workspace" { type = string } +variable "environments" { type = any } +provider "aws" { + region = var.environments[var.test_workspace].region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +mock_provider "helm" {} +mock_provider "kubernetes" {} +override_data { + target = data.aws_eks_cluster.this + values = { + endpoint = "https://offline.invalid" + certificate_authority = [{ data = "dGVzdA==" }] + vpc_config = [{ vpc_id = "vpc-offline", subnet_ids = ["subnet-offline-a", "subnet-offline-b"], cluster_security_group_id = "sg-offline", security_group_ids = [], endpoint_private_access = true, endpoint_public_access = true, public_access_cidrs = [] }] + } +} +override_data { + target = data.aws_route53_zone.public + values = { zone_id = "ZOFFLINETEST", name = "hudson.tf", private_zone = false } +} +override_data { + target = data.aws_iam_role.controller + values = { arn = "arn:aws:iam::111122223333:role/offline-alb" } +} +override_data { + target = data.aws_subnets.public + values = { ids = ["subnet-offline-a", "subnet-offline-b"] } +} +override_data { + target = module.observability.data.aws_lb.grafana + values = { dns_name = "offline.elb.amazonaws.com", zone_id = "ZOFFLINEALB" } +} +run "monitoring_plan" { + command = plan + assert { + condition = output.grafana_service == "monitoring-grafana" + error_message = "Ingress and the documented password lookup must address the chart's Grafana service." + } + assert { + condition = output.grafana_url == "https://${var.environments[var.test_workspace].grafana_hostname}" + error_message = "Grafana output must use the configured HTTPS hostname." + } +} + +run "reject_incorrect_hosted_zone" { + command = plan + override_data { + target = data.aws_route53_zone.public + values = { zone_id = "ZOFFLINETEST", name = "other.invalid", private_zone = false } + } + expect_failures = [data.aws_route53_zone.public] +} diff --git a/environments/observability/variables.tf b/environments/observability/variables.tf new file mode 100644 index 0000000..c3e31f3 --- /dev/null +++ b/environments/observability/variables.tf @@ -0,0 +1,40 @@ +variable "environments" { + description = "Environment settings keyed by Terraform workspace; loaded automatically from terraform.tfvars." + type = map(object({ + # Cluster AWS region. + region = string + # Existing EKS cluster name. + cluster_name = string + # Pinned kube-prometheus-stack chart. + chart_version = string + # Monitoring namespace. + namespace = string + # Prometheus retention duration. + prometheus_retention = string + # Prometheus storage in integer Gi units. + prometheus_storage_size = string + # Grafana storage capacity. + grafana_storage_size = string + # Prometheus memory request. + prometheus_memory_request = string + # Prometheus memory limit. + prometheus_memory_limit = string + # Existing public Route 53 hosted zone name. + domain_name = string + # Public HTTPS Grafana hostname. + grafana_hostname = string + # Unique ALB name, maximum 32 characters. + load_balancer_name = string + # AWS Load Balancer Controller chart pin. + controller_chart_version = string + # Client CIDRs permitted to reach Grafana. + ingress_cidrs = list(string) + # Exact existing public Route 53 zone ID; zone names are not unique. + hosted_zone_id = string + })) + + validation { + condition = terraform.workspace != "default" && contains(keys(var.environments), terraform.workspace) + error_message = "Select a named workspace with settings in terraform.tfvars, such as dev or prod." + } +} diff --git a/environments/observability/versions.tf b/environments/observability/versions.tf new file mode 100644 index 0000000..bbbc461 --- /dev/null +++ b/environments/observability/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + kubernetes = { source = "hashicorp/kubernetes", version = "~> 2.38.0" } + helm = { + source = "hashicorp/helm" + version = "~> 3.3.0" + } + } +} diff --git a/environments/outputs.tf b/environments/outputs.tf new file mode 100644 index 0000000..10440b7 --- /dev/null +++ b/environments/outputs.tf @@ -0,0 +1,49 @@ +output "region" { + description = "AWS region." + value = local.settings.region +} + +output "cluster_name" { + description = "EKS cluster name." + value = module.cluster.cluster_name +} + +output "cluster_endpoint" { + description = "Kubernetes API endpoint." + value = module.cluster.cluster_endpoint +} + +output "vpc_id" { + description = "Environment VPC." + value = module.network.vpc_id +} + +output "private_subnet_ids" { + description = "Private subnets." + value = module.network.private_subnet_ids +} + +output "nat_public_ips" { + description = "NAT egress addresses." + value = module.network.nat_public_ips +} + +output "oidc_provider_arn" { + description = "IRSA provider ARN." + value = module.cluster.oidc_provider_arn +} + +output "oidc_issuer_url" { + description = "IRSA issuer URL." + value = module.cluster.oidc_issuer_url +} + +output "irsa_role_arns" { + description = "Add-on IAM roles." + value = module.cluster.irsa_role_arns +} + +output "environment" { + description = "Selected Terraform workspace." + value = terraform.workspace +} diff --git a/environments/terraform.tfvars b/environments/terraform.tfvars new file mode 100644 index 0000000..83c0988 --- /dev/null +++ b/environments/terraform.tfvars @@ -0,0 +1,47 @@ +environments = { + dev = { + region = "us-east-1" + cluster_name = "eks-platform-reference-dev" + vpc_cidr = "10.20.0.0/16" + subnets = { + a = { availability_zone = "us-east-1a", public_cidr = "10.20.0.0/24", private_cidr = "10.20.16.0/20" } + b = { availability_zone = "us-east-1b", public_cidr = "10.20.1.0/24", private_cidr = "10.20.32.0/20" } + } + single_nat_gateway = true + internet_cidr = "0.0.0.0/0" + kubernetes_version = "1.35" + # GitHub-hosted runners have changing IPs. Use restricted CIDRs with a fixed-egress runner. + public_access_cidrs = ["0.0.0.0/0"] + instance_types = ["t3.medium"] + node_sizes = { min = 2, desired = 2, max = 3 } + node_disk_size = 30 + log_retention_days = 7 + deploy_role_name = "github-eks-platform-dev-apply" + plan_role_name = "github-eks-platform-dev-plan" + administrator_arns = { + patrick = "arn:aws:iam::626193791566:user/patrick" + } + } + + prod = { + region = "us-east-1" + cluster_name = "eks-platform-reference-prod" + vpc_cidr = "10.30.0.0/16" + subnets = { + a = { availability_zone = "us-east-1a", public_cidr = "10.30.0.0/24", private_cidr = "10.30.16.0/20" } + b = { availability_zone = "us-east-1b", public_cidr = "10.30.1.0/24", private_cidr = "10.30.32.0/20" } + c = { availability_zone = "us-east-1c", public_cidr = "10.30.2.0/24", private_cidr = "10.30.48.0/20" } + } + single_nat_gateway = false + internet_cidr = "0.0.0.0/0" + kubernetes_version = "1.35" + # GitHub-hosted runners have changing IPs. Use restricted CIDRs with a fixed-egress runner. + public_access_cidrs = ["0.0.0.0/0"] + instance_types = ["m6i.large"] + node_sizes = { min = 3, desired = 3, max = 6 } + node_disk_size = 30 + log_retention_days = 30 + deploy_role_name = "github-eks-platform-prod-apply" + plan_role_name = "github-eks-platform-prod-plan" + } +} diff --git a/environments/tests/platform.tftest.hcl b/environments/tests/platform.tftest.hcl new file mode 100644 index 0000000..74ebaf9 --- /dev/null +++ b/environments/tests/platform.tftest.hcl @@ -0,0 +1,52 @@ +variable "test_workspace" { type = string } +variable "environments" { type = any } +# Offline provider plans still run real AWS resource schemas and IAM JSON rendering. +provider "aws" { + region = var.environments[var.test_workspace].region + access_key = "offline-testing" + secret_key = "offline-testing" + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} +override_data { + target = data.aws_caller_identity.current + values = { account_id = "111122223333", arn = "arn:aws:iam::111122223333:user/test", user_id = "test" } +} +override_data { + target = data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.cluster.data.aws_partition.current + values = { partition = "aws", dns_suffix = "amazonaws.com", reverse_dns_prefix = "com.amazonaws" } +} +override_data { + target = module.cluster.data.aws_eks_addon_version.this + values = { version = "v1.0.0-eksbuild.1" } +} +run "platform_plan" { + command = plan + assert { + condition = output.environment == var.test_workspace && output.cluster_name == var.environments[var.test_workspace].cluster_name + error_message = "Workspace selection must choose the matching environment settings." + } + assert { + condition = length(output.nat_public_ips) == (var.environments[var.test_workspace].single_nat_gateway ? 1 : length(var.environments[var.test_workspace].subnets)) + error_message = "NAT topology must match the environment's cost and availability choice." + } + assert { + condition = length(output.private_subnet_ids) == length(var.environments[var.test_workspace].subnets) + error_message = "Every configured zone needs a private node subnet." + } + assert { + condition = toset(keys(output.irsa_role_arns)) == toset(["cni", "ebs", "alb"]) + error_message = "AWS controllers need distinct service account roles." + } +} + +run "reject_missing_workspace_settings" { + command = plan + variables { environments = {} } + expect_failures = [var.environments] +} diff --git a/environments/variables.tf b/environments/variables.tf new file mode 100644 index 0000000..ad34694 --- /dev/null +++ b/environments/variables.tf @@ -0,0 +1,40 @@ +variable "environments" { + description = "Environment settings keyed by Terraform workspace; loaded automatically from terraform.tfvars." + type = map(object({ + # AWS region for the environment. + region = string + # EKS name, shared with bootstrap configuration. + cluster_name = string + # Non-overlapping VPC CIDR. + vpc_cidr = string + # Explicit AZ and subnet CIDRs. + subnets = map(object({ availability_zone = string, public_cidr = string, private_cidr = string })) + # Share one NAT gateway across zones. + single_nat_gateway = bool + # Internet route destination CIDR. + internet_cidr = string + # EKS Kubernetes minor version. + kubernetes_version = string + # Kubernetes API public ingress CIDRs. + public_access_cidrs = list(string) + # Managed node EC2 instance types. + instance_types = list(string) + # Managed node scaling settings. + node_sizes = object({ min = number, desired = number, max = number }) + # Node root volume GiB. + node_disk_size = number + # Control plane log retention. + log_retention_days = number + # Bootstrap-created deploy role name. + deploy_role_name = string + # Bootstrap-created plan role name. + plan_role_name = string + # Stable names of human administrator IAM roles, never STS session ARNs. + administrator_arns = optional(map(string), {}) + })) + + validation { + condition = terraform.workspace != "default" && contains(keys(var.environments), terraform.workspace) + error_message = "Select a named workspace with settings in terraform.tfvars, such as dev or prod." + } +} diff --git a/environments/versions.tf b/environments/versions.tf new file mode 100644 index 0000000..5775bc5 --- /dev/null +++ b/environments/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/modules/cluster/load-balancer-controller-policy.json b/modules/cluster/load-balancer-controller-policy.json new file mode 100644 index 0000000..761d0e7 --- /dev/null +++ b/modules/cluster/load-balancer-controller-policy.json @@ -0,0 +1,251 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateServiceLinkedRole" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "ec2:GetSecurityGroupsForVpc", + "ec2:DescribeIpamPools", + "ec2:DescribeRouteTables", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "StringEquals": { + "ec2:CreateAction": "CreateSecurityGroup" + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation", + "elasticloadbalancing:ModifyIpPools" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "StringEquals": { + "elasticloadbalancing:CreateAction": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule", + "elasticloadbalancing:SetRulePriorities" + ], + "Resource": "*" + } + ] +} diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf new file mode 100644 index 0000000..46e144b --- /dev/null +++ b/modules/cluster/main.tf @@ -0,0 +1,219 @@ +data "aws_partition" "current" {} +locals { + tags = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) + oidc_issuer = replace(aws_iam_openid_connect_provider.this.url, "https://", "") + partition = data.aws_partition.current.partition + irsa = { + alb = { subject = "system:serviceaccount:kube-system:aws-load-balancer-controller", policy = null } + cni = { subject = "system:serviceaccount:kube-system:aws-node", policy = "AmazonEKS_CNI_Policy" } + ebs = { subject = "system:serviceaccount:kube-system:ebs-csi-controller-sa", policy = "service-role/AmazonEBSCSIDriverPolicy" } + } +} +data "aws_iam_policy_document" "cluster_trust" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["eks.amazonaws.com"] + } + } +} +resource "aws_iam_role" "cluster" { + name = "${var.cluster_name}-cluster" + assume_role_policy = data.aws_iam_policy_document.cluster_trust.json + tags = local.tags +} +resource "aws_iam_role_policy_attachment" "cluster" { + role = aws_iam_role.cluster.name + policy_arn = "arn:${local.partition}:iam::aws:policy/AmazonEKSClusterPolicy" +} +resource "aws_cloudwatch_log_group" "cluster" { + name = "/aws/eks/${var.cluster_name}/cluster" + retention_in_days = var.log_retention_days + tags = local.tags +} +resource "aws_eks_cluster" "this" { + name = var.cluster_name + role_arn = aws_iam_role.cluster.arn + version = var.kubernetes_version + enabled_cluster_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"] + access_config { + authentication_mode = "API" + bootstrap_cluster_creator_admin_permissions = false + } + upgrade_policy { support_type = "STANDARD" } + vpc_config { + subnet_ids = var.subnet_ids + endpoint_private_access = true + endpoint_public_access = true + public_access_cidrs = var.public_access_cidrs + } + tags = local.tags + depends_on = [aws_iam_role_policy_attachment.cluster, aws_cloudwatch_log_group.cluster] +} +resource "aws_iam_openid_connect_provider" "this" { + url = aws_eks_cluster.this.identity[0].oidc[0].issuer + client_id_list = ["sts.amazonaws.com"] + tags = local.tags +} +data "aws_iam_policy_document" "irsa" { + for_each = local.irsa + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.this.arn] + } + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:sub" + values = [each.value.subject] + } + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:aud" + values = ["sts.amazonaws.com"] + } + } +} +resource "aws_iam_role" "irsa" { + for_each = local.irsa + name = "${var.cluster_name}-${each.key}" + assume_role_policy = data.aws_iam_policy_document.irsa[each.key].json + tags = local.tags +} +resource "aws_iam_role_policy_attachment" "irsa" { + for_each = { for k, v in local.irsa : k => v if v.policy != null } + role = aws_iam_role.irsa[each.key].name + policy_arn = "arn:${local.partition}:iam::aws:policy/${each.value.policy}" +} +data "aws_iam_policy_document" "node_trust" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} +resource "aws_iam_role" "node" { + name = "${var.cluster_name}-node" + assume_role_policy = data.aws_iam_policy_document.node_trust.json + tags = local.tags +} +resource "aws_iam_role_policy_attachment" "node" { + for_each = toset(["AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryPullOnly"]) + role = aws_iam_role.node.name + policy_arn = "arn:${local.partition}:iam::aws:policy/${each.key}" +} +resource "aws_launch_template" "node" { + name_prefix = "${var.cluster_name}-" + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } + block_device_mappings { + device_name = "/dev/xvda" + ebs { + volume_size = var.node_disk_size + volume_type = "gp3" + encrypted = true + delete_on_termination = true + } + } + dynamic "tag_specifications" { + for_each = toset(["instance", "volume", "network-interface"]) + content { + resource_type = tag_specifications.value + tags = local.tags + } + } + tags = local.tags +} +data "aws_eks_addon_version" "this" { + for_each = toset(["vpc-cni", "coredns", "kube-proxy", "aws-ebs-csi-driver"]) + addon_name = each.key + kubernetes_version = var.kubernetes_version + most_recent = false +} +# CNI must have its IAM role before any node tries to bring up pod networking. +resource "aws_eks_addon" "cni" { + cluster_name = aws_eks_cluster.this.name + addon_name = "vpc-cni" + addon_version = data.aws_eks_addon_version.this["vpc-cni"].version + service_account_role_arn = aws_iam_role.irsa["cni"].arn + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "PRESERVE" + tags = local.tags + depends_on = [aws_iam_role_policy_attachment.irsa] +} +resource "aws_eks_node_group" "this" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "${var.cluster_name}-workers" + node_role_arn = aws_iam_role.node.arn + subnet_ids = var.subnet_ids + instance_types = var.instance_types + ami_type = "AL2023_x86_64_STANDARD" + capacity_type = "ON_DEMAND" + launch_template { + id = aws_launch_template.node.id + version = aws_launch_template.node.latest_version + } + scaling_config { + min_size = var.node_sizes.min + desired_size = var.node_sizes.desired + max_size = var.node_sizes.max + } + update_config { max_unavailable = 1 } + labels = { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" } + tags = local.tags + depends_on = [aws_iam_role_policy_attachment.node, aws_eks_addon.cni] +} +resource "aws_eks_addon" "this" { + for_each = toset(["coredns", "kube-proxy", "aws-ebs-csi-driver"]) + cluster_name = aws_eks_cluster.this.name + addon_name = each.key + addon_version = data.aws_eks_addon_version.this[each.key].version + service_account_role_arn = each.key == "aws-ebs-csi-driver" ? aws_iam_role.irsa["ebs"].arn : null + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "PRESERVE" + tags = local.tags + depends_on = [aws_eks_node_group.this, aws_iam_role_policy_attachment.irsa] +} +resource "aws_eks_access_entry" "this" { + for_each = var.access_entries + cluster_name = aws_eks_cluster.this.name + principal_arn = each.value.principal_arn + tags = local.tags +} +resource "aws_eks_access_policy_association" "this" { + for_each = var.access_entries + cluster_name = aws_eks_cluster.this.name + principal_arn = aws_eks_access_entry.this[each.key].principal_arn + policy_arn = "arn:${local.partition}:eks::aws:cluster-access-policy/${each.value.policy_name}" + access_scope { type = "cluster" } +} + +# Upstream v2.14.1 policy, kept in sync with the pinned Helm controller chart. +resource "aws_iam_role_policy" "load_balancer_controller" { + name = "load-balancer-controller" + role = aws_iam_role.irsa["alb"].id + policy = replace(file("${path.module}/load-balancer-controller-policy.json"), "arn:aws:", "arn:${local.partition}:") +} +# EKS owns these resources, but does not propagate cluster/node-group tags to them. +resource "aws_ec2_tag" "cluster_security_group" { + for_each = local.tags + resource_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id + key = each.key + value = each.value +} +resource "aws_autoscaling_group_tag" "node" { + for_each = local.tags + autoscaling_group_name = aws_eks_node_group.this.resources[0].autoscaling_groups[0].name + tag { + key = each.key + value = each.value + propagate_at_launch = true + } +} diff --git a/modules/cluster/outputs.tf b/modules/cluster/outputs.tf new file mode 100644 index 0000000..132ea34 --- /dev/null +++ b/modules/cluster/outputs.tf @@ -0,0 +1,39 @@ +output "cluster_name" { + description = "Cluster name for CLI and Helm authentication." + value = aws_eks_cluster.this.name +} + +output "cluster_endpoint" { + description = "Kubernetes API endpoint." + value = aws_eks_cluster.this.endpoint +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded Kubernetes CA." + value = aws_eks_cluster.this.certificate_authority[0].data +} + +output "cluster_security_group_id" { + description = "EKS managed security group." + value = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id +} + +output "oidc_provider_arn" { + description = "IRSA OIDC provider ARN." + value = aws_iam_openid_connect_provider.this.arn +} + +output "oidc_issuer_url" { + description = "IRSA issuer for future service account trust policies." + value = aws_iam_openid_connect_provider.this.url +} + +output "irsa_role_arns" { + description = "Service account roles for CNI and EBS CSI." + value = { for k, role in aws_iam_role.irsa : k => role.arn } +} + +output "node_group_name" { + description = "Managed node group name." + value = aws_eks_node_group.this.node_group_name +} diff --git a/modules/cluster/variables.tf b/modules/cluster/variables.tf new file mode 100644 index 0000000..9d932f9 --- /dev/null +++ b/modules/cluster/variables.tf @@ -0,0 +1,55 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "cluster_name" { + description = "EKS cluster name and prefix for its IAM roles." + type = string +} + +variable "kubernetes_version" { + description = "EKS Kubernetes minor version in standard support." + type = string +} + +variable "subnet_ids" { + description = "Private subnet IDs for the cluster and nodes." + type = list(string) +} + +variable "public_access_cidrs" { + description = "CIDRs permitted to reach the public Kubernetes API." + type = list(string) +} + +variable "instance_types" { + description = "EC2 instance types for the managed node group." + type = list(string) +} + +variable "node_sizes" { + description = "Minimum, desired and maximum node counts." + type = object({ min = number, desired = number, max = number }) +} + +variable "node_disk_size" { + description = "Encrypted root volume size in GiB." + type = number +} + +variable "log_retention_days" { + description = "CloudWatch control plane log retention." + type = number +} + +variable "access_entries" { + description = "Stable entry names mapped to IAM principals and EKS access policy names." + type = map(object({ principal_arn = string, policy_name = string })) +} diff --git a/modules/cluster/versions.tf b/modules/cluster/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/modules/cluster/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/modules/network/main.tf b/modules/network/main.tf new file mode 100644 index 0000000..1b2da9e --- /dev/null +++ b/modules/network/main.tf @@ -0,0 +1,72 @@ +locals { + tags = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) + primary_nat_key = sort(keys(var.subnets))[0] + nat_subnets = var.single_nat_gateway ? { for k, v in var.subnets : k => v if k == local.primary_nat_key } : var.subnets +} +resource "aws_vpc" "this" { + cidr_block = var.vpc_cidr + enable_dns_support = true + enable_dns_hostnames = true + tags = merge(local.tags, { Name = var.name }) +} +resource "aws_internet_gateway" "this" { + vpc_id = aws_vpc.this.id + tags = merge(local.tags, { Name = var.name }) +} +resource "aws_subnet" "public" { + for_each = var.subnets + vpc_id = aws_vpc.this.id + availability_zone = each.value.availability_zone + cidr_block = each.value.public_cidr + map_public_ip_on_launch = false + tags = merge(local.tags, { Name = "${var.name}-public-${each.key}", "kubernetes.io/role/elb" = "1" }) +} +resource "aws_subnet" "private" { + for_each = var.subnets + vpc_id = aws_vpc.this.id + availability_zone = each.value.availability_zone + cidr_block = each.value.private_cidr + tags = merge(local.tags, { Name = "${var.name}-private-${each.key}", "kubernetes.io/role/internal-elb" = "1" }) +} +resource "aws_eip" "nat" { + for_each = local.nat_subnets + domain = "vpc" + tags = merge(local.tags, { Name = "${var.name}-nat-${each.key}" }) +} +resource "aws_nat_gateway" "this" { + for_each = local.nat_subnets + allocation_id = aws_eip.nat[each.key].id + subnet_id = aws_subnet.public[each.key].id + tags = merge(local.tags, { Name = "${var.name}-${each.key}" }) + depends_on = [aws_internet_gateway.this] +} +resource "aws_route_table" "public" { + vpc_id = aws_vpc.this.id + tags = merge(local.tags, { Name = "${var.name}-public" }) +} +resource "aws_route" "public" { + route_table_id = aws_route_table.public.id + destination_cidr_block = var.internet_cidr + gateway_id = aws_internet_gateway.this.id +} +resource "aws_route_table_association" "public" { + for_each = var.subnets + subnet_id = aws_subnet.public[each.key].id + route_table_id = aws_route_table.public.id +} +resource "aws_route_table" "private" { + for_each = var.subnets + vpc_id = aws_vpc.this.id + tags = merge(local.tags, { Name = "${var.name}-private-${each.key}" }) +} +resource "aws_route" "private" { + for_each = var.subnets + route_table_id = aws_route_table.private[each.key].id + destination_cidr_block = var.internet_cidr + nat_gateway_id = aws_nat_gateway.this[var.single_nat_gateway ? local.primary_nat_key : each.key].id +} +resource "aws_route_table_association" "private" { + for_each = var.subnets + subnet_id = aws_subnet.private[each.key].id + route_table_id = aws_route_table.private[each.key].id +} diff --git a/modules/network/outputs.tf b/modules/network/outputs.tf new file mode 100644 index 0000000..f59891a --- /dev/null +++ b/modules/network/outputs.tf @@ -0,0 +1,19 @@ +output "vpc_id" { + description = "VPC identifier." + value = aws_vpc.this.id +} + +output "public_subnet_ids" { + description = "Public subnets in stable key order." + value = [for s in aws_subnet.public : s.id] +} + +output "private_subnet_ids" { + description = "Private subnets in stable key order." + value = [for s in aws_subnet.private : s.id] +} + +output "nat_public_ips" { + description = "Outbound IP addresses by availability zone." + value = { for k, e in aws_eip.nat : k => e.public_ip } +} diff --git a/modules/network/variables.tf b/modules/network/variables.tf new file mode 100644 index 0000000..5eec17e --- /dev/null +++ b/modules/network/variables.tf @@ -0,0 +1,35 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "name" { + description = "Prefix for network resource names." + type = string +} + +variable "vpc_cidr" { + description = "IPv4 CIDR for the VPC." + type = string +} + +variable "subnets" { + description = "One public and one private subnet per named availability zone." + type = map(object({ availability_zone = string, public_cidr = string, private_cidr = string })) +} + +variable "single_nat_gateway" { + description = "Use one shared NAT gateway instead of one in each availability zone." + type = bool +} + +variable "internet_cidr" { + description = "Destination CIDR for internet routes." + type = string +} diff --git a/modules/network/versions.tf b/modules/network/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/modules/network/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/modules/observability/ingress.tf b/modules/observability/ingress.tf new file mode 100644 index 0000000..b3a9cd1 --- /dev/null +++ b/modules/observability/ingress.tf @@ -0,0 +1,102 @@ + +resource "helm_release" "load_balancer_controller" { + name = "aws-load-balancer-controller" + namespace = "kube-system" + repository = "https://aws.github.io/eks-charts" + chart = "aws-load-balancer-controller" + version = var.controller_chart_version + atomic = true + wait = true + timeout = 600 + values = [yamlencode({ + clusterName = var.cluster_name + region = var.region + vpcId = var.vpc_id + replicaCount = 2 + defaultTags = local.labels + serviceAccount = { + create = true + name = "aws-load-balancer-controller" + annotations = { "eks.amazonaws.com/role-arn" = var.controller_role_arn } + } + })] +} +resource "aws_acm_certificate" "grafana" { + domain_name = var.grafana_hostname + validation_method = "DNS" + tags = local.labels + lifecycle { create_before_destroy = true } +} +resource "aws_route53_record" "validation" { + for_each = { + for option in aws_acm_certificate.grafana.domain_validation_options : option.domain_name => { + name = option.resource_record_name + type = option.resource_record_type + value = option.resource_record_value + } + } + zone_id = var.hosted_zone_id + name = each.value.name + type = each.value.type + ttl = 60 + records = [each.value.value] +} +resource "aws_acm_certificate_validation" "grafana" { + certificate_arn = aws_acm_certificate.grafana.arn + validation_record_fqdns = [for record in aws_route53_record.validation : record.fqdn] +} +resource "kubernetes_ingress_v1" "grafana" { + wait_for_load_balancer = true + metadata { + name = "grafana" + namespace = kubernetes_namespace_v1.monitoring.metadata[0].name + labels = local.labels + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/load-balancer-name" = var.load_balancer_name + "alb.ingress.kubernetes.io/subnets" = join(",", var.public_subnet_ids) + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.ingress_cidrs) + "alb.ingress.kubernetes.io/listen-ports" = jsonencode([{ HTTP = 80 }, { HTTPS = 443 }]) + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/ssl-policy" = "ELBSecurityPolicy-TLS13-1-2-2021-06" + "alb.ingress.kubernetes.io/certificate-arn" = aws_acm_certificate_validation.grafana.certificate_arn + "alb.ingress.kubernetes.io/healthcheck-path" = "/api/health" + "alb.ingress.kubernetes.io/tags" = join(",", [for key, value in local.labels : "${key}=${value}"]) + } + } + spec { + ingress_class_name = "alb" + rule { + host = var.grafana_hostname + http { + path { + path = "/" + path_type = "Prefix" + backend { + service { + name = "${helm_release.monitoring.name}-grafana" + port { number = 80 } + } + } + } + } + } + } + depends_on = [helm_release.load_balancer_controller] +} +# The controller owns the ALB; Terraform reads it after the ingress receives its hostname. +data "aws_lb" "grafana" { + name = var.load_balancer_name + depends_on = [kubernetes_ingress_v1.grafana] +} +resource "aws_route53_record" "grafana" { + zone_id = var.hosted_zone_id + name = var.grafana_hostname + type = "A" + alias { + name = data.aws_lb.grafana.dns_name + zone_id = data.aws_lb.grafana.zone_id + evaluate_target_health = true + } +} diff --git a/modules/observability/main.tf b/modules/observability/main.tf new file mode 100644 index 0000000..b01864b --- /dev/null +++ b/modules/observability/main.tf @@ -0,0 +1,75 @@ +locals { + labels = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) +} +resource "helm_release" "monitoring" { + name = "monitoring" + namespace = var.namespace + create_namespace = false + repository = "https://prometheus-community.github.io/helm-charts" + chart = "kube-prometheus-stack" + version = var.chart_version + atomic = true + cleanup_on_fail = true + timeout = 900 + wait = true + depends_on = [kubernetes_namespace_v1.monitoring] + values = [yamlencode({ + commonLabels = local.labels + # EKS does not expose these managed control-plane scrape targets. + kubeEtcd = { enabled = false } + kubeControllerManager = { enabled = false } + kubeScheduler = { enabled = false } + kubeProxy = { enabled = false } + defaultRules = { rules = { etcd = false, kubeControllerManager = false, kubeSchedulerAlerting = false, kubeSchedulerRecording = false, kubeProxy = false } } + alertmanager = { enabled = false } + grafana = { + # Grafana's chart generates its own password; no password in tfvars or Helm values. + adminPassword = null + "grafana.ini" = { + server = { domain = var.grafana_hostname, root_url = "https://${var.grafana_hostname}" } + users = { allow_sign_up = false } + "auth.anonymous" = { enabled = false } + security = { cookie_secure = true, cookie_samesite = "lax" } + } + service = { type = "ClusterIP" } + persistence = { enabled = true, storageClassName = kubernetes_storage_class_v1.monitoring.metadata[0].name, size = var.grafana_storage_size } + resources = { requests = { cpu = "100m", memory = "128Mi" }, limits = { memory = "512Mi" } } + podLabels = local.labels + } + prometheus = { + service = { type = "ClusterIP" } + prometheusSpec = { + retention = var.prometheus_retention + retentionSize = "${floor(tonumber(trimsuffix(var.prometheus_storage_size, "Gi")) * 0.8)}GB" + externalLabels = local.labels + podMetadata = { labels = local.labels } + resources = { requests = { cpu = "200m", memory = var.prometheus_memory_request }, limits = { memory = var.prometheus_memory_limit } } + storageSpec = { volumeClaimTemplate = { spec = { + storageClassName = kubernetes_storage_class_v1.monitoring.metadata[0].name + accessModes = ["ReadWriteOnce"] + resources = { requests = { storage = var.prometheus_storage_size } } + } } } + } + } + })] +} + +resource "kubernetes_namespace_v1" "monitoring" { + metadata { + name = var.namespace + labels = local.labels + } +} +resource "kubernetes_storage_class_v1" "monitoring" { + metadata { + name = "monitoring-gp3" + labels = local.labels + } + storage_provisioner = "ebs.csi.aws.com" + reclaim_policy = "Delete" + volume_binding_mode = "WaitForFirstConsumer" + allow_volume_expansion = true + parameters = merge({ type = "gp3", encrypted = "true" }, { + for i, k in sort(keys(local.labels)) : "tagSpecification_${i + 1}" => "${k}=${local.labels[k]}" + }) +} diff --git a/modules/observability/outputs.tf b/modules/observability/outputs.tf new file mode 100644 index 0000000..4d55ed5 --- /dev/null +++ b/modules/observability/outputs.tf @@ -0,0 +1,28 @@ +output "namespace" { + description = "Monitoring namespace." + value = helm_release.monitoring.namespace +} + +output "release_name" { + description = "Helm release name." + value = helm_release.monitoring.name +} + +output "grafana_service" { + description = "Grafana ClusterIP service for port forwarding." + value = "${helm_release.monitoring.name}-grafana" +} + +output "grafana_secret" { + description = "Secret holding the generated Grafana admin password." + value = "${helm_release.monitoring.name}-grafana" +} + +output "grafana_url" { + description = "Public Grafana HTTPS URL, with authentication required." + value = "https://${aws_route53_record.grafana.name}" +} +output "load_balancer_dns_name" { + description = "Controller-managed ALB DNS name." + value = data.aws_lb.grafana.dns_name +} diff --git a/modules/observability/variables.tf b/modules/observability/variables.tf new file mode 100644 index 0000000..ae5faa6 --- /dev/null +++ b/modules/observability/variables.tf @@ -0,0 +1,95 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "chart_version" { + description = "Pinned kube-prometheus-stack chart version." + type = string +} + +variable "namespace" { + description = "Kubernetes namespace for monitoring." + type = string +} + +variable "prometheus_retention" { + description = "Prometheus local retention duration." + type = string +} + +variable "prometheus_storage_size" { + description = "Prometheus gp3 PVC capacity." + type = string +} + +variable "grafana_storage_size" { + description = "Grafana gp3 PVC capacity." + type = string +} + +variable "prometheus_memory_request" { + description = "Prometheus memory request in Kubernetes units." + type = string +} + +variable "prometheus_memory_limit" { + description = "Prometheus memory limit in Kubernetes units." + type = string +} + +variable "region" { + description = "AWS region, supplied explicitly to the ALB controller." + type = string +} + +variable "cluster_name" { + description = "EKS cluster name for the ALB controller." + type = string +} + +variable "vpc_id" { + description = "Cluster VPC for ALB reconciliation." + type = string +} + +variable "public_subnet_ids" { + description = "Public subnets for an internet-facing ALB in at least two zones." + type = list(string) +} + +variable "controller_role_arn" { + description = "IRSA role for the AWS Load Balancer Controller." + type = string +} + +variable "controller_chart_version" { + description = "Pinned AWS Load Balancer Controller chart version." + type = string +} + +variable "hosted_zone_id" { + description = "Existing public Route 53 hosted zone ID." + type = string +} + +variable "grafana_hostname" { + description = "Public Grafana hostname covered by the ACM certificate." + type = string +} + +variable "load_balancer_name" { + description = "Unique ALB name, at most 32 characters." + type = string +} + +variable "ingress_cidrs" { + description = "Client IPv4 CIDRs allowed to connect to Grafana." + type = list(string) +} diff --git a/modules/observability/versions.tf b/modules/observability/versions.tf new file mode 100644 index 0000000..fc563ab --- /dev/null +++ b/modules/observability/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { source = "hashicorp/aws", version = "~> 6.67.0" } + kubernetes = { source = "hashicorp/kubernetes", version = "~> 2.38.0" } + helm = { source = "hashicorp/helm", version = "~> 3.3.0" } + } +} diff --git a/modules/pipeline/main.tf b/modules/pipeline/main.tf new file mode 100644 index 0000000..bbdd650 --- /dev/null +++ b/modules/pipeline/main.tf @@ -0,0 +1,133 @@ +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} +locals { + tags = merge(var.tags, { environment = var.environment, "managed-by" = "terraform", project = "eks-platform-reference" }) + cluster_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-*" +} +data "aws_iam_policy_document" "trust" { + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [var.oidc_provider_arn] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = ["repo:${var.github_repository}:environment:${var.github_environment}"] + } + } +} +resource "aws_iam_role" "this" { + name = var.role_name + assume_role_policy = data.aws_iam_policy_document.trust.json + max_session_duration = 3600 + tags = local.tags +} +data "aws_iam_policy_document" "permissions" { + statement { + sid = "StateBucket" + actions = ["s3:ListBucket"] + resources = [var.state_bucket_arn] + condition { + test = "StringLike" + variable = "s3:prefix" + values = ["environments/", "${var.state_prefix}*"] + } + } + statement { + sid = "StateObjects" + actions = var.read_only ? ["s3:GetObject"] : ["s3:GetObject", "s3:PutObject"] + resources = ["${var.state_bucket_arn}/${var.state_prefix}*"] + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "StateLockFiles" + actions = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"] + resources = ["${var.state_bucket_arn}/${var.state_prefix}*.tflock"] + } + } + statement { + sid = "RegionalServices" + actions = var.read_only ? ["ec2:Describe*", "eks:Describe*", "eks:List*", "logs:Describe*", "logs:ListTagsForResource", "autoscaling:Describe*", "elasticloadbalancing:Describe*", "acm:DescribeCertificate", "acm:ListCertificates", "acm:ListTagsForCertificate"] : ["ec2:*", "eks:*", "logs:CreateLogGroup", "logs:DeleteLogGroup", "logs:PutRetentionPolicy", "logs:DeleteRetentionPolicy", "logs:Describe*", "logs:ListTagsForResource", "logs:TagResource", "logs:UntagResource", "autoscaling:Describe*", "autoscaling:CreateOrUpdateTags", "autoscaling:DeleteTags", "elasticloadbalancing:Describe*", "acm:RequestCertificate", "acm:DeleteCertificate", "acm:DescribeCertificate", "acm:ListCertificates", "acm:ListTagsForCertificate", "acm:AddTagsToCertificate", "acm:RemoveTagsFromCertificate"] + resources = ["*"] + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [var.region] + } + } + statement { + sid = "DNSRead" + actions = ["route53:ListHostedZones", "route53:ListHostedZonesByName"] + resources = ["*"] + } + statement { + sid = "DNSZone" + actions = var.read_only ? ["route53:GetHostedZone", "route53:ListResourceRecordSets", "route53:ListTagsForResource"] : ["route53:GetHostedZone", "route53:ListResourceRecordSets", "route53:ListTagsForResource", "route53:ChangeResourceRecordSets"] + resources = ["arn:${data.aws_partition.current.partition}:route53:::hostedzone/${var.hosted_zone_id}"] + } + statement { + sid = "DNSChangeStatus" + actions = ["route53:GetChange"] + resources = ["arn:${data.aws_partition.current.partition}:route53:::change/*"] + } + statement { + sid = "IAMRead" + actions = ["iam:Get*", "iam:List*"] + resources = ["*"] + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "ClusterRoles" + actions = ["iam:CreateRole", "iam:DeleteRole", "iam:UpdateAssumeRolePolicy", "iam:AttachRolePolicy", "iam:DetachRolePolicy", "iam:TagRole", "iam:UntagRole", "iam:PutRolePolicy", "iam:DeleteRolePolicy"] + resources = [local.cluster_role_arn] + } + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "PassClusterRoles" + actions = ["iam:PassRole"] + resources = [local.cluster_role_arn] + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["eks.amazonaws.com", "ec2.amazonaws.com"] + } + } + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "ClusterOIDC" + actions = ["iam:CreateOpenIDConnectProvider", "iam:DeleteOpenIDConnectProvider", "iam:TagOpenIDConnectProvider", "iam:UntagOpenIDConnectProvider", "iam:UpdateOpenIDConnectProviderThumbprint"] + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${var.region}.amazonaws.com/id/*"] + } + } + dynamic "statement" { + for_each = var.read_only ? [] : [1] + content { + sid = "EKSServiceLinkedRoles" + actions = ["iam:CreateServiceLinkedRole"] + resources = ["*"] + condition { + test = "StringEquals" + variable = "iam:AWSServiceName" + values = ["eks.amazonaws.com", "eks-nodegroup.amazonaws.com", "autoscaling.amazonaws.com"] + } + } + } +} +resource "aws_iam_role_policy" "this" { + name = "terraform" + role = aws_iam_role.this.id + policy = data.aws_iam_policy_document.permissions.json +} diff --git a/modules/pipeline/outputs.tf b/modules/pipeline/outputs.tf new file mode 100644 index 0000000..4a8a992 --- /dev/null +++ b/modules/pipeline/outputs.tf @@ -0,0 +1,9 @@ +output "role_arn" { + description = "Role ARN for the GitHub environment AWS_ROLE_ARN variable." + value = aws_iam_role.this.arn +} + +output "oidc_subject" { + description = "Exact accepted GitHub OIDC subject." + value = "repo:${var.github_repository}:environment:${var.github_environment}" +} diff --git a/modules/pipeline/variables.tf b/modules/pipeline/variables.tf new file mode 100644 index 0000000..77c335b --- /dev/null +++ b/modules/pipeline/variables.tf @@ -0,0 +1,60 @@ +variable "environment" { + description = "Environment used for resource tags and naming." + type = string +} + +variable "tags" { + description = "Additional resource tags; required ownership tags take precedence." + type = map(string) + default = {} +} + +variable "github_repository" { + description = "Repository portion of GitHub's OIDC sub_claim_prefix, without repo:. Include immutable owner/repository IDs when GitHub reports them." + type = string +} + +variable "github_environment" { + description = "Exact GitHub environment in the OIDC subject." + type = string +} + +variable "oidc_provider_arn" { + description = "Shared account-level GitHub OIDC provider ARN." + type = string +} + +variable "role_name" { + description = "IAM role name for this environment and permission level." + type = string +} + +variable "cluster_name" { + description = "Cluster whose IAM role prefix the deploy role may manage." + type = string +} + +variable "region" { + description = "AWS region in which service operations are permitted." + type = string +} + +variable "state_bucket_arn" { + description = "State bucket ARN." + type = string +} + +variable "state_prefix" { + description = "Environment-specific state key prefix including trailing slash." + type = string +} + +variable "read_only" { + description = "Create a plan role without AWS write permissions." + type = bool +} + +variable "hosted_zone_id" { + description = "Existing Route 53 zone in which certificates and ingress records are managed." + type = string +} diff --git a/modules/pipeline/versions.tf b/modules/pipeline/versions.tf new file mode 100644 index 0000000..2a1568d --- /dev/null +++ b/modules/pipeline/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.16.0, < 2.0.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.67.0" + } + } +} diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100755 index 0000000..f5ad676 --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Capture and audit this reference's AWS resources. All AWS calls are read-only.""" +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +PROJECT = "eks-platform-reference" + + +def aws(region, service, operation, missing_ok=False, **options): + command = ["aws", service, operation, "--region", region, "--output", "json", "--no-cli-pager"] + for key, value in options.items(): + command += ["--" + key.replace("_", "-"), json.dumps(value) if isinstance(value, (list, dict)) else str(value)] + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode: + # Missing resources are expected during audit; denied or failed calls are not proof of deletion. + missing_errors = { + ("eks", "describe-cluster"): "ResourceNotFoundException", + ("iam", "get-role"): "NoSuchEntity", + ("acm", "describe-certificate"): "ResourceNotFoundException", + ("elbv2", "describe-load-balancers"): "LoadBalancerNotFound", + ("elbv2", "describe-target-groups"): "TargetGroupNotFound", + } + missing_error = missing_errors.get((service, operation)) + if missing_ok and missing_error and f"({missing_error})" in result.stderr: + return {} + raise RuntimeError(f"{service} {operation}: {result.stderr.strip()}") + return json.loads(result.stdout) + + +def owned(tags, environment): + values = {tag["Key"]: tag["Value"] for tag in tags or []} + return values.get("project") == PROJECT and values.get("environment") == environment + + +def collect(region, environment, cluster_name): + resources = set() + + def add(kind, identifier): + resources.add((kind, identifier)) + + filters = [{"Name": "tag:project", "Values": [PROJECT]}, {"Name": "tag:environment", "Values": [environment]}] + cluster = aws(region, "eks", "describe-cluster", missing_ok=True, name=cluster_name).get("cluster") + if cluster: + add("eks-cluster", cluster["name"]) + for name in aws(region, "eks", "list-nodegroups", cluster_name=cluster_name)["nodegroups"]: + add("eks-nodegroup", name) + for name in aws(region, "eks", "list-addons", cluster_name=cluster_name)["addons"]: + add("eks-addon", name) + + vpcs = aws(region, "ec2", "describe-vpcs", filters=filters)["Vpcs"] + vpc_ids = {vpc["VpcId"] for vpc in vpcs} + if cluster: + vpc_ids.add(cluster["resourcesVpcConfig"]["vpcId"]) + subnet_ids = set() + for vpc_id in vpc_ids: + add("vpc", vpc_id) + scoped = [{"Name": "vpc-id", "Values": [vpc_id]}] + for operation, response_key, id_key, kind in [ + ("describe-subnets", "Subnets", "SubnetId", "subnet"), + ("describe-route-tables", "RouteTables", "RouteTableId", "route-table"), + ("describe-security-groups", "SecurityGroups", "GroupId", "security-group"), + ("describe-network-interfaces", "NetworkInterfaces", "NetworkInterfaceId", "network-interface"), + ("describe-vpc-endpoints", "VpcEndpoints", "VpcEndpointId", "vpc-endpoint"), + ]: + for item in aws(region, "ec2", operation, filters=scoped)[response_key]: + add(kind, item[id_key]) + if kind == "subnet": + subnet_ids.add(item[id_key]) + for gateway in aws(region, "ec2", "describe-nat-gateways", filter=scoped)["NatGateways"]: + if gateway["State"] != "deleted": + add("nat-gateway", gateway["NatGatewayId"]) + for address in gateway["NatGatewayAddresses"]: + if address.get("AllocationId"): + add("elastic-ip", address["AllocationId"]) + for gateway in aws(region, "ec2", "describe-internet-gateways", filters=[{"Name": "attachment.vpc-id", "Values": [vpc_id]}])["InternetGateways"]: + add("internet-gateway", gateway["InternetGatewayId"]) + for reservation in aws(region, "ec2", "describe-instances", filters=scoped)["Reservations"]: + for instance in reservation["Instances"]: + if instance["State"]["Name"] != "terminated": + add("instance", instance["InstanceId"]) + for mapping in instance.get("BlockDeviceMappings", []): + if mapping.get("Ebs"): + add("ebs-volume", mapping["Ebs"]["VolumeId"]) + + for group in aws(region, "autoscaling", "describe-auto-scaling-groups")["AutoScalingGroups"]: + group_subnets = set(group.get("VPCZoneIdentifier", "").split(",")) + if group_subnets & subnet_ids or owned(group.get("Tags"), environment): + add("auto-scaling-group", group["AutoScalingGroupName"]) + + # These objects can outlive their VPC or cluster. Scan tags independently. + for operation, response_key, id_key, kind in [ + ("describe-addresses", "Addresses", "AllocationId", "elastic-ip"), + ("describe-volumes", "Volumes", "VolumeId", "ebs-volume"), + ("describe-launch-templates", "LaunchTemplates", "LaunchTemplateId", "launch-template"), + ]: + for item in aws(region, "ec2", operation, filters=filters)[response_key]: + add(kind, item[id_key]) + for snapshot in aws(region, "ec2", "describe-snapshots", owner_ids=["self"], filters=filters)["Snapshots"]: + add("ebs-snapshot", snapshot["SnapshotId"]) + for operation, response_key, id_key, kind in [ + ("describe-load-balancers", "LoadBalancers", "LoadBalancerArn", "load-balancer"), + ("describe-target-groups", "TargetGroups", "TargetGroupArn", "target-group"), + ]: + items = aws(region, "elbv2", operation)[response_key] + arns = [item[id_key] for item in items] + tags_by_arn = {} + for start in range(0, len(arns), 20): + response = aws(region, "elbv2", "describe-tags", resource_arns=arns[start:start + 20]) + tags_by_arn.update({entry["ResourceArn"]: entry["Tags"] for entry in response["TagDescriptions"]}) + for item in items: + identifier = item[id_key] + if item.get("VpcId") in vpc_ids or owned(tags_by_arn[identifier], environment): + add(kind, identifier) + for group in aws(region, "logs", "describe-log-groups", log_group_name_prefix=f"/aws/eks/{cluster_name}/")["logGroups"]: + add("log-group", group["logGroupName"]) + for role in aws(region, "iam", "list-roles")["Roles"]: + if role["RoleName"].startswith(cluster_name + "-"): + add("iam-role", role["RoleName"]) + for provider in aws(region, "iam", "list-open-id-connect-providers")["OpenIDConnectProviderList"]: + info = aws(region, "iam", "get-open-id-connect-provider", open_id_connect_provider_arn=provider["Arn"]) + if info["Url"].startswith(f"oidc.eks.{region}.") and owned(info.get("Tags"), environment): + add("irsa-provider", provider["Arn"]) + for certificate in aws(region, "acm", "list-certificates")["CertificateSummaryList"]: + arn = certificate["CertificateArn"] + tags = aws(region, "acm", "list-tags-for-certificate", certificate_arn=arn)["Tags"] + if owned(tags, environment): + add("acm-certificate", arn) + return resources + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["snapshot", "audit"]) + parser.add_argument("--region", required=True) + parser.add_argument("--environment", required=True, choices=["dev", "prod"]) + parser.add_argument("--cluster-name", required=True) + parser.add_argument("--snapshot", required=True, type=Path) + parser.add_argument("--report", type=Path) + args = parser.parse_args() + account = aws(args.region, "sts", "get-caller-identity")["Account"] + metadata = {"account_id": account, "region": args.region, "environment": args.environment, "cluster_name": args.cluster_name} + if args.mode == "snapshot": + current = collect(args.region, args.environment, args.cluster_name) + payload = dict(metadata, captured_at=datetime.now(timezone.utc).isoformat(), resources=[{"type": kind, "id": identifier} for kind, identifier in sorted(current)]) + args.snapshot.parent.mkdir(parents=True, exist_ok=True) + args.snapshot.write_text(json.dumps(payload, indent=2) + "\n") + print(f"Captured {len(current)} resources in {args.snapshot}") + return 0 + previous = json.loads(args.snapshot.read_text()) + if any(previous[key] != value for key, value in metadata.items()): + raise RuntimeError("Snapshot account, region, environment or cluster does not match this audit") + current = collect(args.region, args.environment, args.cluster_name) + captured = {(item["type"], item["id"]) for item in previous["resources"]} + # Check exact captured objects with their describe APIs even after all ownership tags are removed. + remaining = set(current) + for kind, identifier in captured - current: + if exists(args.region, kind, identifier): + remaining.add((kind, identifier)) + report = dict(metadata, audited_at=datetime.now(timezone.utc).isoformat(), remaining=[{"type": kind, "id": identifier} for kind, identifier in sorted(remaining)]) + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n") + for kind, identifier in sorted(remaining): + print(f"REMAINS {kind}: {identifier}") + if remaining: + print(f"Audit failed: {len(remaining)} resources remain.") + return 1 + print("Environment resource audit passed. Check DNS, Terraform state, and shared resources using docs/cleanup.md.") + return 0 + + +def exists(region, kind, identifier): + # Filter-based EC2 describes return an empty collection for absent objects. + ec2 = { + "vpc": ("describe-vpcs", "Vpcs", "vpc-id"), + "subnet": ("describe-subnets", "Subnets", "subnet-id"), + "route-table": ("describe-route-tables", "RouteTables", "route-table-id"), + "security-group": ("describe-security-groups", "SecurityGroups", "group-id"), + "network-interface": ("describe-network-interfaces", "NetworkInterfaces", "network-interface-id"), + "vpc-endpoint": ("describe-vpc-endpoints", "VpcEndpoints", "vpc-endpoint-id"), + "internet-gateway": ("describe-internet-gateways", "InternetGateways", "internet-gateway-id"), + "elastic-ip": ("describe-addresses", "Addresses", "allocation-id"), + "ebs-volume": ("describe-volumes", "Volumes", "volume-id"), + "ebs-snapshot": ("describe-snapshots", "Snapshots", "snapshot-id"), + } + if kind in ec2: + operation, key, filter_name = ec2[kind] + return bool(aws(region, "ec2", operation, filters=[{"Name": filter_name, "Values": [identifier]}])[key]) + if kind == "auto-scaling-group": + return bool(aws(region, "autoscaling", "describe-auto-scaling-groups", auto_scaling_group_names=[identifier])["AutoScalingGroups"]) + if kind == "launch-template": + return any(template["LaunchTemplateId"] == identifier for template in aws(region, "ec2", "describe-launch-templates")["LaunchTemplates"]) + if kind == "instance": + response = aws(region, "ec2", "describe-instances", filters=[{"Name": "instance-id", "Values": [identifier]}]) + return any(instance["State"]["Name"] != "terminated" for reservation in response["Reservations"] for instance in reservation["Instances"]) + if kind == "nat-gateway": + return any(gateway["State"] != "deleted" for gateway in aws(region, "ec2", "describe-nat-gateways", filter=[{"Name": "nat-gateway-id", "Values": [identifier]}])["NatGateways"]) + if kind in {"eks-nodegroup", "eks-addon"}: + # Absence of the cluster already proves its managed nodegroups and add-ons are gone. + # Instances, volumes and ENIs are checked independently. + return False + if kind == "eks-cluster": + return bool(aws(region, "eks", "describe-cluster", missing_ok=True, name=identifier)) + if kind == "log-group": + return any(group["logGroupName"] == identifier for group in aws(region, "logs", "describe-log-groups", log_group_name_prefix=identifier)["logGroups"]) + if kind == "iam-role": + return bool(aws(region, "iam", "get-role", missing_ok=True, role_name=identifier)) + if kind == "irsa-provider": + return any(provider["Arn"] == identifier for provider in aws(region, "iam", "list-open-id-connect-providers")["OpenIDConnectProviderList"]) + if kind == "acm-certificate": + return bool(aws(region, "acm", "describe-certificate", missing_ok=True, certificate_arn=identifier)) + if kind in {"load-balancer", "target-group"}: + if kind == "load-balancer": + return bool(aws(region, "elbv2", "describe-load-balancers", missing_ok=True, load_balancer_arns=[identifier]).get("LoadBalancers")) + return bool(aws(region, "elbv2", "describe-target-groups", missing_ok=True, target_group_arns=[identifier]).get("TargetGroups")) + raise RuntimeError(f"No deletion check for resource type {kind}") + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (RuntimeError, OSError, KeyError, ValueError) as error: + print(f"Audit incomplete: {error}", file=sys.stderr) + sys.exit(2) diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100755 index 0000000..1546c34 --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +terraform fmt -check -recursive +for root in bootstrap environments environments/observability; do + ( + cd "$root" + terraform init -backend=false -input=false -lockfile=readonly + if [[ "$root" == bootstrap ]]; then + terraform validate + else + for workspace in dev prod; do + TF_WORKSPACE="$workspace" terraform validate + done + fi + ) +done diff --git a/scripts/comment-plan.sh b/scripts/comment-plan.sh new file mode 100755 index 0000000..8df9232 --- /dev/null +++ b/scripts/comment-plan.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${TARGET:?}" "${GITHUB_REPOSITORY:?}" "${PR_NUMBER:?}" "${HEAD_SHA:?}" \ + "${GITHUB_SERVER_URL:?}" "${GITHUB_RUN_ID:?}" "${PLAN_JOB_STATUS:?}" "${PLAN_REPORT:?}" "${GH_TOKEN:?}" +case "$TARGET" in dev|prod) ;; *) echo "Invalid environment" >&2; exit 1 ;; esac +directory=$(mktemp -d) +trap 'rm -rf "$directory"' EXIT +marker="" +{ + printf '%s\n### Terraform plan: %s\n\n' "$marker" "$TARGET" + printf 'Job result: **%s**. Commit: `%s`. [Workflow run](%s/%s/actions/runs/%s).\n\n' \ + "$PLAN_JOB_STATUS" "$HEAD_SHA" "$GITHUB_SERVER_URL" "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" + if [[ -s "$PLAN_REPORT" ]]; then + cat "$PLAN_REPORT" + printf '\nReplacements also count toward Add and Destroy. Counts describe managed resources; output-only changes are not included.\n' + printf '\nA deferred monitoring plan requires an existing cluster. Check the workflow logs for any failure or incomplete results.\n' + else + printf 'No plan results were produced. Check the workflow logs.\n' + fi +} > "$directory/comment.md" +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + cat "$directory/comment.md" >> "$GITHUB_STEP_SUMMARY" +fi +jq -n --rawfile body "$directory/comment.md" '{body: $body}' > "$directory/payload.json" +comments=$(gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments") +comment_id=$(jq -r --arg marker "$marker" ' + [.[][] | select(.user.login == "github-actions[bot]" and ((.body // "") | startswith($marker)))] | + sort_by(.id) | last | .id // empty +' <<< "$comments") +if [[ -n "$comment_id" ]]; then + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$comment_id" --input "$directory/payload.json" --silent +else + gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --input "$directory/payload.json" --silent +fi diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..b7e2fd0 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +: "${TARGET:?}" "${EVENT:?}" "${OPERATION:?}" "${AWS_REGION:?}" "${STATE_BUCKET:?}" +case "$TARGET" in dev|prod) ;; *) echo "Invalid environment" >&2; exit 1 ;; esac +case "$EVENT:$OPERATION" in + pull_request:plan|workflow_dispatch:plan|workflow_dispatch:apply) ;; + *) echo "Invalid event and operation combination" >&2; exit 1 ;; +esac +export TF_WORKSPACE="$TARGET" +if [[ -n "${PLAN_REPORT:-}" ]]; then + printf '| Root | Result | Add | Change | Destroy | Replace |\n| --- | --- | ---: | ---: | ---: | ---: |\n' > "$PLAN_REPORT" +fi +report_status() { + if [[ -n "${PLAN_REPORT:-}" ]]; then + printf '| %s | %s | — | — | — | — |\n' "$1" "$2" >> "$PLAN_REPORT" + fi +} +cd environments +# Read cluster details from the reviewed plan rather than a second naming convention. +init_root() { + terraform init -input=false -lockfile=readonly \ + -backend-config="bucket=$STATE_BUCKET" \ + -backend-config="region=$AWS_REGION" \ + -backend-config="key=$1.tfstate" \ + -backend-config="workspace_key_prefix=environments" \ + -backend-config="encrypt=true" \ + -backend-config="use_lockfile=true" + if [[ "$OPERATION" == plan ]]; then + env -u TF_WORKSPACE terraform workspace select "$TARGET" + else + env -u TF_WORKSPACE terraform workspace select -or-create "$TARGET" + fi +} +plan_root() { + local root="$1" + local -a options=(-input=false -no-color -out=deploy.tfplan) + if [[ "$OPERATION" == plan ]]; then + # The plan role cannot create locks or mutate infrastructure. Apply replans under a lock. + options+=(-lock=false) + else + options+=(-lock-timeout=5m) + fi + if ! terraform plan "${options[@]}"; then + report_status "$root" Failed + return 1 + fi + if ! plan_json=$(terraform show -json deploy.tfplan); then + report_status "$root" Failed + return 1 + fi + if [[ -n "${PLAN_REPORT:-}" ]]; then + jq -r --arg root "$root" ' + [.resource_changes[]?.change.actions] as $actions | + [$root, "Planned", + ([$actions[] | select(index("create"))] | length), + ([$actions[] | select(index("update"))] | length), + ([$actions[] | select(index("delete"))] | length), + ([$actions[] | select(index("create") and index("delete"))] | length)] | + "| " + (map(tostring) | join(" | ")) + " |" + ' <<< "$plan_json" >> "$PLAN_REPORT" + fi +} +init_root platform +plan_root Platform +if [[ "$OPERATION" == apply ]] && jq -e \ + '.resource_changes // [] | any(.type == "aws_eks_cluster" and (.change.actions | index("delete") != null))' \ + <<< "$plan_json" >/dev/null; then + echo "Cluster deletion or replacement requires monitoring-first cleanup and audit; see docs/cleanup.md before applying." >&2 + exit 1 +fi +cluster_name=$(jq -r '.planned_values.outputs.cluster_name.value' <<< "$plan_json") +cluster_region=$(jq -r '.planned_values.outputs.region.value' <<< "$plan_json") +if [[ "$AWS_REGION" != "$cluster_region" ]]; then + echo "GitHub AWS_REGION must match the Terraform region input" >&2 + exit 1 +fi +if [[ "$OPERATION" == apply ]]; then + terraform apply -input=false -lock-timeout=5m deploy.tfplan +fi +if ! status=$(aws eks describe-cluster --name "$cluster_name" --region "$cluster_region" --query cluster.status --output text 2>&1); then + if [[ "$OPERATION" == plan && "$status" == *ResourceNotFoundException* ]]; then + report_status Monitoring 'Deferred: cluster does not exist' + printf 'Monitoring plan deferred: this cluster does not exist yet. Run a manual apply to create the platform first.\n' + exit 0 + fi + printf '%s\n' "$status" >&2 + exit 1 +fi +if [[ "$status" != ACTIVE ]]; then + printf 'Cluster is %s; monitoring requires an ACTIVE cluster.\n' "$status" >&2 + exit 1 +fi +cd observability +init_root observability +plan_root Monitoring +if [[ "$OPERATION" == apply ]]; then + terraform apply -input=false -lock-timeout=5m deploy.tfplan +fi diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..a3581bf --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +# Synthetic fixtures and overridden AWS data sources make all plans credential-free. +( + cd bootstrap + terraform test -var-file=tests/fixtures.tfvars +) +for target in dev prod; do + ( + cd environments + TF_WORKSPACE="$target" terraform test -var="test_workspace=$target" + cd observability + TF_WORKSPACE="$target" terraform test -var="test_workspace=$target" + ) +done +python3 -m unittest discover -s tests -v diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..f967fec --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,65 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("audit", Path(__file__).parents[1] / "scripts/audit.py") +audit = importlib.util.module_from_spec(spec) +spec.loader.exec_module(audit) + + +class AuditTests(unittest.TestCase): + def test_missing_cluster_is_absent_but_denial_is_incomplete(self): + with patch.object(audit.subprocess, "run", return_value=SimpleNamespace(returncode=255, stderr="An error occurred (ResourceNotFoundException)", stdout="")): + self.assertEqual(audit.aws("test-region", "eks", "describe-cluster", missing_ok=True, name="test"), {}) + with patch.object(audit.subprocess, "run", return_value=SimpleNamespace(returncode=255, stderr="An error occurred (AccessDeniedException)", stdout="")): + with self.assertRaises(RuntimeError): + audit.aws("test-region", "eks", "describe-cluster", missing_ok=True, name="test") + + def test_unexpected_missing_error_is_not_a_pass(self): + with patch.object(audit.subprocess, "run", return_value=SimpleNamespace(returncode=255, stderr="An error occurred (ResourceNotFoundException)", stdout="")): + with self.assertRaises(RuntimeError): + audit.aws("test-region", "ec2", "describe-vpcs", missing_ok=True) + + def test_terminal_instances_and_nat_are_not_leftovers(self): + with patch.object(audit, "aws", return_value={"Reservations": [{"Instances": [{"State": {"Name": "terminated"}}]}]}): + self.assertFalse(audit.exists("test-region", "instance", "i-test")) + with patch.object(audit, "aws", return_value={"NatGateways": [{"State": "deleted"}]}): + self.assertFalse(audit.exists("test-region", "nat-gateway", "nat-test")) + with patch.object(audit, "aws", return_value={"NatGateways": [{"State": "deleting"}]}): + self.assertTrue(audit.exists("test-region", "nat-gateway", "nat-test")) + + def run_audit(self, directory, remaining, captured_exists, account="test-account"): + snapshot = Path(directory) / "before.json" + snapshot.write_text(json.dumps({"account_id": "test-account", "region": "test-region", "environment": "dev", "cluster_name": "test-cluster", "resources": [{"type": "ebs-volume", "id": "vol-test"}]})) + argv = ["audit.py", "audit", "--environment", "dev", "--region", "test-region", "--cluster-name", "test-cluster", "--snapshot", str(snapshot)] + with patch.object(audit.sys, "argv", argv), patch.object(audit, "aws", return_value={"Account": account}), patch.object(audit, "collect", return_value=remaining) as collect, patch.object(audit, "exists", return_value=captured_exists), patch("builtins.print"): + result = audit.main() + return result, collect + + def test_untagged_captured_volume_still_fails_audit(self): + with tempfile.TemporaryDirectory() as directory: + result, _ = self.run_audit(directory, set(), True) + self.assertEqual(result, 1) + + def test_no_resources_passes(self): + with tempfile.TemporaryDirectory() as directory: + result, _ = self.run_audit(directory, set(), False) + self.assertEqual(result, 0) + + def test_wrong_account_is_rejected_before_resource_inventory(self): + with tempfile.TemporaryDirectory() as directory: + snapshot = Path(directory) / "before.json" + snapshot.write_text(json.dumps({"account_id": "right-account", "region": "test-region", "environment": "dev", "cluster_name": "test-cluster", "resources": []})) + argv = ["audit.py", "audit", "--environment", "dev", "--region", "test-region", "--cluster-name", "test-cluster", "--snapshot", str(snapshot)] + with patch.object(audit.sys, "argv", argv), patch.object(audit, "aws", return_value={"Account": "wrong-account"}), patch.object(audit, "collect") as collect: + with self.assertRaises(RuntimeError): + audit.main() + collect.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_deploy.py b/tests/test_deploy.py new file mode 100644 index 0000000..4407713 --- /dev/null +++ b/tests/test_deploy.py @@ -0,0 +1,166 @@ +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).parents[1] + +TERRAFORM = '''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +with open(os.environ['DEPLOY_TEST_LOG'], 'a') as log: + log.write(json.dumps({'args': args, 'workspace': os.environ.get('TF_WORKSPACE'), 'directory': str(Path.cwd().relative_to(os.environ['DEPLOY_TEST_REPO']))}) + '\\n') +operation = next(arg for arg in args if not arg.startswith('-')) +if operation == 'workspace' and os.environ.get('TF_WORKSPACE'): + print('Workspace selection blocked by TF_WORKSPACE', file=sys.stderr) + sys.exit(1) +stage = str(Path.cwd().relative_to(os.environ['DEPLOY_TEST_REPO'])) + ':' + operation +if stage == os.environ.get('DEPLOY_TEST_FAIL_STAGE'): + print('Synthetic Terraform failure', file=sys.stderr) + sys.exit(1) +if operation == 'show': + print(json.dumps({'planned_values': {'outputs': {'cluster_name': {'value': 'test-cluster'}, 'region': {'value': 'test-region'}}}, 'resource_changes': [{'type': 'aws_eks_cluster', 'change': {'actions': json.loads(os.environ['DEPLOY_TEST_CLUSTER_ACTIONS']), 'after': {'password': 'sensitive-test-value'}}}]})) +''' +AWS = '''#!/usr/bin/env python3 +import os, sys +result = os.environ.get('DEPLOY_TEST_AWS_RESULT', 'ACTIVE') +if result == 'ACTIVE': + print('ACTIVE') +else: + print('An error occurred (' + result + ')', file=sys.stderr) + sys.exit(255) +''' + + +class DeployTests(unittest.TestCase): + def deploy(self, event, aws_result="ACTIVE", region="test-region", operation=None, target="dev", cluster_actions=None, fail_stage=""): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + log = directory / "commands.jsonl" + for name, content in [("terraform", TERRAFORM), ("aws", AWS)]: + executable = directory / name + executable.write_text(content) + executable.chmod(0o755) + if operation is None: + operation = "plan" if event == "pull_request" else "apply" + environment = dict(os.environ, PATH=str(directory) + os.pathsep + os.environ["PATH"], DEPLOY_TEST_LOG=str(log), DEPLOY_TEST_REPO=str(REPO), DEPLOY_TEST_AWS_RESULT=aws_result, DEPLOY_TEST_CLUSTER_ACTIONS=json.dumps(cluster_actions or ["create"]), DEPLOY_TEST_FAIL_STAGE=fail_stage, TARGET=target, EVENT=event, OPERATION=operation, AWS_REGION=region, STATE_BUCKET="test-state-bucket") + report = directory / "plan.md" + environment["PLAN_REPORT"] = str(report) + result = subprocess.run(["bash", str(REPO / "scripts/deploy.sh")], cwd=directory, env=environment, capture_output=True, text=True) + result.plan_report = report.read_text() if report.exists() else "" + commands = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + return result, commands + + def test_pr_plans_both_roots_without_apply_or_remote_locks(self): + result, commands = self.deploy("pull_request") + self.assertEqual(result.returncode, 0, result.stderr) + plans = [command for command in commands if command["args"][0] == "plan"] + self.assertEqual(len(plans), 2) + self.assertTrue(all("-lock=false" in command["args"] for command in plans)) + self.assertFalse(any(arg.startswith("-var-file") for command in plans for arg in command["args"])) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + self.assertTrue(all(command["workspace"] is None for command in commands if command["args"][0] == "workspace")) + + def test_apply_plans_platform_before_monitoring(self): + result, commands = self.deploy("workflow_dispatch", operation="apply") + self.assertEqual(result.returncode, 0, result.stderr) + changes = [(command["directory"], command["args"][0]) for command in commands if command["args"][0] in {"plan", "apply"}] + self.assertEqual(changes, [("environments", "plan"), ("environments", "apply"), ("environments/observability", "plan"), ("environments/observability", "apply")]) + self.assertTrue(all(command["workspace"] == "dev" for command in commands if command["args"][0] in {"plan", "apply"})) + + def test_only_missing_cluster_defers_monitoring(self): + result, commands = self.deploy("pull_request", "ResourceNotFoundException") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Monitoring plan deferred", result.stdout) + self.assertEqual(sum(command["args"][0] == "plan" for command in commands), 1) + denied, _ = self.deploy("pull_request", "AccessDeniedException") + self.assertNotEqual(denied.returncode, 0) + self.assertNotIn("Monitoring plan deferred", denied.stdout) + + def test_region_mismatch_prevents_apply(self): + result, commands = self.deploy("workflow_dispatch", operation="apply", region="wrong-region") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + + def test_manual_prod_plan_is_read_only(self): + result, commands = self.deploy("workflow_dispatch", operation="plan", target="prod", cluster_actions=["delete", "create"]) + self.assertEqual(result.returncode, 0, result.stderr) + plans = [command for command in commands if command["args"][0] == "plan"] + self.assertEqual(len(plans), 2) + self.assertTrue(all("-lock=false" in command["args"] for command in plans)) + self.assertTrue(all(command["workspace"] == "prod" for command in plans)) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + + def test_manual_apply_retains_state_locking(self): + result, commands = self.deploy("workflow_dispatch", operation="apply") + self.assertEqual(result.returncode, 0, result.stderr) + changes = [command for command in commands if command["args"][0] in {"plan", "apply"}] + self.assertEqual([command["args"][0] for command in changes], ["plan", "apply", "plan", "apply"]) + self.assertTrue(all("-lock-timeout=5m" in command["args"] for command in changes)) + + def test_manual_plan_defers_missing_cluster(self): + result, commands = self.deploy("workflow_dispatch", "ResourceNotFoundException", operation="plan") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sum(command["args"][0] == "plan" for command in commands), 1) + + def test_pr_cannot_select_apply(self): + result, commands = self.deploy("pull_request", operation="apply") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(commands) + + def test_cluster_deletion_or_replacement_prevents_apply(self): + for actions in (["delete"], ["delete", "create"], ["create", "delete"]): + with self.subTest(actions=actions): + result, commands = self.deploy("workflow_dispatch", operation="apply", cluster_actions=actions) + self.assertNotEqual(result.returncode, 0) + self.assertIn("monitoring", result.stderr) + self.assertFalse(any(command["args"][0] == "apply" for command in commands)) + + def test_in_place_cluster_update_is_allowed(self): + result, commands = self.deploy("workflow_dispatch", operation="apply", cluster_actions=["update"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sum(command["args"][0] == "apply" for command in commands), 2) + + def test_failed_terraform_stage_stops_later_stages(self): + stages = [("environments", "plan"), ("environments", "apply"), ("environments/observability", "plan"), ("environments/observability", "apply")] + for index, (directory, operation) in enumerate(stages): + with self.subTest(directory=directory, operation=operation): + result, commands = self.deploy("workflow_dispatch", operation="apply", fail_stage=directory + ":" + operation) + self.assertNotEqual(result.returncode, 0) + observed = [(command["directory"], command["args"][0]) for command in commands if command["args"][0] in {"plan", "apply"}] + self.assertEqual(observed, stages[:index + 1]) + + def test_push_cannot_deploy(self): + result, commands = self.deploy("push", operation="apply") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(commands) + + def test_plan_report_counts_replacements_without_values(self): + result, _ = self.deploy("pull_request", cluster_actions=["delete", "create"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("| Platform | Planned | 1 | 0 | 1 | 1 |", result.plan_report) + self.assertIn("| Monitoring | Planned | 1 | 0 | 1 | 1 |", result.plan_report) + self.assertNotIn("sensitive-test-value", result.plan_report) + self.assertNotIn("test-cluster", result.plan_report) + + def test_plan_report_marks_deferred_and_failed_results(self): + deferred, _ = self.deploy("pull_request", "ResourceNotFoundException") + self.assertIn("| Monitoring | Deferred: cluster does not exist |", deferred.plan_report) + failed, _ = self.deploy("pull_request", fail_stage="environments/observability:plan") + self.assertNotEqual(failed.returncode, 0) + self.assertIn("| Platform | Planned | 1 | 0 | 0 | 0 |", failed.plan_report) + self.assertIn("| Monitoring | Failed |", failed.plan_report) + + def test_noop_and_update_counts(self): + for actions, counts in [(["no-op"], "0 | 0 | 0 | 0"), (["update"], "0 | 1 | 0 | 0")]: + with self.subTest(actions=actions): + result, _ = self.deploy("pull_request", cluster_actions=actions) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("| Platform | Planned | " + counts + " |", result.plan_report) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_plan_comment.py b/tests/test_plan_comment.py new file mode 100644 index 0000000..0c6d891 --- /dev/null +++ b/tests/test_plan_comment.py @@ -0,0 +1,71 @@ +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).parents[1] +GH = '''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +record = {'args': args} +if '--input' in args: + record['payload'] = json.loads(Path(args[args.index('--input') + 1]).read_text()) +with open(os.environ['COMMENT_TEST_LOG'], 'a') as log: + log.write(json.dumps(record) + '\\n') +if '--method' not in args: + print(os.environ['COMMENT_TEST_COMMENTS']) +''' + + +class PlanCommentTests(unittest.TestCase): + def comment(self, comments=None, report="| Platform | Planned | 1 | 0 | 0 | 0 |", status="success"): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + executable = directory / "gh" + executable.write_text(GH) + executable.chmod(0o755) + report_path = directory / "plan.md" + if report is not None: + report_path.write_text(report) + log = directory / "commands.jsonl" + environment = dict(os.environ, PATH=str(directory) + os.pathsep + os.environ["PATH"], + COMMENT_TEST_LOG=str(log), COMMENT_TEST_COMMENTS=json.dumps(comments or [[]]), + TARGET="dev", GITHUB_REPOSITORY="owner/repo", PR_NUMBER="1", HEAD_SHA="abc123", + GITHUB_SERVER_URL="https://github.com", GITHUB_RUN_ID="42", GH_TOKEN="test-token", + PLAN_REPORT=str(report_path), PLAN_JOB_STATUS=status, GITHUB_STEP_SUMMARY=str(directory / "summary.md")) + result = subprocess.run(["bash", str(REPO / "scripts/comment-plan.sh")], env=environment, capture_output=True, text=True) + calls = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual(result.returncode, 0, result.stderr) + return calls + + def test_creates_comment_with_counts_commit_and_run_link(self): + calls = self.comment() + self.assertIn("--paginate", calls[0]["args"]) + self.assertIn("POST", calls[1]["args"]) + self.assertIn("repos/owner/repo/issues/1/comments", calls[1]["args"]) + body = calls[1]["payload"]["body"] + self.assertTrue(body.startswith("")) + self.assertIn("abc123", body) + self.assertIn("https://github.com/owner/repo/actions/runs/42", body) + self.assertIn("| Platform | Planned | 1 | 0 | 0 | 0 |", body) + + def test_updates_only_matching_bot_comment_across_pages(self): + comments = [[{"id": 3, "user": {"login": "github-actions[bot]"}, "body": " old"}], + [{"id": 4, "user": {"login": "human"}, "body": ""}, + {"id": 5, "user": {"login": "github-actions[bot]"}, "body": ""}]] + calls = self.comment(comments) + self.assertIn("PATCH", calls[1]["args"]) + self.assertIn("repos/owner/repo/issues/comments/3", calls[1]["args"]) + + def test_failure_before_plan_produces_clear_comment(self): + calls = self.comment(report=None, status="failure") + body = calls[1]["payload"]["body"] + self.assertIn("**failure**", body) + self.assertIn("No plan results were produced", body) + + +if __name__ == "__main__": + unittest.main()