diff --git a/docs/configurations/deployments/helpers.md b/docs/configurations/deployments/helpers.md index 0b636505..976dd303 100644 --- a/docs/configurations/deployments/helpers.md +++ b/docs/configurations/deployments/helpers.md @@ -17,20 +17,20 @@ ______________________________________________________________________ ## Helper Modules -| Module | Example directory | Use when | -| -------------------------------------- | ---------------------------------------------------------- | -------------------------------------------------------------------- | -| `modules/helpers/aws_config_recording` | `environments/prod/regions/eu-west-1/aws_config_recording` | You need configuration history for selected AWS resource types. | -| `modules/helpers/ami_policy` | `environments/prod/ami_policy` | Forge owns AMI usage policy support. | -| `modules/helpers/ami_sharing` | `environments/prod/regions/eu-west-1/ami_sharing` | Runner AMIs must be shared across accounts or regions. | -| `modules/helpers/cloud_custodian` | `environments/prod/cloud_custodian` | You run cleanup or governance policies from Forge. | -| `modules/helpers/cloud_formation` | `environments/prod/cloud_formation` | Integrations need CloudFormation admin/execution roles. | -| `modules/helpers/dedicated_mac_hosts` | `environments/prod/regions/eu-west-1/dedicated_mac_hosts` | Forge owns EC2 Mac Dedicated Host capacity. | -| `modules/helpers/ecr` | `environments/prod/regions/eu-west-1/ecr` | Forge owns ECR repositories for runner or helper images. | -| `modules/helpers/forge_subscription` | `environments/prod/forge_subscription` | Tenant accounts need Forge-managed IAM, Packer, S3, or ECR access. | -| `modules/helpers/microvm` | `environments/prod/regions/eu-west-1/microvm` | Provides regional MicroVM publishing/runtime foundations and egress. | -| `modules/helpers/opt_in_regions` | `environments/prod/opt_in_regions` | You need to enable AWS opt-in regions before regional deploys. | -| `modules/helpers/service_linked_roles` | `environments/prod/service_linked_roles` | The account needs the EC2 Spot service-linked role. | -| `modules/helpers/storage` | `environments/prod/storage` | Forge owns operational S3 buckets for logs, artifacts, or templates. | +| Module | Example directory | Use when | +| -------------------------------------- | ---------------------------------------------------------- | --------------------------------------------------------------------- | +| `modules/helpers/aws_config_recording` | `environments/prod/regions/eu-west-1/aws_config_recording` | You need AWS Config history with a Splunk-compatible S3/SQS producer. | +| `modules/helpers/ami_policy` | `environments/prod/ami_policy` | Forge owns AMI usage policy support. | +| `modules/helpers/ami_sharing` | `environments/prod/regions/eu-west-1/ami_sharing` | Runner AMIs must be shared across accounts or regions. | +| `modules/helpers/cloud_custodian` | `environments/prod/cloud_custodian` | You run cleanup or governance policies from Forge. | +| `modules/helpers/cloud_formation` | `environments/prod/cloud_formation` | Integrations need CloudFormation admin/execution roles. | +| `modules/helpers/dedicated_mac_hosts` | `environments/prod/regions/eu-west-1/dedicated_mac_hosts` | Forge owns EC2 Mac Dedicated Host capacity. | +| `modules/helpers/ecr` | `environments/prod/regions/eu-west-1/ecr` | Forge owns ECR repositories for runner or helper images. | +| `modules/helpers/forge_subscription` | `environments/prod/forge_subscription` | Tenant accounts need Forge-managed IAM, Packer, S3, or ECR access. | +| `modules/helpers/microvm` | `environments/prod/regions/eu-west-1/microvm` | Provides regional MicroVM publishing/runtime foundations and egress. | +| `modules/helpers/opt_in_regions` | `environments/prod/opt_in_regions` | You need to enable AWS opt-in regions before regional deploys. | +| `modules/helpers/service_linked_roles` | `environments/prod/service_linked_roles` | The account needs the EC2 Spot service-linked role. | +| `modules/helpers/storage` | `environments/prod/storage` | Forge owns operational S3 buckets for logs, artifacts, or templates. | ______________________________________________________________________ @@ -78,6 +78,20 @@ cd examples/deployments/helpers/terragrunt/environments/prod/regions/eu-west-1/a cd examples/deployments/helpers/terragrunt/environments/prod/regions/eu-west-1/dedicated_mac_hosts ``` +After applying `aws_config_recording`, inspect the values for a Splunk Data +Manager custom S3 input: + +```bash +cd examples/deployments/helpers/terragrunt/environments/prod/regions/eu-west-1/aws_config_recording +terragrunt output -json splunk_s3_logs +terragrunt output -json splunk_s3_logs | jq -r '.sqs.url' +terragrunt output -json splunk_s3_logs | jq -r '.bucket_arn' +``` + +These outputs establish only the AWS producer contract. Configure the custom +Splunk parser and validate a real AWS Config `.json.gz` object through to +indexed events before calling the integration operational. + Plan the full helper environment only after the individual helper plans are understood: @@ -92,8 +106,7 @@ ______________________________________________________________________ 1. `service_linked_roles` when the account needs the EC2 Spot service-linked role. 1. `opt_in_regions` before deploying into opt-in regions. -1. `storage` before `aws_config_recording` so the long-term bucket is available as its delivery dependency. -1. `aws_config_recording` before allocating Dedicated Hosts so their full history is captured. +1. `aws_config_recording` before allocating Dedicated Hosts so their full history is captured; it creates its own delivery bucket and notification queue. 1. `dedicated_mac_hosts` only after reviewing Mac host cost and minimum allocation periods. 1. `ecr` if Forge builds or stores runner/helper images. 1. `ami_sharing` if runner AMIs live in a central image account. diff --git a/docs/configurations/deployments/integrations.md b/docs/configurations/deployments/integrations.md index 2eb7937a..19be3576 100644 --- a/docs/configurations/deployments/integrations.md +++ b/docs/configurations/deployments/integrations.md @@ -46,6 +46,40 @@ environments/prod/regions/eu-west-1/splunk_dependency_monitor Deploy it once per Forge AWS region, after `splunk_o11y_conf_shared` and `splunk_secrets`. +The Splunk Cloud Data Manager example includes a disabled custom S3 input for +AWS Config history. The helper and integration examples have separate +Terragrunt roots, so transfer the producer values explicitly: + +1. Apply the regional helper: + + ```bash + cd examples/deployments/helpers/terragrunt/environments/prod/regions/eu-west-1/aws_config_recording + terragrunt apply + terragrunt output -json splunk_s3_logs + terragrunt output -json splunk_s3_logs | jq -r '.sqs.url' + terragrunt output -json splunk_s3_logs | jq -r '.bucket_arn' + ``` + +1. In + `examples/deployments/integrations/terragrunt/environments/prod/splunk_cloud_data_manager/config.yml`, + copy `sqs.url` into `sqs_urls` and `bucket_arn` into + `s3_bucket_patterns` for `forge-aws-config-prod`. + +1. Leave `kms_key_arns` empty because `bucket_kms_key_arn` is `null`; the + helper bucket uses SSE-S3. + +1. Confirm the helper resources and Data Manager deployment use the same AWS + account, then set the input's `enabled` value to `true`. + +1. Apply `environments/prod/splunk_cloud_data_manager_common`, followed by + `environments/prod/splunk_cloud_data_manager`. + +The input name, index, IAM region, and `source_type` remain owned by the +integration example. Before enabling it, deploy the custom Splunk parser and +validate one real AWS Config `.json.gz` object from S3 notification through to +the expected indexed events. The AWS producer shape alone is not end-to-end +validation. + Skip both pages if your observability stack is not Splunk. ______________________________________________________________________ @@ -58,6 +92,7 @@ ______________________________________________________________________ | `environments/prod/_environment_wide_settings/_environment.yml` | AWS account, default region, AWS profile, and remote state. | | `environments/prod/github_webhook_relay_destination/config.yml` | Destination EventBridge and reader role settings. | | `environments/prod/github_webhook_relay_destination_receivers/config.yml` | Receiver bundle configuration. | +| `environments/prod/splunk_cloud_data_manager/config.yml` | Splunk URL, index, AWS Config source type, and input name. | | `environments/prod/regions/eu-west-1/teleport/config.yml` | Teleport cluster, namespace, chart, and EKS access values. | | `release_versions.yml` | Integration module sources, refs, and `module_path` values. | @@ -96,3 +131,5 @@ ______________________________________________________________________ - Splunk modules can depend on `splunk_secrets`, helper buckets, and CloudFormation helper roles. Use externally managed resources if your company already provides them. +- The AWS Config SQS queue is dedicated to Data Manager. Do not attach another + consumer that can remove its notifications before Splunk reads them. diff --git a/docs/integrations/splunk.md b/docs/integrations/splunk.md index 4dafa831..a3290653 100644 --- a/docs/integrations/splunk.md +++ b/docs/integrations/splunk.md @@ -114,6 +114,30 @@ Splunk contract. The full secret list is in [Splunk Secrets](splunk-secrets.md). +## AWS Config Custom S3 Input + +The `aws_config_recording` helper creates an AWS-side producer compatible with +the Data Manager `s3-custom-logs` fields. Apply it and inspect the handoff +values: + +```bash +cd examples/deployments/helpers/terragrunt/environments/prod/regions/eu-west-1/aws_config_recording +terragrunt apply +terragrunt output -json splunk_s3_logs +terragrunt output -json splunk_s3_logs | jq -r '.sqs.url' +terragrunt output -json splunk_s3_logs | jq -r '.bucket_arn' +``` + +Copy the SQS URL and bucket ARN into the disabled `forge-aws-config-prod` +custom-log item, leave `kms_key_arns` empty because the bucket uses SSE-S3, +and confirm Data Manager runs in the same AWS account. The example source type +is `forgecicd:aws:config:s3`. + +This is producer compatibility, not proof of successful ingestion. Configure +the source type's parser for the AWS Config `configurationItems` envelope, +then validate a real `.json.gz` object from S3 notification through queue +consumption to the expected indexed events before enabling production use. + ## Apply Secrets First ```bash diff --git a/docs/reference/module-catalog.md b/docs/reference/module-catalog.md index 0b4a0461..1c68e618 100644 --- a/docs/reference/module-catalog.md +++ b/docs/reference/module-catalog.md @@ -29,20 +29,20 @@ EC2-only deployments can skip `examples/deployments/infra`. ## Helper Modules -| Module | Role | Required? | Example root | Validation | -| -------------------------------------- | ------------------------------------------------------------ | --------- | ------------------------------ | ----------------------------------------------- | -| `modules/helpers/aws_config_recording` | AWS Config history for caller-selected AWS resource types. | Optional | `examples/deployments/helpers` | Recorder is active for the configured types. | -| `modules/helpers/ami_policy` | AMI policy support for approved runner images. | Optional | `examples/deployments/helpers` | Policy plan plus AMI usage review. | -| `modules/helpers/ami_sharing` | Shares runner AMIs across accounts or regions. | Optional | `examples/deployments/helpers` | Target account can describe and launch AMI. | -| `modules/helpers/cloud_custodian` | Cleanup and policy jobs for stale resources. | Optional | `examples/deployments/helpers` | Custodian dry run and scheduled job output. | -| `modules/helpers/cloud_formation` | CloudFormation admin/execution roles for setup paths. | Optional | `examples/deployments/helpers` | Stack role assumption check. | -| `modules/helpers/dedicated_mac_hosts` | Mac Dedicated Hosts, host groups, and license configuration. | Optional | `examples/deployments/helpers` | Host allocation and group membership review. | -| `modules/helpers/ecr` | ECR repositories for runner and CI helper containers. | Optional | `examples/deployments/helpers` | Push/pull smoke for configured repositories. | -| `modules/helpers/forge_subscription` | Tenant-side access for ForgeMT jobs and artifacts. | Optional | `examples/deployments/helpers` | Tenant role can access intended S3/ECR/secrets. | -| `modules/helpers/microvm` | Regional MicroVM publishing and VPC egress foundation. | Optional | `examples/deployments/helpers` | Artifact plan and connector `ACTIVE` state. | -| `modules/helpers/opt_in_regions` | Enables AWS opt-in regions. | Sometimes | `examples/deployments/helpers` | AWS account region status is enabled. | -| `modules/helpers/service_linked_roles` | Creates the EC2 Spot service-linked role. | Sometimes | `examples/deployments/helpers` | Role exists before EC2 runner launch. | -| `modules/helpers/storage` | S3 buckets for artifacts, templates, logs, and integrations. | Optional | `examples/deployments/helpers` | Bucket policy, encryption, and access checks. | +| Module | Role | Required? | Example root | Validation | +| -------------------------------------- | ------------------------------------------------------------ | --------- | ------------------------------ | ------------------------------------------------ | +| `modules/helpers/aws_config_recording` | AWS Config history with a Data Manager-compatible producer. | Optional | `examples/deployments/helpers` | Recorder, queue, DLQ, and `.json.gz` event plan. | +| `modules/helpers/ami_policy` | AMI policy support for approved runner images. | Optional | `examples/deployments/helpers` | Policy plan plus AMI usage review. | +| `modules/helpers/ami_sharing` | Shares runner AMIs across accounts or regions. | Optional | `examples/deployments/helpers` | Target account can describe and launch AMI. | +| `modules/helpers/cloud_custodian` | Cleanup and policy jobs for stale resources. | Optional | `examples/deployments/helpers` | Custodian dry run and scheduled job output. | +| `modules/helpers/cloud_formation` | CloudFormation admin/execution roles for setup paths. | Optional | `examples/deployments/helpers` | Stack role assumption check. | +| `modules/helpers/dedicated_mac_hosts` | Mac Dedicated Hosts, host groups, and license configuration. | Optional | `examples/deployments/helpers` | Host allocation and group membership review. | +| `modules/helpers/ecr` | ECR repositories for runner and CI helper containers. | Optional | `examples/deployments/helpers` | Push/pull smoke for configured repositories. | +| `modules/helpers/forge_subscription` | Tenant-side access for ForgeMT jobs and artifacts. | Optional | `examples/deployments/helpers` | Tenant role can access intended S3/ECR/secrets. | +| `modules/helpers/microvm` | Regional MicroVM publishing and VPC egress foundation. | Optional | `examples/deployments/helpers` | Artifact plan and connector `ACTIVE` state. | +| `modules/helpers/opt_in_regions` | Enables AWS opt-in regions. | Sometimes | `examples/deployments/helpers` | AWS account region status is enabled. | +| `modules/helpers/service_linked_roles` | Creates the EC2 Spot service-linked role. | Sometimes | `examples/deployments/helpers` | Role exists before EC2 runner launch. | +| `modules/helpers/storage` | S3 buckets for artifacts, templates, logs, and integrations. | Optional | `examples/deployments/helpers` | Bucket policy, encryption, and access checks. | Helpers are not runtime platform modules. Deploy them only when ForgeMT owns that operating concern. diff --git a/docs/reference/module-dependencies.md b/docs/reference/module-dependencies.md index c73bf75c..5fa24e3c 100644 --- a/docs/reference/module-dependencies.md +++ b/docs/reference/module-dependencies.md @@ -46,8 +46,8 @@ ______________________________________________________________________ | Module | Deploy before platform? | Why | | -------------------------------------- | ------------------------------ | ---------------------------------------------------------------- | -| `modules/helpers/storage` | Before AWS Config recording | Supplies the long-term S3 delivery bucket used by the example. | -| `modules/helpers/aws_config_recording` | Before recorded resources | Captures configuration history from resource creation. | +| `modules/helpers/storage` | Optional | Creates shared artifact buckets; it has no Config delivery role. | +| `modules/helpers/aws_config_recording` | Before recorded resources | Creates its delivery bucket and queue, then captures history. | | `modules/helpers/opt_in_regions` | Yes, for opt-in regions | Regional resources cannot deploy until the region is enabled. | | `modules/helpers/service_linked_roles` | Usually, for EC2 Spot | Some accounts need the EC2 Spot service-linked role first. | | `modules/helpers/ami_policy` | Optional | Account policy support for AMI usage. | @@ -73,6 +73,9 @@ ______________________________________________________________________ events to it. - Deploy `splunk_secrets` and `splunk_o11y_conf_shared` before each regional `splunk_dependency_monitor` instance. +- For AWS Config ingestion, apply the regional `aws_config_recording` helper, + copy its `splunk_s3_logs` queue URL and bucket ARN into the disabled Data + Manager custom S3 input, validate parsing, and then enable the input. ______________________________________________________________________ diff --git a/docs/reference/module-layout.md b/docs/reference/module-layout.md index baebf27c..675a7c67 100644 --- a/docs/reference/module-layout.md +++ b/docs/reference/module-layout.md @@ -48,7 +48,7 @@ ______________________________________________________________________ | Module | Purpose | | -------------------------------------- | ------------------------------------------------------------------------------------------- | -| `modules/helpers/aws_config_recording` | Records caller-selected AWS resource types with AWS Config. | +| `modules/helpers/aws_config_recording` | Records selected AWS resources with a Data Manager-compatible S3/SQS producer. | | `modules/helpers/ami_policy` | IAM policy support for Forge AMI usage. | | `modules/helpers/ami_sharing` | Shares runner AMIs across accounts or regions. | | `modules/helpers/cloud_custodian` | Runs cleanup and policy automation. | diff --git a/examples/deployments/helpers/terragrunt/_global_settings/aws_config_recording.hcl b/examples/deployments/helpers/terragrunt/_global_settings/aws_config_recording.hcl index bb4647d2..d76f0643 100644 --- a/examples/deployments/helpers/terragrunt/_global_settings/aws_config_recording.hcl +++ b/examples/deployments/helpers/terragrunt/_global_settings/aws_config_recording.hcl @@ -7,7 +7,6 @@ locals { env_data = read_terragrunt_config(find_in_parent_folders("_environment_wide_settings/_environment.hcl")) default_aws_profile = local.env_data.locals.default_aws_profile - aws_account_id = local.env_data.locals.aws_account_id region_data = read_terragrunt_config(find_in_parent_folders("_region_wide_settings/_region.hcl")) region = local.region_data.locals.region_aws @@ -30,17 +29,10 @@ locals { } } -dependencies { - paths = [ - find_in_parent_folders("storage") - ] -} - inputs = { aws_profile = local.default_aws_profile aws_region = local.region - delivery_bucket_name = "${local.aws_account_id}-long-term-storage" recorded_resource_types = local.config_recording_data.locals.recorded_resource_types tags = local.tags diff --git a/examples/deployments/integrations/terragrunt/environments/prod/splunk_cloud_data_manager/config.yml b/examples/deployments/integrations/terragrunt/environments/prod/splunk_cloud_data_manager/config.yml index 577177d0..ab356806 100644 --- a/examples/deployments/integrations/terragrunt/environments/prod/splunk_cloud_data_manager/config.yml +++ b/examples/deployments/integrations/terragrunt/environments/prod/splunk_cloud_data_manager/config.yml @@ -32,6 +32,17 @@ s3_logs_config: sqs_urls: [] s3_bucket_patterns: [] kms_key_arns: [] + - enabled: false + name: forge-aws-config-prod + iam_region: eu-west-1 + index: forge-prod-index + source_type: forgecicd:aws:config:s3 + # Copy splunk_s3_logs.sqs.url from the aws_config_recording output. + sqs_urls: [] + # Copy splunk_s3_logs.bucket_arn from the aws_config_recording output. + s3_bucket_patterns: [] + # splunk_s3_logs.bucket_kms_key_arn is null because the bucket uses SSE-S3. + kms_key_arns: [] ct-logs: [] s3-access-logs: [] elb-access-logs: [] diff --git a/examples/templates/integrations/splunk_cloud_data_manager/config.yml b/examples/templates/integrations/splunk_cloud_data_manager/config.yml index 747b6a30..73450ea0 100644 --- a/examples/templates/integrations/splunk_cloud_data_manager/config.yml +++ b/examples/templates/integrations/splunk_cloud_data_manager/config.yml @@ -34,6 +34,17 @@ s3_logs_config: s3_bucket_patterns: - # e.g., forge-runner-logs-*; use * when no narrower prefix exists kms_key_arns: [] # required; use [] when no KMS keys apply + - enabled: false # enable only after the producer output and Splunk parser are validated + name: forge-aws-config-prod + iam_region: + index: + source_type: forgecicd:aws:config:s3 + # Copy splunk_s3_logs.sqs.url from the aws_config_recording output. + sqs_urls: [] + # Copy splunk_s3_logs.bucket_arn from the aws_config_recording output. + s3_bucket_patterns: [] + # splunk_s3_logs.bucket_kms_key_arn is null because the bucket uses SSE-S3. + kms_key_arns: [] ct-logs: [] # AWS CloudTrail inputs s3-access-logs: [] # Amazon S3 access-log inputs elb-access-logs: [] # Elastic Load Balancing access-log inputs diff --git a/modules/helpers/aws_config_recording/README.md b/modules/helpers/aws_config_recording/README.md index 5e66ce3d..e8ef8b15 100644 --- a/modules/helpers/aws_config_recording/README.md +++ b/modules/helpers/aws_config_recording/README.md @@ -1,12 +1,15 @@ # AWS Config Recording -This module enables continuous AWS Config recording for caller-selected AWS resource types. +This module enables continuous AWS Config recording for caller-selected AWS resource types and, by default, creates the regional delivery path for those records. ## What It Manages - A configuration recorder for the resource types supplied through `recorded_resource_types`. - An IAM role with the AWS managed Config recorder policy. -- A delivery channel targeting the existing bucket supplied through `delivery_bucket_name`. +- A private, encrypted, versioned S3 bucket for AWS Config snapshots and history. +- An encrypted SQS queue and dead-letter queue for regional `.json.gz` Config object notifications. +- Least-privilege bucket and queue policies for AWS Config delivery and S3 notifications. +- A delivery channel targeting the managed bucket. - An enabled recorder status. ## Operational Notes @@ -14,8 +17,49 @@ This module enables continuous AWS Config recording for caller-selected AWS reso - Deploy one instance of this module per AWS account and Region where configuration recording is required. - The account and Region must not already have a customer-managed configuration recorder or delivery channel with conflicting names. - The default IAM role name includes `aws_region`, preventing global IAM name collisions when the module is deployed in multiple Regions. -- The delivery bucket can be in another Region or account. -- Manage the bucket and its AWS Config permissions externally; this module never creates or changes the bucket or its policy. +- The default bucket name is `-forge-aws-config-` and the default queue name is `-forge-aws-config-events-`. +- Notifications match keys under `AWSLogs//Config//` that end in `.json.gz`. This includes Config history and snapshot objects; S3 key filters cannot select the nested `ConfigHistory` segment alone. +- SQS messages contain S3 event metadata. Consumers retrieve the compressed object from S3 and should tolerate duplicate or out-of-order notifications. They must also detect and ignore the initial S3 `s3:TestEvent`, which has no Config object and uses a different payload shape. +- The main queue uses a five-minute visibility timeout for Splunk SQS-based S3 ingestion and moves a message to the encrypted dead-letter queue after five failed receives. Both queues retain messages for 14 days. +- To preserve an existing central delivery bucket, set `delivery_bucket_name`. In that compatibility mode this module creates no S3 bucket, SQS queue, or bucket notification, and the external bucket owner must manage delivery permissions and notifications. +- Removing an existing `delivery_bucket_name` switches future delivery to the new regional bucket; historical objects remain in the external bucket and are not copied by this module. +- Setting `delivery_bucket_name` after using managed mode plans removal of the managed bucket and queue. Because `force_destroy` is disabled, first retain or migrate the bucket in state, or explicitly remove every object version and delete marker before applying the ownership change. +- S3 notification configuration is atomic. Do not add a second Terraform owner for notifications on the managed bucket. + +## Splunk Data Manager + +This module is only the AWS-side producer. Its managed S3, SQS, and policy contract is compatible with a Splunk Data Manager custom S3 input, but the module does not configure Splunk or prove that AWS Config objects are parsed and indexed correctly. + +When both modules are composed in one Terraform root, use the +`splunk_s3_logs` output directly with the Forge `splunk_cloud_data_manager` +module's `s3-custom-logs` input: + +```hcl +s3_logs_config = { + s3-custom-logs = [{ + enabled = true + name = "forge-aws-config-prod" + iam_region = "eu-west-1" + index = "forge-prod-index" + source_type = "forgecicd:aws:config:s3" + sqs_urls = [module.aws_config_recording.splunk_s3_logs.sqs.url] + s3_bucket_patterns = [module.aws_config_recording.splunk_s3_logs.bucket_arn] + kms_key_arns = [] + }] +} +``` + +The Forge Terragrunt helper and integration examples are independent deployment +roots. For those examples, apply the helper, run +`terragrunt output -json splunk_s3_logs`, and copy `sqs.url` and `bucket_arn` +into the disabled AWS Config item in the Data Manager `config.yml` before +enabling it. + +- Keep the queue dedicated to one Data Manager input. Competing consumers can delete notifications before Splunk reads them. +- `kms_key_arns` is empty because the managed bucket uses SSE-S3 (`AES256`), not SSE-KMS. +- AWS Config files wrap records in a `configurationItems` array. Before enabling ingestion, deploy and validate the custom source type's Splunk parsing so each configuration item has the intended event boundary and timestamp. Merely naming the source type `forgecicd:aws:config:s3` does not select the Splunk AWS add-on's Config decoder through this custom-log path. +- Validate the complete path with a real AWS Config `.json.gz` object: confirm its S3 notification reaches the queue, Data Manager consumes it, and the expected individual events appear in the selected index. Queue activity alone is not end-to-end success. +- Notifications apply only to newly created objects. Existing objects require a separate historical-ingestion or controlled-copy procedure. ## Requirements @@ -29,7 +73,7 @@ This module enables continuous AWS Config recording for caller-selected AWS reso | Name | Version | | ---- | ------- | -| [aws](#provider\_aws) | 6.57.1 | +| [aws](#provider\_aws) | 6.59.0 | ## Modules @@ -44,8 +88,22 @@ No modules. | [aws_config_delivery_channel.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/config_delivery_channel) | resource | | [aws_iam_role.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_s3_bucket.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_lifecycle_configuration.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource | +| [aws_s3_bucket_notification.config_events](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource | +| [aws_s3_bucket_ownership_controls.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource | +| [aws_s3_bucket_policy.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource | +| [aws_s3_bucket_public_access_block.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | +| [aws_s3_bucket_server_side_encryption_configuration.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | +| [aws_s3_bucket_versioning.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource | | [aws_servicecatalogappregistry_application.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicecatalogappregistry_application) | resource | +| [aws_sqs_queue.config_events](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue.config_events_dlq](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue_policy.config_events](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.config_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.config_delivery_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.config_events](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | ## Inputs @@ -55,7 +113,7 @@ No modules. | [aws\_profile](#input\_aws\_profile) | AWS profile to use. | `string` | n/a | yes | | [aws\_region](#input\_aws\_region) | Default AWS region. | `string` | n/a | yes | | [default\_tags](#input\_default\_tags) | A map of tags to apply to resources. | `map(string)` | n/a | yes | -| [delivery\_bucket\_name](#input\_delivery\_bucket\_name) | Name of the existing S3 bucket that receives AWS Config data. The bucket can be in another Region or account. | `string` | n/a | yes | +| [delivery\_bucket\_name](#input\_delivery\_bucket\_name) | Optional name of an existing S3 delivery bucket. When null, the module creates a regional bucket and SQS notification queue. | `string` | `null` | no | | [delivery\_channel\_name](#input\_delivery\_channel\_name) | Name of the AWS Config delivery channel. | `string` | `"default"` | no | | [iam\_role\_name](#input\_iam\_role\_name) | Name of the IAM role used by AWS Config. Defaults to forge-aws-config-recorder-. | `string` | `null` | no | | [recorded\_resource\_types](#input\_recorded\_resource\_types) | AWS Config resource types to record, using identifiers such as AWS::EC2::Instance. | `set(string)` | n/a | yes | @@ -67,6 +125,9 @@ No modules. | Name | Description | | ---- | ----------- | | [configuration\_recorder\_name](#output\_configuration\_recorder\_name) | Name of the enabled AWS Config configuration recorder. | +| [delivery\_bucket\_arn](#output\_delivery\_bucket\_arn) | ARN of the S3 bucket receiving AWS Config snapshots and history. | | [delivery\_bucket\_name](#output\_delivery\_bucket\_name) | Name of the S3 bucket receiving AWS Config snapshots and history. | | [recorded\_resource\_types](#output\_recorded\_resource\_types) | AWS resource types recorded by AWS Config. | +| [splunk\_s3\_logs](#output\_splunk\_s3\_logs) | Managed AWS Config S3 and SQS resources for a Splunk Data Manager s3-custom-logs input, or null when an external bucket is used. | +| [sqs](#output\_sqs) | SQS queue receiving managed AWS Config S3 object notifications, or null when an external bucket is used. | diff --git a/modules/helpers/aws_config_recording/data.tf b/modules/helpers/aws_config_recording/data.tf index 2a870afe..771fec19 100644 --- a/modules/helpers/aws_config_recording/data.tf +++ b/modules/helpers/aws_config_recording/data.tf @@ -1,3 +1,5 @@ +data "aws_caller_identity" "current" {} + data "aws_partition" "current" {} data "aws_iam_policy_document" "config_assume_role" { @@ -11,3 +13,114 @@ data "aws_iam_policy_document" "config_assume_role" { } } } + +data "aws_iam_policy_document" "config_delivery_bucket" { + count = local.create_delivery_resources ? 1 : 0 + + statement { + sid = "AWSConfigBucketPermissionsCheck" + effect = "Allow" + actions = [ + "s3:GetBucketAcl", + "s3:ListBucket", + ] + resources = [aws_s3_bucket.config_delivery[0].arn] + + principals { + type = "Service" + identifiers = ["config.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "AWS:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + + condition { + test = "ArnLike" + variable = "AWS:SourceArn" + values = ["arn:${data.aws_partition.current.partition}:config:${var.aws_region}:${data.aws_caller_identity.current.account_id}:*"] + } + } + + statement { + sid = "AWSConfigBucketDelivery" + effect = "Allow" + actions = ["s3:PutObject"] + resources = ["${aws_s3_bucket.config_delivery[0].arn}/AWSLogs/${data.aws_caller_identity.current.account_id}/Config/*"] + + principals { + type = "Service" + identifiers = ["config.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "AWS:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + + condition { + test = "StringEquals" + variable = "s3:x-amz-acl" + values = ["bucket-owner-full-control"] + } + + condition { + test = "ArnLike" + variable = "AWS:SourceArn" + values = ["arn:${data.aws_partition.current.partition}:config:${var.aws_region}:${data.aws_caller_identity.current.account_id}:*"] + } + } + + statement { + sid = "DenyInsecureTransport" + effect = "Deny" + actions = ["s3:*"] + resources = [ + aws_s3_bucket.config_delivery[0].arn, + "${aws_s3_bucket.config_delivery[0].arn}/*", + ] + + principals { + type = "*" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} + +data "aws_iam_policy_document" "config_events" { + count = local.create_delivery_resources ? 1 : 0 + + statement { + sid = "AllowConfigBucketNotifications" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.config_events[0].arn] + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_s3_bucket.config_delivery[0].arn] + } + + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + } +} diff --git a/modules/helpers/aws_config_recording/main.tf b/modules/helpers/aws_config_recording/main.tf index bbcc9d84..db522224 100644 --- a/modules/helpers/aws_config_recording/main.tf +++ b/modules/helpers/aws_config_recording/main.tf @@ -9,6 +9,144 @@ resource "aws_iam_role_policy_attachment" "config" { policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AWS_ConfigRole" } +resource "aws_s3_bucket" "config_delivery" { + count = local.create_delivery_resources ? 1 : 0 + + #checkov:skip=CKV_AWS_145:SSE-S3 is compatible with AWS Config delivery and avoids a KMS key dependency for the archive. + #checkov:skip=CKV_AWS_144:Each recorder writes to a regional bucket; cross-region replication is outside this module's archival contract. + #checkov:skip=CKV_AWS_18:CloudTrail provides control-plane audit records; a separate access-log bucket is not part of this helper. + bucket = local.delivery_bucket_name + force_destroy = false + tags = local.all_security_tags +} + +resource "aws_s3_bucket_ownership_controls" "config_delivery" { + count = local.create_delivery_resources ? 1 : 0 + + # AWS Config delivers objects with the bucket-owner-full-control ACL. + #checkov:skip=CKV2_AWS_65:BucketOwnerPreferred preserves the ACL required by the AWS Config delivery contract. + bucket = aws_s3_bucket.config_delivery[0].id + + rule { + object_ownership = "BucketOwnerPreferred" + } +} + +resource "aws_s3_bucket_versioning" "config_delivery" { + count = local.create_delivery_resources ? 1 : 0 + + bucket = aws_s3_bucket.config_delivery[0].id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "config_delivery" { + count = local.create_delivery_resources ? 1 : 0 + + bucket = aws_s3_bucket.config_delivery[0].id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "config_delivery" { + count = local.create_delivery_resources ? 1 : 0 + + bucket = aws_s3_bucket.config_delivery[0].id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true + skip_destroy = true +} + +resource "aws_s3_bucket_lifecycle_configuration" "config_delivery" { + count = local.create_delivery_resources ? 1 : 0 + + bucket = aws_s3_bucket.config_delivery[0].id + + rule { + id = "abort-incomplete-config-uploads" + status = "Enabled" + + filter {} + + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } + + depends_on = [aws_s3_bucket_versioning.config_delivery] +} + +resource "aws_s3_bucket_policy" "config_delivery" { + count = local.create_delivery_resources ? 1 : 0 + + bucket = aws_s3_bucket.config_delivery[0].id + policy = data.aws_iam_policy_document.config_delivery_bucket[0].json + + depends_on = [ + aws_s3_bucket_ownership_controls.config_delivery, + aws_s3_bucket_public_access_block.config_delivery, + aws_s3_bucket_server_side_encryption_configuration.config_delivery, + aws_s3_bucket_versioning.config_delivery, + ] +} + +resource "aws_sqs_queue" "config_events_dlq" { + count = local.create_delivery_resources ? 1 : 0 + + name = local.config_events_dlq_name + message_retention_seconds = 1209600 # 14 days + sqs_managed_sse_enabled = true + tags = local.all_security_tags +} + +resource "aws_sqs_queue" "config_events" { + count = local.create_delivery_resources ? 1 : 0 + + name = local.config_events_queue_name + message_retention_seconds = 1209600 # 14 days + visibility_timeout_seconds = 300 + sqs_managed_sse_enabled = true + tags = local.all_security_tags + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.config_events_dlq[0].arn + maxReceiveCount = 5 + }) +} + +resource "aws_sqs_queue_policy" "config_events" { + count = local.create_delivery_resources ? 1 : 0 + + queue_url = aws_sqs_queue.config_events[0].url + policy = data.aws_iam_policy_document.config_events[0].json +} + +# S3 notification configuration is atomic. This resource must remain the only +# Terraform owner of notifications for the managed delivery bucket. +resource "aws_s3_bucket_notification" "config_events" { + count = local.create_delivery_resources ? 1 : 0 + + bucket = aws_s3_bucket.config_delivery[0].id + + queue { + id = "aws-config-json-gzip" + queue_arn = aws_sqs_queue.config_events[0].arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "AWSLogs/${data.aws_caller_identity.current.account_id}/Config/${var.aws_region}/" + filter_suffix = ".json.gz" + } + + depends_on = [aws_sqs_queue_policy.config_events] +} + resource "aws_config_configuration_recorder" "this" { #checkov:skip=CKV2_AWS_45:Selective recording is intentional; callers supply recorded_resource_types. #checkov:skip=CKV2_AWS_48:Global resources are recorded only when explicitly selected by the caller. @@ -33,9 +171,14 @@ resource "aws_config_configuration_recorder" "this" { resource "aws_config_delivery_channel" "this" { name = var.delivery_channel_name - s3_bucket_name = var.delivery_bucket_name + s3_bucket_name = local.delivery_bucket_name - depends_on = [aws_config_configuration_recorder.this] + depends_on = [ + aws_config_configuration_recorder.this, + aws_s3_bucket_lifecycle_configuration.config_delivery, + aws_s3_bucket_notification.config_events, + aws_s3_bucket_policy.config_delivery, + ] } resource "aws_config_configuration_recorder_status" "this" { diff --git a/modules/helpers/aws_config_recording/outputs.tf b/modules/helpers/aws_config_recording/outputs.tf index 47e7b27a..66366144 100644 --- a/modules/helpers/aws_config_recording/outputs.tf +++ b/modules/helpers/aws_config_recording/outputs.tf @@ -3,12 +3,41 @@ output "configuration_recorder_name" { value = aws_config_configuration_recorder.this.name } +output "delivery_bucket_arn" { + description = "ARN of the S3 bucket receiving AWS Config snapshots and history." + value = local.create_delivery_resources ? ( + aws_s3_bucket.config_delivery[0].arn + ) : "arn:${data.aws_partition.current.partition}:s3:::${local.delivery_bucket_name}" +} + output "delivery_bucket_name" { description = "Name of the S3 bucket receiving AWS Config snapshots and history." - value = var.delivery_bucket_name + value = local.delivery_bucket_name } output "recorded_resource_types" { description = "AWS resource types recorded by AWS Config." value = var.recorded_resource_types } + +output "sqs" { + description = "SQS queue receiving managed AWS Config S3 object notifications, or null when an external bucket is used." + value = local.create_delivery_resources ? { + arn = aws_sqs_queue.config_events[0].arn + dead_letter_queue_arn = aws_sqs_queue.config_events_dlq[0].arn + url = aws_sqs_queue.config_events[0].url + } : null +} + +output "splunk_s3_logs" { + description = "Managed AWS Config S3 and SQS resources for a Splunk Data Manager s3-custom-logs input, or null when an external bucket is used." + value = local.create_delivery_resources ? { + bucket_arn = aws_s3_bucket.config_delivery[0].arn + bucket_kms_key_arn = null + sqs = { + arn = aws_sqs_queue.config_events[0].arn + dead_letter_queue_arn = aws_sqs_queue.config_events_dlq[0].arn + url = aws_sqs_queue.config_events[0].url + } + } : null +} diff --git a/modules/helpers/aws_config_recording/tags.tf b/modules/helpers/aws_config_recording/tags.tf index 9581aa60..e02338da 100644 --- a/modules/helpers/aws_config_recording/tags.tf +++ b/modules/helpers/aws_config_recording/tags.tf @@ -5,5 +5,12 @@ locals { var.tags, aws_servicecatalogappregistry_application.this.application_tag, ) - iam_role_name = coalesce(var.iam_role_name, "forge-aws-config-recorder-${var.aws_region}") + create_delivery_resources = var.delivery_bucket_name == null + delivery_bucket_name = coalesce( + var.delivery_bucket_name, + "${data.aws_caller_identity.current.account_id}-forge-aws-config-${var.aws_region}", + ) + config_events_queue_name = "${data.aws_caller_identity.current.account_id}-forge-aws-config-events-${var.aws_region}" + config_events_dlq_name = "${data.aws_caller_identity.current.account_id}-forge-aws-config-events-dlq-${var.aws_region}" + iam_role_name = coalesce(var.iam_role_name, "forge-aws-config-recorder-${var.aws_region}") } diff --git a/modules/helpers/aws_config_recording/tests/behavior.tftest.hcl b/modules/helpers/aws_config_recording/tests/behavior.tftest.hcl index 76061df7..19664e3c 100644 --- a/modules/helpers/aws_config_recording/tests/behavior.tftest.hcl +++ b/modules/helpers/aws_config_recording/tests/behavior.tftest.hcl @@ -1,5 +1,14 @@ mock_provider "aws" {} +override_resource { + target = aws_servicecatalogappregistry_application.this + values = { + application_tag = { + "awsApplication" = "arn:aws:resource-groups:eu-west-1:123456789012:group/helpers_aws_config_recording_eu-west-1" + } + } +} + override_resource { target = aws_iam_role.config values = { @@ -7,6 +16,39 @@ override_resource { } } +override_resource { + target = aws_s3_bucket.config_delivery + values = { + arn = "arn:aws:s3:::123456789012-forge-aws-config-eu-west-1" + id = "123456789012-forge-aws-config-eu-west-1" + } +} + +override_resource { + target = aws_sqs_queue.config_events + values = { + arn = "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-eu-west-1" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/123456789012-forge-aws-config-events-eu-west-1" + } +} + +override_resource { + target = aws_sqs_queue.config_events_dlq + values = { + arn = "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-dlq-eu-west-1" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/123456789012-forge-aws-config-events-dlq-eu-west-1" + } +} + +override_data { + target = data.aws_caller_identity.current + values = { + account_id = "123456789012" + arn = "arn:aws:iam::123456789012:user/test" + user_id = "AIDATEST" + } +} + override_data { target = data.aws_partition.current values = { @@ -22,9 +64,8 @@ override_data { } variables { - aws_profile = "test" - aws_region = "eu-west-1" - delivery_bucket_name = "forge-config-123456789012-eu-west-1" + aws_profile = "test" + aws_region = "eu-west-1" recorded_resource_types = [ "AWS::EC2::Instance", "AWS::S3::Bucket", @@ -37,7 +78,7 @@ variables { } } -run "aws_config_recording_contract" { +run "managed_delivery_contract" { command = plan assert { @@ -47,14 +88,69 @@ run "aws_config_recording_contract" { && toset(aws_config_configuration_recorder.this.recording_group[0].resource_types) == toset(["AWS::EC2::Instance", "AWS::S3::Bucket"]) && aws_config_configuration_recorder.this.recording_mode[0].recording_frequency == "CONTINUOUS" && aws_config_configuration_recorder_status.this.is_enabled == true - && aws_config_delivery_channel.this.s3_bucket_name == "forge-config-123456789012-eu-west-1" + && aws_config_delivery_channel.this.s3_bucket_name == "123456789012-forge-aws-config-eu-west-1" + ) + error_message = "AWS Config must continuously record the configured resource types into the managed regional bucket." + } + + assert { + condition = ( + aws_s3_bucket.config_delivery[0].bucket == "123456789012-forge-aws-config-eu-west-1" + && aws_s3_bucket.config_delivery[0].force_destroy == false + && aws_s3_bucket.config_delivery[0].tags.Product == "Forge" + && aws_s3_bucket.config_delivery[0].tags.Env == "test" + && aws_s3_bucket_ownership_controls.config_delivery[0].rule[0].object_ownership == "BucketOwnerPreferred" + && aws_s3_bucket_versioning.config_delivery[0].versioning_configuration[0].status == "Enabled" + && one(one(aws_s3_bucket_server_side_encryption_configuration.config_delivery[0].rule).apply_server_side_encryption_by_default).sse_algorithm == "AES256" + && aws_s3_bucket_public_access_block.config_delivery[0].block_public_acls == true + && aws_s3_bucket_public_access_block.config_delivery[0].block_public_policy == true + && aws_s3_bucket_public_access_block.config_delivery[0].ignore_public_acls == true + && aws_s3_bucket_public_access_block.config_delivery[0].restrict_public_buckets == true + && aws_s3_bucket_lifecycle_configuration.config_delivery[0].rule[0].status == "Enabled" + && aws_s3_bucket_lifecycle_configuration.config_delivery[0].rule[0].abort_incomplete_multipart_upload[0].days_after_initiation == 7 + && length(aws_s3_bucket_policy.config_delivery) == 1 + ) + error_message = "The managed AWS Config bucket must be private, encrypted, versioned, lifecycle-managed, tagged, and protected from destructive removal." + } + + assert { + condition = ( + aws_sqs_queue.config_events[0].name == "123456789012-forge-aws-config-events-eu-west-1" + && aws_sqs_queue.config_events[0].sqs_managed_sse_enabled == true + && aws_sqs_queue.config_events[0].message_retention_seconds == 1209600 + && aws_sqs_queue.config_events[0].visibility_timeout_seconds == 300 + && jsondecode(aws_sqs_queue.config_events[0].redrive_policy).deadLetterTargetArn == "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-dlq-eu-west-1" + && jsondecode(aws_sqs_queue.config_events[0].redrive_policy).maxReceiveCount == 5 + && aws_sqs_queue.config_events_dlq[0].name == "123456789012-forge-aws-config-events-dlq-eu-west-1" + && aws_sqs_queue.config_events_dlq[0].sqs_managed_sse_enabled == true + && aws_sqs_queue.config_events_dlq[0].message_retention_seconds == 1209600 + && length(aws_sqs_queue_policy.config_events) == 1 + && one(aws_s3_bucket_notification.config_events[0].queue).queue_arn == "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-eu-west-1" + && toset(one(aws_s3_bucket_notification.config_events[0].queue).events) == toset(["s3:ObjectCreated:*"]) + && one(aws_s3_bucket_notification.config_events[0].queue).filter_prefix == "AWSLogs/123456789012/Config/eu-west-1/" + && one(aws_s3_bucket_notification.config_events[0].queue).filter_suffix == ".json.gz" ) - error_message = "AWS Config must continuously record the configured resource types and enable the recorder." + error_message = "The managed bucket must notify its encrypted SQS queue for regional AWS Config .json.gz object creation events." } + assert { + condition = ( + output.delivery_bucket_name == "123456789012-forge-aws-config-eu-west-1" + && output.delivery_bucket_arn == "arn:aws:s3:::123456789012-forge-aws-config-eu-west-1" + && output.sqs.arn == "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-eu-west-1" + && output.sqs.dead_letter_queue_arn == "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-dlq-eu-west-1" + && output.sqs.url == "https://sqs.eu-west-1.amazonaws.com/123456789012/123456789012-forge-aws-config-events-eu-west-1" + && output.splunk_s3_logs.bucket_arn == "arn:aws:s3:::123456789012-forge-aws-config-eu-west-1" + && output.splunk_s3_logs.bucket_kms_key_arn == null + && output.splunk_s3_logs.sqs.arn == "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-eu-west-1" + && output.splunk_s3_logs.sqs.dead_letter_queue_arn == "arn:aws:sqs:eu-west-1:123456789012:123456789012-forge-aws-config-events-dlq-eu-west-1" + && output.splunk_s3_logs.sqs.url == "https://sqs.eu-west-1.amazonaws.com/123456789012/123456789012-forge-aws-config-events-eu-west-1" + ) + error_message = "Managed delivery outputs must expose the bucket, SQS, DLQ, and Splunk Data Manager producer contract." + } } -run "uses_input_delivery_bucket" { +run "uses_external_delivery_bucket" { command = plan variables { @@ -62,8 +158,31 @@ run "uses_input_delivery_bucket" { } assert { - condition = aws_config_delivery_channel.this.s3_bucket_name == "central-config-bucket-eu-west-1" - error_message = "AWS Config must use the input delivery bucket without creating or managing S3 resources." + condition = ( + aws_config_delivery_channel.this.s3_bucket_name == "central-config-bucket-eu-west-1" + && output.delivery_bucket_name == "central-config-bucket-eu-west-1" + && output.delivery_bucket_arn == "arn:aws:s3:::central-config-bucket-eu-west-1" + && output.sqs == null + && output.splunk_s3_logs == null + ) + error_message = "AWS Config must preserve the external delivery bucket contract and expose no managed queue." + } + + assert { + condition = ( + length(aws_s3_bucket.config_delivery) == 0 + && length(aws_s3_bucket_ownership_controls.config_delivery) == 0 + && length(aws_s3_bucket_versioning.config_delivery) == 0 + && length(aws_s3_bucket_server_side_encryption_configuration.config_delivery) == 0 + && length(aws_s3_bucket_public_access_block.config_delivery) == 0 + && length(aws_s3_bucket_lifecycle_configuration.config_delivery) == 0 + && length(aws_s3_bucket_policy.config_delivery) == 0 + && length(aws_sqs_queue.config_events_dlq) == 0 + && length(aws_sqs_queue.config_events) == 0 + && length(aws_sqs_queue_policy.config_events) == 0 + && length(aws_s3_bucket_notification.config_events) == 0 + ) + error_message = "External delivery mode must not create or manage S3 or SQS resources." } } diff --git a/modules/helpers/aws_config_recording/tests/interface_contract.tftest.hcl b/modules/helpers/aws_config_recording/tests/interface_contract.tftest.hcl index f0a9cc81..087952e6 100644 --- a/modules/helpers/aws_config_recording/tests/interface_contract.tftest.hcl +++ b/modules/helpers/aws_config_recording/tests/interface_contract.tftest.hcl @@ -20,8 +20,11 @@ run "helpers_aws_config_recording_interface_contract" { ] expected_output_values = [ "configuration_recorder_name", + "delivery_bucket_arn", "delivery_bucket_name", "recorded_resource_types", + "sqs", + "splunk_s3_logs", ] expected_interface_literals = [ "variable \"delivery_bucket_name\"", @@ -31,8 +34,11 @@ run "helpers_aws_config_recording_interface_contract" { "variable \"recorder_name\"", "variable \"delivery_channel_name\"", "output \"configuration_recorder_name\"", + "output \"delivery_bucket_arn\"", "output \"delivery_bucket_name\"", "output \"recorded_resource_types\"", + "output \"sqs\"", + "output \"splunk_s3_logs\"", ] } diff --git a/modules/helpers/aws_config_recording/tests/source_inventory.tftest.hcl b/modules/helpers/aws_config_recording/tests/source_inventory.tftest.hcl index 0b258473..82d014ba 100644 --- a/modules/helpers/aws_config_recording/tests/source_inventory.tftest.hcl +++ b/modules/helpers/aws_config_recording/tests/source_inventory.tftest.hcl @@ -12,6 +12,129 @@ run "helpers_aws_config_recording_source_inventory" { "resource \"aws_config_delivery_channel\" \"this\"", "resource \"aws_config_configuration_recorder_status\" \"this\"", "resource \"aws_iam_role\" \"config\"", + "resource \"aws_s3_bucket\" \"config_delivery\"", + "resource \"aws_s3_bucket_ownership_controls\" \"config_delivery\"", + "resource \"aws_s3_bucket_versioning\" \"config_delivery\"", + "resource \"aws_s3_bucket_server_side_encryption_configuration\" \"config_delivery\"", + "resource \"aws_s3_bucket_public_access_block\" \"config_delivery\"", + "resource \"aws_s3_bucket_lifecycle_configuration\" \"config_delivery\"", + "resource \"aws_s3_bucket_policy\" \"config_delivery\"", + "resource \"aws_sqs_queue\" \"config_events_dlq\"", + "resource \"aws_sqs_queue\" \"config_events\"", + "resource \"aws_sqs_queue_policy\" \"config_events\"", + "resource \"aws_s3_bucket_notification\" \"config_events\"", + "data \"aws_caller_identity\" \"current\"", + "data \"aws_iam_policy_document\" \"config_delivery_bucket\"", + "data \"aws_iam_policy_document\" \"config_events\"", + "identifiers = [\"config.amazonaws.com\"]", + "identifiers = [\"s3.amazonaws.com\"]", + "\"s3:GetBucketAcl\"", + "\"s3:ListBucket\"", + "actions = [\"s3:PutObject\"]", + "actions = [\"sqs:SendMessage\"]", + "resources = [\"$${aws_s3_bucket.config_delivery[0].arn}/AWSLogs/$${data.aws_caller_identity.current.account_id}/Config/*\"]", + "values = [\"arn:$${data.aws_partition.current.partition}:config:$${var.aws_region}:$${data.aws_caller_identity.current.account_id}:*\"]", + "values = [aws_s3_bucket.config_delivery[0].arn]", + "variable = \"AWS:SourceAccount\"", + "variable = \"s3:x-amz-acl\"", + "variable = \"aws:SourceArn\"", + "variable = \"aws:SourceAccount\"", + "variable = \"aws:SecureTransport\"", + "values = [\"bucket-owner-full-control\"]", + "values = [\"false\"]", + "sqs_managed_sse_enabled = true", + "message_retention_seconds = 1209600", + "visibility_timeout_seconds = 300", + "deadLetterTargetArn = aws_sqs_queue.config_events_dlq[0].arn", + "maxReceiveCount = 5", + "events = [\"s3:ObjectCreated:*\"]", + "filter_suffix = \".json.gz\"", + "depends_on = [aws_sqs_queue_policy.config_events]", + join("\n", [ + " statement {", + " sid = \"AWSConfigBucketPermissionsCheck\"", + " effect = \"Allow\"", + " actions = [", + " \"s3:GetBucketAcl\",", + " \"s3:ListBucket\",", + " ]", + " resources = [aws_s3_bucket.config_delivery[0].arn]", + "", + " principals {", + " type = \"Service\"", + " identifiers = [\"config.amazonaws.com\"]", + " }", + "", + " condition {", + " test = \"StringEquals\"", + " variable = \"AWS:SourceAccount\"", + " values = [data.aws_caller_identity.current.account_id]", + " }", + "", + " condition {", + " test = \"ArnLike\"", + " variable = \"AWS:SourceArn\"", + " values = [\"arn:$${data.aws_partition.current.partition}:config:$${var.aws_region}:$${data.aws_caller_identity.current.account_id}:*\"]", + " }", + " }", + ]), + join("\n", [ + " statement {", + " sid = \"AWSConfigBucketDelivery\"", + " effect = \"Allow\"", + " actions = [\"s3:PutObject\"]", + " resources = [\"$${aws_s3_bucket.config_delivery[0].arn}/AWSLogs/$${data.aws_caller_identity.current.account_id}/Config/*\"]", + "", + " principals {", + " type = \"Service\"", + " identifiers = [\"config.amazonaws.com\"]", + " }", + "", + " condition {", + " test = \"StringEquals\"", + " variable = \"AWS:SourceAccount\"", + " values = [data.aws_caller_identity.current.account_id]", + " }", + "", + " condition {", + " test = \"StringEquals\"", + " variable = \"s3:x-amz-acl\"", + " values = [\"bucket-owner-full-control\"]", + " }", + "", + " condition {", + " test = \"ArnLike\"", + " variable = \"AWS:SourceArn\"", + " values = [\"arn:$${data.aws_partition.current.partition}:config:$${var.aws_region}:$${data.aws_caller_identity.current.account_id}:*\"]", + " }", + " }", + ]), + join("\n", [ + " statement {", + " sid = \"AllowConfigBucketNotifications\"", + " effect = \"Allow\"", + "", + " principals {", + " type = \"Service\"", + " identifiers = [\"s3.amazonaws.com\"]", + " }", + "", + " actions = [\"sqs:SendMessage\"]", + " resources = [aws_sqs_queue.config_events[0].arn]", + "", + " condition {", + " test = \"ArnEquals\"", + " variable = \"aws:SourceArn\"", + " values = [aws_s3_bucket.config_delivery[0].arn]", + " }", + "", + " condition {", + " test = \"StringEquals\"", + " variable = \"aws:SourceAccount\"", + " values = [data.aws_caller_identity.current.account_id]", + " }", + " }", + ]), "var.recorded_resource_types", "provider \"aws\"", ] @@ -23,7 +146,7 @@ run "helpers_aws_config_recording_source_inventory" { } assert { - condition = output.expected_literal_count == 6 - error_message = "Source inventory must keep 6 AWS Config Terraform blocks and literals pinned." + condition = output.expected_literal_count == 47 + error_message = "Source inventory must keep 47 AWS Config, managed S3, and Splunk-ready SQS Terraform blocks and exact policy contracts pinned." } } diff --git a/modules/helpers/aws_config_recording/variables.tf b/modules/helpers/aws_config_recording/variables.tf index bb1cf106..b3d3aace 100644 --- a/modules/helpers/aws_config_recording/variables.tf +++ b/modules/helpers/aws_config_recording/variables.tf @@ -19,12 +19,14 @@ variable "tags" { } variable "delivery_bucket_name" { - description = "Name of the existing S3 bucket that receives AWS Config data. The bucket can be in another Region or account." + description = "Optional name of an existing S3 delivery bucket. When null, the module creates a regional bucket and SQS notification queue." type = string + default = null + nullable = true validation { - condition = can(regex("^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", var.delivery_bucket_name)) - error_message = "The delivery bucket name must be a valid 3-63 character S3 bucket name." + condition = var.delivery_bucket_name == null || can(regex("^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", var.delivery_bucket_name)) + error_message = "delivery_bucket_name must be null or a valid 3-63 character S3 bucket name." } } diff --git a/modules/helpers/storage/README.md b/modules/helpers/storage/README.md index 364d231b..db097f6b 100644 --- a/modules/helpers/storage/README.md +++ b/modules/helpers/storage/README.md @@ -10,7 +10,6 @@ Forge produces operational artifacts, logs, and temporary data across many tenan - Long-term and short-term S3 buckets. - Versioning, ownership controls, server-side encryption, and public access blocks. -- An account-scoped bucket policy allowing AWS Config delivery to the long-term bucket. - Lifecycle configuration for the short-term bucket. - Bucket settings exported for downstream modules. @@ -19,6 +18,15 @@ Forge produces operational artifacts, logs, and temporary data across many tenan - Use the short-term bucket for generated or transient artifacts, not compliance evidence. - Retention periods and encryption defaults should match the account data-classification policy. - Bucket names and outputs are often consumed by integration modules, so avoid renames without migration planning. +- This module does not grant AWS Config delivery permissions. Use the managed + delivery bucket from `modules/helpers/aws_config_recording`, or manage an + external delivery bucket and its policy outside this module. +- Upgrading from a release that managed `aws_s3_bucket_policy.config_delivery` + removes that long-term bucket policy. Move each recorder to its managed or + externally owned delivery bucket before applying this storage change. +- Removing the policy does not delete the long-term bucket or historical + `AWSLogs//Config/` objects. Retain, migrate, or backfill those + objects separately according to the account retention policy. ## Requirements @@ -32,7 +40,7 @@ Forge produces operational artifacts, logs, and temporary data across many tenan | Name | Version | | ---- | ------- | -| [aws](#provider\_aws) | 6.57.1 | +| [aws](#provider\_aws) | 6.59.0 | ## Modules @@ -47,7 +55,6 @@ No modules. | [aws_s3_bucket_lifecycle_configuration.s3_short_term](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_lifecycle_configuration) | resource | | [aws_s3_bucket_ownership_controls.s3_long_term](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource | | [aws_s3_bucket_ownership_controls.s3_short_term](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_ownership_controls) | resource | -| [aws_s3_bucket_policy.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy) | resource | | [aws_s3_bucket_public_access_block.s3_long_term](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | | [aws_s3_bucket_public_access_block.s3_short_term](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | | [aws_s3_bucket_server_side_encryption_configuration.s3_long_term](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | @@ -56,7 +63,6 @@ No modules. | [aws_s3_bucket_versioning.s3_short_term](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_versioning) | resource | | [aws_servicecatalogappregistry_application.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicecatalogappregistry_application) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.config_delivery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs diff --git a/modules/helpers/storage/data.tf b/modules/helpers/storage/data.tf index 50b9d168..8fc4b38c 100644 --- a/modules/helpers/storage/data.tf +++ b/modules/helpers/storage/data.tf @@ -1,48 +1 @@ data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "config_delivery" { - statement { - sid = "AWSConfigBucketPermissionsCheck" - effect = "Allow" - actions = [ - "s3:GetBucketAcl", - "s3:ListBucket", - ] - resources = [aws_s3_bucket.s3_long_term.arn] - - principals { - type = "Service" - identifiers = ["config.amazonaws.com"] - } - - condition { - test = "StringEquals" - variable = "AWS:SourceAccount" - values = [data.aws_caller_identity.current.account_id] - } - } - - statement { - sid = "AWSConfigBucketDelivery" - effect = "Allow" - actions = ["s3:PutObject"] - resources = ["${aws_s3_bucket.s3_long_term.arn}/AWSLogs/${data.aws_caller_identity.current.account_id}/Config/*"] - - principals { - type = "Service" - identifiers = ["config.amazonaws.com"] - } - - condition { - test = "StringEquals" - variable = "AWS:SourceAccount" - values = [data.aws_caller_identity.current.account_id] - } - - condition { - test = "StringEquals" - variable = "s3:x-amz-acl" - values = ["bucket-owner-full-control"] - } - } -} diff --git a/modules/helpers/storage/main.tf b/modules/helpers/storage/main.tf index 47e1ac3e..e0d19a44 100644 --- a/modules/helpers/storage/main.tf +++ b/modules/helpers/storage/main.tf @@ -50,13 +50,6 @@ resource "aws_s3_bucket_public_access_block" "s3_long_term" { skip_destroy = true } -# Allow AWS Config recorders in this account to deliver configuration history -# and snapshots to the long-term bucket. -resource "aws_s3_bucket_policy" "config_delivery" { - bucket = aws_s3_bucket.s3_long_term.id - policy = data.aws_iam_policy_document.config_delivery.json -} - # Short-term storage (i.e. temporary/feature-branch builds, core dumps, and # other artifacts we aren't obligated to retain long-term). resource "aws_s3_bucket" "s3_short_term" { diff --git a/modules/helpers/storage/tests/behavior.tftest.hcl b/modules/helpers/storage/tests/behavior.tftest.hcl index 0dfdc657..a94cb80d 100644 --- a/modules/helpers/storage/tests/behavior.tftest.hcl +++ b/modules/helpers/storage/tests/behavior.tftest.hcl @@ -9,13 +9,6 @@ override_data { } } -override_data { - target = data.aws_iam_policy_document.config_delivery - values = { - json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"AWSConfigBucketPermissionsCheck\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"config.amazonaws.com\"},\"Action\":[\"s3:GetBucketAcl\",\"s3:ListBucket\"],\"Resource\":\"arn:aws:s3:::123456789012-long-term-storage\"},{\"Sid\":\"AWSConfigBucketDelivery\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"config.amazonaws.com\"},\"Action\":\"s3:PutObject\",\"Resource\":\"arn:aws:s3:::123456789012-long-term-storage/AWSLogs/123456789012/Config/*\"}]}" - } -} - override_resource { target = aws_s3_bucket.s3_long_term values = { @@ -58,7 +51,6 @@ run "storage_buckets_contract" { && aws_s3_bucket_public_access_block.s3_long_term.ignore_public_acls == true && aws_s3_bucket_public_access_block.s3_long_term.restrict_public_buckets == true && aws_s3_bucket_public_access_block.s3_long_term.skip_destroy == true - && aws_s3_bucket_policy.config_delivery.bucket == "123456789012-long-term-storage" ) error_message = "Storage helper must keep long-term bucket naming, tags, versioning, AES256 encryption, and public access block settings." } diff --git a/modules/helpers/storage/tests/source_inventory.tftest.hcl b/modules/helpers/storage/tests/source_inventory.tftest.hcl index b26b0dbd..9360b547 100644 --- a/modules/helpers/storage/tests/source_inventory.tftest.hcl +++ b/modules/helpers/storage/tests/source_inventory.tftest.hcl @@ -14,13 +14,6 @@ run "helpers_storage_source_inventory" { "resource \"aws_s3_bucket_versioning\" \"s3_long_term\"", "resource \"aws_s3_bucket_server_side_encryption_configuration\" \"s3_long_term\"", "resource \"aws_s3_bucket_public_access_block\" \"s3_long_term\"", - "resource \"aws_s3_bucket_policy\" \"config_delivery\"", - "data \"aws_iam_policy_document\" \"config_delivery\"", - "identifiers = [\"config.amazonaws.com\"]", - "variable = \"AWS:SourceAccount\"", - "\"s3:GetBucketAcl\"", - "\"s3:ListBucket\"", - "actions = [\"s3:PutObject\"]", "resource \"aws_s3_bucket\" \"s3_short_term\"", "resource \"aws_s3_bucket_ownership_controls\" \"s3_short_term\"", "resource \"aws_s3_bucket_lifecycle_configuration\" \"s3_short_term\"", @@ -39,7 +32,7 @@ run "helpers_storage_source_inventory" { } assert { - condition = output.expected_literal_count == 22 - error_message = "Source inventory must keep 22 module-specific Terraform blocks and AWS Config delivery permissions pinned." + condition = output.expected_literal_count == 15 + error_message = "Source inventory must keep 15 storage bucket resources, outputs, and provider blocks pinned." } } diff --git a/tests/quality/examples_contract_test.py b/tests/quality/examples_contract_test.py index 383f0067..e9b2cdd1 100644 --- a/tests/quality/examples_contract_test.py +++ b/tests/quality/examples_contract_test.py @@ -130,3 +130,95 @@ def test_dependency_monitor_example_is_regional_and_ordered() -> None: assert "github_api_version: '2022-11-28'" in regional_config assert 'name_prefix:' not in regional_config assert 'tenant_configs:' not in regional_config + + +def test_splunk_data_manager_example_documents_aws_config_handoff() -> None: + integration_root = ( + EXAMPLES / 'deployments' / 'integrations' / 'terragrunt' + ) + global_config = integration_root.joinpath( + '_global_settings', + 'splunk_cloud_data_manager.hcl', + ).read_text(encoding='utf-8') + deployment_config = integration_root.joinpath( + 'environments', + 'prod', + 'splunk_cloud_data_manager', + 'config.yml', + ).read_text(encoding='utf-8') + template_config = EXAMPLES.joinpath( + 'templates', + 'integrations', + 'splunk_cloud_data_manager', + 'config.yml', + ).read_text(encoding='utf-8') + producer_outputs = REPO_ROOT.joinpath( + 'modules', + 'helpers', + 'aws_config_recording', + 'outputs.tf', + ).read_text(encoding='utf-8') + + assert 'dependency "aws_config_recording"' not in global_config + assert ( + 's3_logs_config = ' + 'local.splunk_cloud.locals.s3_logs_config' + ) in global_config + + assert deployment_config.count('name: forge-aws-config-prod') == 1 + assert ( + ' - enabled: false\n name: forge-aws-config-prod' + in deployment_config + ) + deployment_aws_config = deployment_config.split( + ' name: forge-aws-config-prod', 1 + )[1].split('\n ct-logs:', 1)[0] + + for required in [ + 'name: forge-s3-logs-prod', + 'source_type: forgecicd:runner-logs:s3', + ]: + assert required in deployment_config + + for required in [ + 'iam_region: eu-west-1', + 'index: forge-prod-index', + 'source_type: forgecicd:aws:config:s3', + 'splunk_s3_logs.sqs.url', + 'splunk_s3_logs.bucket_arn', + 'splunk_s3_logs.bucket_kms_key_arn', + 'sqs_urls: []', + 's3_bucket_patterns: []', + 'kms_key_arns: []', + ]: + assert required in deployment_aws_config + + assert template_config.count('name: forge-aws-config-prod') == 1 + assert ( + ' - enabled: false # enable only after ' + 'the producer output and Splunk parser are validated\n' + ' name: forge-aws-config-prod' + ) in template_config + template_aws_config = template_config.split( + ' name: forge-aws-config-prod', 1 + )[1].split('\n ct-logs:', 1)[0] + + for required in [ + 'source_type: forgecicd:aws:config:s3', + 'splunk_s3_logs.sqs.url', + 'splunk_s3_logs.bucket_arn', + 'splunk_s3_logs.bucket_kms_key_arn', + 'sqs_urls: []', + 's3_bucket_patterns: []', + 'kms_key_arns: []', + ]: + assert required in template_aws_config + + for required in [ + 'output "splunk_s3_logs"', + 'bucket_arn', + 'bucket_kms_key_arn', + 'sqs = {', + 'url', + ]: + assert required in producer_outputs