diff --git a/infra/modules/ecs-service/adot-collector-config.yaml.tpl b/infra/modules/ecs-service/adot-collector-config.yaml.tpl index d226c5873..ee6d6726e 100644 --- a/infra/modules/ecs-service/adot-collector-config.yaml.tpl +++ b/infra/modules/ecs-service/adot-collector-config.yaml.tpl @@ -16,6 +16,15 @@ receivers: transport: udp processors: +%{ if length(drop_spans) > 0 ~} + filter/drop_spans: + error_mode: ignore + traces: + span: +%{ for expr in drop_spans ~} + - '${expr}' +%{ endfor ~} +%{ endif ~} batch/traces: timeout: 1s send_batch_size: 50 @@ -65,7 +74,7 @@ service: pipelines: traces: receivers: [otlp, awsxray] - processors: [batch/traces] + processors: [${length(drop_spans) > 0 ? "filter/drop_spans, " : ""}batch/traces] exporters: [awsxray] metrics: receivers: [otlp] diff --git a/infra/modules/ecs-service/adot.tf b/infra/modules/ecs-service/adot.tf index 616453935..d76a87c79 100644 --- a/infra/modules/ecs-service/adot.tf +++ b/infra/modules/ecs-service/adot.tf @@ -11,6 +11,7 @@ resource "aws_ssm_parameter" "adot_collector_config" { value = templatefile("${path.module}/adot-collector-config.yaml.tpl", { aws_region = data.aws_region.current.region env_name = var.env_name + drop_spans = var.adot_drop_span_expressions }) description = "OpenTelemetry collector configuration for the ${var.application} ADOT sidecar in ${var.env_name}" diff --git a/infra/modules/ecs-service/variables.tf b/infra/modules/ecs-service/variables.tf index d7ef0f190..3f53081f5 100644 --- a/infra/modules/ecs-service/variables.tf +++ b/infra/modules/ecs-service/variables.tf @@ -213,6 +213,12 @@ variable "opentelemetry_head_sampler_ratio" { default = "1" } +variable "adot_drop_span_expressions" { + type = list(string) + description = "OTTL span expressions for the filter/drop_spans processor. Matching spans are dropped before export to X-Ray. An empty list disables the filter entirely." + default = [] +} + variable "healthcheck" { type = object({ command = list(string) diff --git a/infra/modules/forms-runner/queue_worker.tf b/infra/modules/forms-runner/queue_worker.tf index 15d8e1d88..449cdad18 100644 --- a/infra/modules/forms-runner/queue_worker.tf +++ b/infra/modules/forms-runner/queue_worker.tf @@ -2,11 +2,54 @@ locals { queue_worker_name = "${module.ecs_service.task_container_definition.name}-queue-worker" queue_worker_log_group_name = "/aws/ecs/${local.queue_worker_name}-${var.env_name}" - # Take the exported task container definition and override some parts of it - # Note: the ENV variables aren't overridden because it's not possible to cherry pick them + # OTTL expressions for the ADOT filter/drop_spans processor. + # These drop the high-volume SolidQueue polling spans that create noise in X-Ray + # without obscuring the job work we actually care about (S3, SES, submissions). + queue_worker_drop_span_expressions = [ + "IsMatch(name, \"SolidQueue::.*\")", + "name == \"ActiveRecord.transaction\"", + "IsMatch(name, \"(BEGIN|COMMIT|SELECT|INSERT|UPDATE|DELETE) forms-runner-queue\")", + ] + + # Round up CPU/memory to next valid Fargate tier when OTEL is enabled, + # mirroring the logic in the ecs-service module. + queue_worker_total_cpu = var.cpu + module.ecs_service.adot_sidecar_cpu + queue_worker_total_memory = var.memory + module.ecs_service.adot_sidecar_memory + + queue_worker_fargate_cpu = ( + local.queue_worker_total_cpu <= 256 ? 256 : + local.queue_worker_total_cpu <= 512 ? 512 : + local.queue_worker_total_cpu <= 1024 ? 1024 : + local.queue_worker_total_cpu <= 2048 ? 2048 : + local.queue_worker_total_cpu <= 4096 ? 4096 : + local.queue_worker_total_cpu <= 8192 ? 8192 : 16384 + ) + + queue_worker_min_memory_for_cpu = ( + local.queue_worker_fargate_cpu <= 256 ? 512 : + local.queue_worker_fargate_cpu <= 512 ? 1024 : + local.queue_worker_fargate_cpu <= 1024 ? 2048 : + local.queue_worker_fargate_cpu <= 2048 ? 4096 : + local.queue_worker_fargate_cpu <= 4096 ? 8192 : + local.queue_worker_fargate_cpu <= 8192 ? 16384 : 32768 + ) + + queue_worker_fargate_memory = max( + local.queue_worker_min_memory_for_cpu, + local.queue_worker_total_memory <= 512 ? 512 : + local.queue_worker_total_memory <= 1024 ? 1024 : + local.queue_worker_total_memory <= 2048 ? 2048 : + local.queue_worker_total_memory <= 4096 ? 4096 : + local.queue_worker_total_memory <= 8192 ? 8192 : + local.queue_worker_total_memory <= 16384 ? 16384 : + local.queue_worker_total_memory <= 32768 ? 32768 : 65536 + ) + + # Take the exported task container definition and override some parts of it. + # Note: the ENV variables aren't overridden because it's not possible to cherry pick them. # This means DISABLE_SOLID_QUEUE is always set to true, but that instruction is overridden - # by the command `bin/jobs` which starts the SolidQueue worker but not the Rails server - queue_worker_container_definitions = merge( + # by the command `bin/jobs` which starts the SolidQueue worker but not the Rails server. + queue_worker_container_definition = merge( module.ecs_service.task_container_definition, { name = local.queue_worker_name, @@ -30,15 +73,18 @@ locals { awslogs-stream-prefix = local.queue_worker_log_group_name } } - # Explicitly disable opentelemetry for the queue worker, as we only want to trace user interactions in the main app, and not background jobs. - # Override the `dependsOn` set in the main task container definition, as we don't provision the collector here. - dependsOn = [], - # Also, strip out any OTEL_ or _OTEL envars, to disable opentelemetry for this container, even if it's enabled for the main app. - environment = [ - for env in module.ecs_service.task_container_definition.environment : - env - if !startswith(env.name, "OTEL_") && !endswith(env.name, "_OTEL") - ] + + # When OTEL is enabled, the ADOT sidecar is provisioned alongside this container; + # restore the dependsOn so the app waits for the collector to start. + dependsOn = var.enable_opentelemetry ? [ + { + containerName = "aws-otel-collector", + condition = "START" + } + ] : [], + + # Pass all env vars through unchanged — OTEL_* vars are needed when the sidecar is present. + environment = module.ecs_service.task_container_definition.environment secrets = [ { @@ -64,16 +110,51 @@ locals { ] } ) + + # ADOT sidecar for the queue worker — same image as the main app's sidecar but with + # a queue-worker-specific config that includes the polling-noise filter. + queue_worker_adot_container_definition = var.enable_opentelemetry ? { + name = "aws-otel-collector", + image = module.ecs_service.adot_image, + essential = false, + readonlyRootFilesystem = true, + cpu = module.ecs_service.adot_sidecar_cpu, + memory = module.ecs_service.adot_sidecar_memory, + secrets = [ + { + name = "AOT_CONFIG_CONTENT" + valueFrom = aws_ssm_parameter.queue_worker_adot_collector_config[0].arn + } + ], + logConfiguration = { + logDriver = "awslogs", + options = { + awslogs-group = "${local.queue_worker_log_group_name}/adot-collector", + awslogs-region = "eu-west-2", + awslogs-stream-prefix = "adot" + } + }, + healthCheck = { + command = ["CMD", "/healthcheck"], + interval = 30, + timeout = 5, + retries = 5, + startPeriod = 10 + }, + } : null } resource "aws_ecs_task_definition" "queue_worker" { - family = "${var.env_name}-${local.queue_worker_name}" - container_definitions = jsonencode([local.queue_worker_container_definitions]) + family = "${var.env_name}-${local.queue_worker_name}" + container_definitions = var.enable_opentelemetry ? jsonencode([ + local.queue_worker_container_definition, + local.queue_worker_adot_container_definition, + ]) : jsonencode([local.queue_worker_container_definition]) execution_role_arn = aws_iam_role.ecs_task_exec_role.arn task_role_arn = module.ecs_service.task_definition.task_role_arn requires_compatibilities = module.ecs_service.task_definition.requires_compatibilities - cpu = var.cpu - memory = var.memory + cpu = var.enable_opentelemetry ? local.queue_worker_fargate_cpu : var.cpu + memory = var.enable_opentelemetry ? local.queue_worker_fargate_memory : var.memory network_mode = "awsvpc" // As this terraform module doesn't deal with updating app code, we see drift every time it's applied because the image is changed elsewhere. @@ -196,10 +277,13 @@ data "aws_iam_policy_document" "queue_worker_ecs_task_exec_additional_policy" { actions = [ "ssm:GetParameters" ] - resources = [ - for parameter in local.queue_worker_container_definitions.secrets : parameter.valueFrom - if startswith(parameter.valueFrom, "arn:aws:ssm:") - ] + resources = concat( + [ + for parameter in local.queue_worker_container_definition.secrets : parameter.valueFrom + if startswith(parameter.valueFrom, "arn:aws:ssm:") + ], + var.enable_opentelemetry ? [aws_ssm_parameter.queue_worker_adot_collector_config[0].arn] : [], + ) effect = "Allow" } } @@ -218,6 +302,25 @@ resource "aws_ssm_parameter" "queue_worker_sentry_dsn" { } } +data "aws_region" "current" {} + +resource "aws_ssm_parameter" "queue_worker_adot_collector_config" { + #checkov:skip=CKV2_AWS_34:Not a secret; the config template is committed to this public repository + #checkov:skip=CKV2_FORMS_AWS_7:Value is not a secret set outside Terraform; it must track changes to the config template + + count = var.enable_opentelemetry ? 1 : 0 + + name = "/${local.queue_worker_name}-${var.env_name}/adot-collector-config" + type = "String" + value = templatefile("${path.module}/../ecs-service/adot-collector-config.yaml.tpl", { + aws_region = data.aws_region.current.region + env_name = var.env_name + drop_spans = local.queue_worker_drop_span_expressions + }) + + description = "OpenTelemetry collector configuration for the ${local.queue_worker_name} ADOT sidecar in ${var.env_name}" +} + resource "aws_cloudwatch_log_group" "queue_worker" { #checkov:skip=CKV_AWS_338:We're happy with 30 days retention for now #checkov:skip=CKV_AWS_158:Default AWS SSE is sufficient, no need for CM KMS. @@ -225,6 +328,14 @@ resource "aws_cloudwatch_log_group" "queue_worker" { retention_in_days = 30 } +resource "aws_cloudwatch_log_group" "queue_worker_adot" { + #checkov:skip=CKV_AWS_338:We're happy with 30 days retention for now + #checkov:skip=CKV_AWS_158:Default AWS SSE is sufficient, no need for CM KMS. + count = var.enable_opentelemetry ? 1 : 0 + name = "${local.queue_worker_log_group_name}/adot-collector" + retention_in_days = 30 +} + module "cribl_well_known" { source = "../well-known/cribl" }