Trace the queue worker, dropping polling noise

infra/modules/ecs-service/adot-collector-config.yaml.tpl

@@ -16,6 +16,15 @@ receivers:
     transport: udp
 
 processors:
+%{ if length(drop_spans) > 0 ~}
+  filter/drop_spans:
+    error_mode: ignore
+    traces:
+      span:
+%{ for expr in drop_spans ~}
+        - '${expr}'
+%{ endfor ~}
+%{ endif ~}
   batch/traces:
     timeout: 1s
     send_batch_size: 50
@@ -65,7 +74,7 @@ service:
   pipelines:
     traces:
       receivers: [otlp, awsxray]
-      processors: [batch/traces]
+      processors: [${length(drop_spans) > 0 ? "filter/drop_spans, " : ""}batch/traces]
       exporters: [awsxray]
     metrics:
       receivers: [otlp]

infra/modules/ecs-service/adot.tf

@@ -11,6 +11,7 @@ resource "aws_ssm_parameter" "adot_collector_config" {
   value = templatefile("${path.module}/adot-collector-config.yaml.tpl", {
     aws_region = data.aws_region.current.region
     env_name   = var.env_name
+    drop_spans = var.adot_drop_span_expressions
   })
 
   description = "OpenTelemetry collector configuration for the ${var.application} ADOT sidecar in ${var.env_name}"

infra/modules/ecs-service/variables.tf

@@ -213,6 +213,12 @@ variable "opentelemetry_head_sampler_ratio" {
   default     = "1"
 }
 
+variable "adot_drop_span_expressions" {
+  type        = list(string)
+  description = "OTTL span expressions for the filter/drop_spans processor. Matching spans are dropped before export to X-Ray. An empty list disables the filter entirely."
+  default     = []
+}
+
 variable "healthcheck" {
   type = object({
     command     = list(string)

infra/modules/forms-runner/queue_worker.tf

@@ -2,11 +2,54 @@ locals {
   queue_worker_name           = "${module.ecs_service.task_container_definition.name}-queue-worker"
   queue_worker_log_group_name = "/aws/ecs/${local.queue_worker_name}-${var.env_name}"
 
-  # Take the exported task container definition and override some parts of it
-  # Note: the ENV variables aren't overridden because it's not possible to cherry pick them
+  # OTTL expressions for the ADOT filter/drop_spans processor.
+  # These drop the high-volume SolidQueue polling spans that create noise in X-Ray
+  # without obscuring the job work we actually care about (S3, SES, submissions).
+  queue_worker_drop_span_expressions = [
+    "IsMatch(name, \"SolidQueue::.*\")",
+    "name == \"ActiveRecord.transaction\"",
+    "IsMatch(name, \"(BEGIN|COMMIT|SELECT|INSERT|UPDATE|DELETE) forms-runner-queue\")",
+  ]
+
+  # Round up CPU/memory to next valid Fargate tier when OTEL is enabled,
+  # mirroring the logic in the ecs-service module.
+  queue_worker_total_cpu    = var.cpu + module.ecs_service.adot_sidecar_cpu
+  queue_worker_total_memory = var.memory + module.ecs_service.adot_sidecar_memory
+
+  queue_worker_fargate_cpu = (
+    local.queue_worker_total_cpu <= 256 ? 256 :
+    local.queue_worker_total_cpu <= 512 ? 512 :
+    local.queue_worker_total_cpu <= 1024 ? 1024 :
+    local.queue_worker_total_cpu <= 2048 ? 2048 :
+    local.queue_worker_total_cpu <= 4096 ? 4096 :
+    local.queue_worker_total_cpu <= 8192 ? 8192 : 16384
+  )
+
+  queue_worker_min_memory_for_cpu = (
+    local.queue_worker_fargate_cpu <= 256 ? 512 :
+    local.queue_worker_fargate_cpu <= 512 ? 1024 :
+    local.queue_worker_fargate_cpu <= 1024 ? 2048 :
+    local.queue_worker_fargate_cpu <= 2048 ? 4096 :
+    local.queue_worker_fargate_cpu <= 4096 ? 8192 :
+    local.queue_worker_fargate_cpu <= 8192 ? 16384 : 32768
+  )
+
+  queue_worker_fargate_memory = max(
+    local.queue_worker_min_memory_for_cpu,
+    local.queue_worker_total_memory <= 512 ? 512 :
+    local.queue_worker_total_memory <= 1024 ? 1024 :
+    local.queue_worker_total_memory <= 2048 ? 2048 :
+    local.queue_worker_total_memory <= 4096 ? 4096 :
+    local.queue_worker_total_memory <= 8192 ? 8192 :
+    local.queue_worker_total_memory <= 16384 ? 16384 :
+    local.queue_worker_total_memory <= 32768 ? 32768 : 65536
+  )
+
+  # Take the exported task container definition and override some parts of it.
+  # Note: the ENV variables aren't overridden because it's not possible to cherry pick them.
   # This means DISABLE_SOLID_QUEUE is always set to true, but that instruction is overridden
-  # by the command `bin/jobs` which starts the SolidQueue worker but not the Rails server
-  queue_worker_container_definitions = merge(
+  # by the command `bin/jobs` which starts the SolidQueue worker but not the Rails server.
+  queue_worker_container_definition = merge(
     module.ecs_service.task_container_definition,
     {
       name = local.queue_worker_name,
@@ -30,15 +73,18 @@ locals {
           awslogs-stream-prefix = local.queue_worker_log_group_name
         }
       }
-      # Explicitly disable opentelemetry for the queue worker, as we only want to trace user interactions in the main app, and not background jobs.
-      # Override the `dependsOn` set in the main task container definition, as we don't provision the collector here.
-      dependsOn = [],
-      # Also, strip out any OTEL_ or _OTEL envars, to disable opentelemetry for this container, even if it's enabled for the main app.
-      environment = [
-        for env in module.ecs_service.task_container_definition.environment :
-        env
-        if !startswith(env.name, "OTEL_") && !endswith(env.name, "_OTEL")
-      ]
+
+      # When OTEL is enabled, the ADOT sidecar is provisioned alongside this container;
+      # restore the dependsOn so the app waits for the collector to start.
+      dependsOn = var.enable_opentelemetry ? [
+        {
+          containerName = "aws-otel-collector",
+          condition     = "START"
+        }
+      ] : [],
+
+      # Pass all env vars through unchanged — OTEL_* vars are needed when the sidecar is present.
+      environment = module.ecs_service.task_container_definition.environment
 
       secrets = [
         {
@@ -64,16 +110,51 @@ locals {
       ]
     }
   )
+
+  # ADOT sidecar for the queue worker — same image as the main app's sidecar but with
+  # a queue-worker-specific config that includes the polling-noise filter.
+  queue_worker_adot_container_definition = var.enable_opentelemetry ? {
+    name                   = "aws-otel-collector",
+    image                  = module.ecs_service.adot_image,
+    essential              = false,
+    readonlyRootFilesystem = true,
+    cpu                    = module.ecs_service.adot_sidecar_cpu,
+    memory                 = module.ecs_service.adot_sidecar_memory,
+    secrets = [
+      {
+        name      = "AOT_CONFIG_CONTENT"
+        valueFrom = aws_ssm_parameter.queue_worker_adot_collector_config[0].arn
+      }
+    ],
+    logConfiguration = {
+      logDriver = "awslogs",
+      options = {
+        awslogs-group         = "${local.queue_worker_log_group_name}/adot-collector",
+        awslogs-region        = "eu-west-2",
+        awslogs-stream-prefix = "adot"
+      }
+    },
+    healthCheck = {
+      command     = ["CMD", "/healthcheck"],
+      interval    = 30,
+      timeout     = 5,
+      retries     = 5,
+      startPeriod = 10
+    },
+  } : null
 }
 
 resource "aws_ecs_task_definition" "queue_worker" {
-  family                   = "${var.env_name}-${local.queue_worker_name}"
-  container_definitions    = jsonencode([local.queue_worker_container_definitions])
+  family = "${var.env_name}-${local.queue_worker_name}"
+  container_definitions = var.enable_opentelemetry ? jsonencode([
+    local.queue_worker_container_definition,
+    local.queue_worker_adot_container_definition,
+  ]) : jsonencode([local.queue_worker_container_definition])
   execution_role_arn       = aws_iam_role.ecs_task_exec_role.arn
   task_role_arn            = module.ecs_service.task_definition.task_role_arn
   requires_compatibilities = module.ecs_service.task_definition.requires_compatibilities
-  cpu                      = var.cpu
-  memory                   = var.memory
+  cpu                      = var.enable_opentelemetry ? local.queue_worker_fargate_cpu : var.cpu
+  memory                   = var.enable_opentelemetry ? local.queue_worker_fargate_memory : var.memory
   network_mode             = "awsvpc"
 
   // As this terraform module doesn't deal with updating app code, we see drift every time it's applied because the image is changed elsewhere.
@@ -196,10 +277,13 @@ data "aws_iam_policy_document" "queue_worker_ecs_task_exec_additional_policy" {
     actions = [
       "ssm:GetParameters"
     ]
-    resources = [
-      for parameter in local.queue_worker_container_definitions.secrets : parameter.valueFrom
-      if startswith(parameter.valueFrom, "arn:aws:ssm:")
-    ]
+    resources = concat(
+      [
+        for parameter in local.queue_worker_container_definition.secrets : parameter.valueFrom
+        if startswith(parameter.valueFrom, "arn:aws:ssm:")
+      ],
+      var.enable_opentelemetry ? [aws_ssm_parameter.queue_worker_adot_collector_config[0].arn] : [],
+    )
     effect = "Allow"
   }
 }
@@ -218,13 +302,40 @@ resource "aws_ssm_parameter" "queue_worker_sentry_dsn" {
   }
 }
 
+data "aws_region" "current" {}
+
+resource "aws_ssm_parameter" "queue_worker_adot_collector_config" {
+  #checkov:skip=CKV2_AWS_34:Not a secret; the config template is committed to this public repository
+  #checkov:skip=CKV2_FORMS_AWS_7:Value is not a secret set outside Terraform; it must track changes to the config template
+
+  count = var.enable_opentelemetry ? 1 : 0
+
+  name = "/${local.queue_worker_name}-${var.env_name}/adot-collector-config"
+  type = "String"
+  value = templatefile("${path.module}/../ecs-service/adot-collector-config.yaml.tpl", {
+    aws_region = data.aws_region.current.region
+    env_name   = var.env_name
+    drop_spans = local.queue_worker_drop_span_expressions
+  })
+
+  description = "OpenTelemetry collector configuration for the ${local.queue_worker_name} ADOT sidecar in ${var.env_name}"
+}
+
 resource "aws_cloudwatch_log_group" "queue_worker" {
   #checkov:skip=CKV_AWS_338:We're happy with 30 days retention for now
   #checkov:skip=CKV_AWS_158:Default AWS SSE is sufficient, no need for CM KMS.
   name              = local.queue_worker_log_group_name
   retention_in_days = 30
 }
 
+resource "aws_cloudwatch_log_group" "queue_worker_adot" {
+  #checkov:skip=CKV_AWS_338:We're happy with 30 days retention for now
+  #checkov:skip=CKV_AWS_158:Default AWS SSE is sufficient, no need for CM KMS.
+  count             = var.enable_opentelemetry ? 1 : 0
+  name              = "${local.queue_worker_log_group_name}/adot-collector"
+  retention_in_days = 30
+}
+
 module "cribl_well_known" {
   source = "../well-known/cribl"
 }