feat(webhook): add optional X-Ray instrumentation for GitHub event delivery latency

lambdas/functions/webhook/src/webhook/index.ts

@@ -1,6 +1,6 @@
 import { Webhooks } from '@octokit/webhooks';
 import { WorkflowJobEvent } from '@octokit/webhooks-types';
-import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
+import { createChildLogger, tracer } from '@aws-github-runner/aws-powertools-util';
 import { IncomingHttpHeaders } from 'http';
 
 import { Response } from '../lambda';
@@ -164,9 +164,34 @@ function readWorkflowJobEvent(
     },
   });
 
+  instrumentGithubLatency(event.workflow_job.created_at);
+
   return { event, eventType };
 }
 
+// Adds X-Ray visibility into the delay between GitHub creating the event and this Lambda
+// processing it. Disabled by default; enable via WEBHOOK_XRAY_GITHUB_LATENCY_ENABLED since
+// the synthetic 'github' subsegment intentionally backdates its start_time, which is an
+// unusual X-Ray pattern not every consumer of this module will want on by default.
+function instrumentGithubLatency(githubCreatedAt: string): void {
+  if (process.env.WEBHOOK_XRAY_GITHUB_LATENCY_ENABLED !== 'true') return;
+
+  const segment = tracer.getSegment();
+  if (!segment) return;
+
+  const createdAtMs = new Date(githubCreatedAt).getTime();
+  const lagMs = Date.now() - createdAtMs;
+
+  tracer.putAnnotation('event_lag_ms', lagMs);
+
+  const githubNode = segment.addNewSubsegment('github');
+  githubNode.namespace = 'remote';
+  githubNode.start_time = createdAtMs / 1000; // X-Ray uses epoch seconds
+  githubNode.addAnnotation('workflow_job_created_at', githubCreatedAt);
+  githubNode.addAnnotation('event_lag_ms', lagMs);
+  githubNode.close();
+}
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function checkBodySize(body: string, headers: IncomingHttpHeaders): { sizeExceeded: boolean; message: any } {
   // GitHub does not specify if the content length is always present, fallback to the body size calculation.

main.tf

@@ -164,7 +164,8 @@ module "webhook" {
   lambda_security_group_ids = var.lambda_security_group_ids
   aws_partition             = var.aws_partition
 
-  log_level = var.log_level
+  log_level                           = var.log_level
+  webhook_xray_github_latency_enabled = var.webhook_xray_github_latency_enabled
 }
 
 module "runners" {

modules/multi-runner/variables.tf

@@ -368,6 +368,12 @@ variable "webhook_lambda_timeout" {
   default     = 10
 }
 
+variable "webhook_xray_github_latency_enabled" {
+  description = "Add X-Ray instrumentation (an annotation plus a synthetic 'github' node) measuring the delay between a GitHub workflow_job event's created_at timestamp and this Lambda's invocation. Disabled by default since the synthetic node intentionally backdates its X-Ray start_time, an unusual tracing pattern."
+  type        = bool
+  default     = false
+}
+
 variable "role_permissions_boundary" {
   description = "Permissions boundary that will be added to the created role for the lambda."
   type        = string

modules/multi-runner/webhook.tf

@@ -40,5 +40,6 @@ module "webhook" {
   lambda_security_group_ids = var.lambda_security_group_ids
   aws_partition             = var.aws_partition
 
-  log_level = var.log_level
+  log_level                           = var.log_level
+  webhook_xray_github_latency_enabled = var.webhook_xray_github_latency_enabled
 }

modules/webhook/direct/variables.tf

@@ -48,5 +48,6 @@ variable "config" {
       arn     = string
       version = string
     }))
+    webhook_xray_github_latency_enabled = optional(bool, false)
   })
 }

modules/webhook/direct/webhook.tf

@@ -29,6 +29,7 @@ resource "aws_lambda_function" "webhook" {
         QUEUE_SELECTION_STRATEGY                 = var.config.queue_selection_strategy
         PARAMETER_RUNNER_MATCHER_CONFIG_PATH     = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name])
         PARAMETER_RUNNER_MATCHER_VERSION         = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter
+        WEBHOOK_XRAY_GITHUB_LATENCY_ENABLED      = var.config.webhook_xray_github_latency_enabled
       } : k => v if v != null
     }
   }

modules/webhook/eventbridge/variables.tf

@@ -48,6 +48,7 @@ variable "config" {
       arn     = string
       version = string
     }))
-    accept_events = optional(list(string), null)
+    accept_events                       = optional(list(string), null)
+    webhook_xray_github_latency_enabled = optional(bool, false)
   })
 }

modules/webhook/eventbridge/webhook.tf

@@ -32,6 +32,7 @@ resource "aws_lambda_function" "webhook" {
         EVENT_BUS_NAME                       = aws_cloudwatch_event_bus.main.name
         PARAMETER_GITHUB_APP_WEBHOOK_SECRET  = var.config.github_app_parameters.webhook_secret.name
         PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name])
+        WEBHOOK_XRAY_GITHUB_LATENCY_ENABLED  = var.config.webhook_xray_github_latency_enabled
       } : k => v if v != null
     }
   }

modules/webhook/variables.tf

@@ -246,3 +246,9 @@ EOF
     accept_events = optional(list(string), null)
   })
 }
+
+variable "webhook_xray_github_latency_enabled" {
+  description = "Add X-Ray instrumentation (an annotation plus a synthetic 'github' node) measuring the delay between a GitHub workflow_job event's created_at timestamp and this Lambda's invocation. Disabled by default since the synthetic node intentionally backdates its X-Ray start_time, an unusual tracing pattern."
+  type        = bool
+  default     = false
+}

modules/webhook/webhook.tf

@@ -92,6 +92,7 @@ module "direct" {
         version = p.version
       }
     ]
+    webhook_xray_github_latency_enabled = var.webhook_xray_github_latency_enabled
   }
 }
 
@@ -135,7 +136,8 @@ module "eventbridge" {
         version = p.version
       }
     ]
-    accept_events = var.eventbridge.accept_events
+    accept_events                       = var.eventbridge.accept_events
+    webhook_xray_github_latency_enabled = var.webhook_xray_github_latency_enabled
   }
 
 }

variables.tf

@@ -180,6 +180,12 @@ variable "webhook_lambda_timeout" {
   default     = 10
 }
 
+variable "webhook_xray_github_latency_enabled" {
+  description = "Add X-Ray instrumentation (an annotation plus a synthetic 'github' node) measuring the delay between a GitHub workflow_job event's created_at timestamp and this Lambda's invocation. Disabled by default."
+  type        = bool
+  default     = false
+}
+
 variable "runners_lambda_zip" {
   description = "File location of the lambda zip file for scaling runners."
   type        = string