From a90534732d9b22850b7dfca027ed5b97833cff34 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Fri, 10 Jul 2026 20:56:48 -0400 Subject: [PATCH 01/17] fix(metrics): hold back cold-start invocation metric until durable tag is known In On-Demand mode, the Extensions API's `/next` poll (driving on_invoke_event) consistently beats the buffered Telemetry API's platform.initStart (which sets the durable_function tag), so the first invocation's aws.lambda.enhanced.invocations metric almost always missed durable_function:true. Hold the sandbox's first invocation metric in a pending field and flush it once platform.initStart has set the durable tag, with fallbacks on the next invoke or shutdown so the metric is never dropped if initStart never arrives. --- .../src/lifecycle/invocation/processor.rs | 120 +++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 7d6943bdf..6552b04d5 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -111,6 +111,17 @@ pub struct Processor { /// on `platform.report`. This flag ensures whichever event arrives first wins and the other is skipped, /// preventing double counting. init_duration_metric_emitted: bool, + /// Timestamp of the invocation metric for the sandbox's first (cold start) invocation in + /// On-Demand mode, held back from `enhanced_metrics` until `platform.initStart` is processed. + /// + /// The Extensions API's `/next` poll (which drives `on_invoke_event`) has no artificial + /// delay, while `platform.initStart` is delivered through the buffered Telemetry API. In + /// On-Demand mode the Invoke event consistently reaches the processor first, so emitting the + /// invocation metric immediately would race `set_durable_function_tag` and miss the + /// `durable_function:true` tag on invocation #1. Holding the metric here until + /// `on_platform_init_start` (or the next invocation, or shutdown) flushes it ensures the tag + /// is applied before the metric is counted. + pending_first_invocation_metric: Option, } impl Processor { @@ -154,12 +165,24 @@ impl Processor { durable_context_tx, restore_time: None, init_duration_metric_emitted: false, + pending_first_invocation_metric: None, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { + // Flush any invocation metric held back from a previous cold start. This is a fallback + // for the case where `platform.initStart` never arrived (or arrived very late) before + // this next invocation; see `pending_first_invocation_metric` for why it's held back. + self.flush_pending_first_invocation_metric(); + + // In On-Demand mode, the very first invocation of a cold sandbox races + // `platform.initStart` (see `pending_first_invocation_metric`); hold its invocation + // metric back instead of emitting it inline below. + let is_on_demand_cold_start = + !self.aws_config.is_managed_instance_mode() && self.context_buffer.is_empty(); + // In Managed Instance mode, if awaiting the first invocation after init, find and update the empty context created on init start if self.aws_config.is_managed_instance_mode() && self.awaiting_first_invocation { if self @@ -224,8 +247,13 @@ impl Processor { .add_enhanced_metric_data(&request_id, enhanced_metric_offsets); } - // Increment the invocation metric - self.enhanced_metrics.increment_invocation_metric(timestamp); + // Increment the invocation metric, unless it's an On-Demand cold start, in which case + // it's held back until `platform.initStart` (or a fallback) flushes it. + if is_on_demand_cold_start { + self.pending_first_invocation_metric = Some(timestamp); + } else { + self.enhanced_metrics.increment_invocation_metric(timestamp); + } self.enhanced_metrics.set_invoked_received(); // Both modes: check for buffered UniversalInstrumentationStart by request_id first. @@ -257,6 +285,14 @@ impl Processor { } } + /// Emits the invocation metric held back by `on_invoke_event` for an On-Demand cold start, + /// if one is pending. See `pending_first_invocation_metric` for why it's held back. + fn flush_pending_first_invocation_metric(&mut self) { + if let Some(timestamp) = self.pending_first_invocation_metric.take() { + self.enhanced_metrics.increment_invocation_metric(timestamp); + } + } + /// On the first invocation, determine if it's a cold start or proactive init. /// /// For every other invocation, it will always be warm start. @@ -339,6 +375,10 @@ impl Processor { { self.enhanced_metrics.set_durable_function_tag(); } + // Flush the held-back invocation metric now that the durable_function tag (if any) has + // been set, so invocation #1 is counted with the correct tag. + self.flush_pending_first_invocation_metric(); + let start_time: i64 = SystemTime::from(time) .duration_since(UNIX_EPOCH) .expect("time went backwards") @@ -990,6 +1030,10 @@ impl Processor { } pub fn on_shutdown_event(&mut self) { + // Don't drop the invocation metric if the sandbox shuts down before + // `platform.initStart` (or a second invocation) had a chance to flush it. + self.flush_pending_first_invocation_metric(); + let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("unable to poll clock, unrecoverable") @@ -2095,6 +2139,78 @@ mod tests { ); } + #[tokio::test] + async fn test_on_invoke_event_cold_start_holds_invocation_metric_until_init_start() { + let mut processor = setup(); + + // Simulate the On-Demand race: the Invoke event reaches the processor before + // `platform.initStart`. The invocation metric must not be emitted yet. + processor.on_invoke_event("req-1".to_string()); + assert!( + processor.pending_first_invocation_metric.is_some(), + "Expected the cold start invocation metric to be held back" + ); + + let now: i64 = std::time::UNIX_EPOCH + .elapsed() + .expect("unable to poll clock, unrecoverable") + .as_secs() + .try_into() + .unwrap_or_default(); + + // `platform.initStart` arrives afterwards and reports a durable runtime. + processor.on_platform_init_start( + Utc::now(), + Some("python:3.14.DurableFunction.v6".to_string()), + ); + + assert!( + processor.pending_first_invocation_metric.is_none(), + "Expected the held-back invocation metric to be flushed by on_platform_init_start" + ); + + let runtime = processor + .runtime + .clone() + .expect("runtime should have been resolved by on_invoke_event"); + let ts = (now / 10) * 10; + let expected_tags = dogstatsd::metric::SortedTags::parse(&format!( + "cold_start:true,durable_function:true,runtime:{runtime}" + )) + .ok(); + let entry = processor + .enhanced_metrics + .aggr_handle + .get_entry_by_id( + crate::metrics::enhanced::constants::INVOCATIONS_METRIC.into(), + expected_tags, + ts, + ) + .await + .unwrap(); + assert!( + entry.is_some(), + "Expected the flushed invocation metric to carry the durable_function:true tag" + ); + } + + #[tokio::test] + async fn test_on_invoke_event_flushes_pending_metric_if_init_start_never_arrives() { + let mut processor = setup(); + + // First invocation of a cold sandbox: metric held back. + processor.on_invoke_event("req-1".to_string()); + assert!(processor.pending_first_invocation_metric.is_some()); + + // A second invocation arrives without platform.initStart ever showing up. The held-back + // metric from req-1 must still be flushed (as a fallback), not lost. + processor.on_invoke_event("req-2".to_string()); + assert!( + processor.pending_first_invocation_metric.is_none(), + "Expected the pending metric to be flushed as a fallback on the next invocation" + ); + } + #[tokio::test] async fn test_init_duration_emitted_from_init_report() { let mut processor = setup(); From 3e0be33394976316d6660cd1003dd7af3ddd7406 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:31:46 -0400 Subject: [PATCH 02/17] test: add integration test for durable-function cold-start metric tag Deploys a real durable-configured Lambda alongside a plain guard function and asserts the cold-start aws.lambda.enhanced.invocations metric carries durable_function:true only for the durable one, exercising the fix in a9053473 end-to-end. Co-Authored-By: Claude Sonnet 5 --- .gitlab/datasources/test-suites.yaml | 1 + integration-tests/bin/app.ts | 4 ++ .../lib/stacks/durable-cold-start.ts | 71 +++++++++++++++++++ .../tests/durable-cold-start.test.ts | 70 ++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 integration-tests/lib/stacks/durable-cold-start.ts create mode 100644 integration-tests/tests/durable-cold-start.test.ts diff --git a/.gitlab/datasources/test-suites.yaml b/.gitlab/datasources/test-suites.yaml index 56ab4026f..d4eb86259 100644 --- a/.gitlab/datasources/test-suites.yaml +++ b/.gitlab/datasources/test-suites.yaml @@ -7,3 +7,4 @@ test_suites: - name: oom - name: lmi-oom - name: payload-size + - name: durable-cold-start diff --git a/integration-tests/bin/app.ts b/integration-tests/bin/app.ts index b34f6b910..9c0ee5e76 100644 --- a/integration-tests/bin/app.ts +++ b/integration-tests/bin/app.ts @@ -10,6 +10,7 @@ import {Oom} from '../lib/stacks/oom'; import {LmiOom} from '../lib/stacks/lmi-oom'; import {CustomMetrics} from '../lib/stacks/custom-metrics'; import {PayloadSize} from '../lib/stacks/payload-size'; +import {DurableColdStart} from '../lib/stacks/durable-cold-start'; import {AuthRoleStack} from '../lib/auth-role'; import {ACCOUNT, IDENTIFIER, REGION} from '../config'; import {CapacityProviderStack} from "../lib/capacity-provider"; @@ -54,6 +55,9 @@ const stacks = [ new PayloadSize(app, `${IDENTIFIER}-payload-size`, { env, }), + new DurableColdStart(app, `${IDENTIFIER}-durable-cold-start`, { + env, + }), ] // Tag all stacks so we can easily clean them up diff --git a/integration-tests/lib/stacks/durable-cold-start.ts b/integration-tests/lib/stacks/durable-cold-start.ts new file mode 100644 index 000000000..a40d54de1 --- /dev/null +++ b/integration-tests/lib/stacks/durable-cold-start.ts @@ -0,0 +1,71 @@ +import * as cdk from 'aws-cdk-lib'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { Construct } from 'constructs'; +import { + createLogGroup, + defaultDatadogEnvVariables, + defaultDatadogSecretPolicy, + getExtensionLayer, + getDefaultNodeLayer, + defaultNodeRuntime, +} from '../util'; + +// Durable-function cold-start metric tag test. +// The Lambda Telemetry API only reports a `platform.initStart` runtimeVersion +// containing "DurableFunction" for functions actually configured with +// `durableConfig`, so this stack deploys one durable and one non-durable +// function to exercise bottlecap's `durable_function:true` tagging on the +// cold-start `aws.lambda.enhanced.invocations` metric end-to-end. +export class DurableColdStart extends cdk.Stack { + constructor(scope: Construct, id: string, props: cdk.StackProps) { + super(scope, id, props); + + const extensionLayer = getExtensionLayer(this); + const nodeLayer = getDefaultNodeLayer(this); + + const durableFunctionName = `${id}-durable-lambda`; + const durableFunction = new lambda.Function(this, durableFunctionName, { + runtime: defaultNodeRuntime, + architecture: lambda.Architecture.ARM_64, + handler: '/opt/nodejs/node_modules/datadog-lambda-js/handler.handler', + code: lambda.Code.fromAsset('./lambda/default-node'), + functionName: durableFunctionName, + timeout: cdk.Duration.seconds(30), + memorySize: 256, + durableConfig: { + executionTimeout: cdk.Duration.minutes(5), + }, + environment: { + ...defaultDatadogEnvVariables, + DD_SERVICE: durableFunctionName, + DD_TRACE_ENABLED: 'true', + DD_LAMBDA_HANDLER: 'index.handler', + }, + logGroup: createLogGroup(this, durableFunctionName), + }); + durableFunction.addToRolePolicy(defaultDatadogSecretPolicy); + durableFunction.addLayers(extensionLayer); + durableFunction.addLayers(nodeLayer); + + const nonDurableFunctionName = `${id}-non-durable-lambda`; + const nonDurableFunction = new lambda.Function(this, nonDurableFunctionName, { + runtime: defaultNodeRuntime, + architecture: lambda.Architecture.ARM_64, + handler: '/opt/nodejs/node_modules/datadog-lambda-js/handler.handler', + code: lambda.Code.fromAsset('./lambda/default-node'), + functionName: nonDurableFunctionName, + timeout: cdk.Duration.seconds(30), + memorySize: 256, + environment: { + ...defaultDatadogEnvVariables, + DD_SERVICE: nonDurableFunctionName, + DD_TRACE_ENABLED: 'true', + DD_LAMBDA_HANDLER: 'index.handler', + }, + logGroup: createLogGroup(this, nonDurableFunctionName), + }); + nonDurableFunction.addToRolePolicy(defaultDatadogSecretPolicy); + nonDurableFunction.addLayers(extensionLayer); + nonDurableFunction.addLayers(nodeLayer); + } +} diff --git a/integration-tests/tests/durable-cold-start.test.ts b/integration-tests/tests/durable-cold-start.test.ts new file mode 100644 index 000000000..93e21727f --- /dev/null +++ b/integration-tests/tests/durable-cold-start.test.ts @@ -0,0 +1,70 @@ +// Verifies bottlecap holds the cold-start `aws.lambda.enhanced.invocations` +// metric until the Telemetry API's `platform.initStart` event reports the +// runtime, so the metric carries `durable_function:true` for a real +// durable-configured Lambda function (#1301). A plain (non-durable) function +// is invoked alongside as a guard against the tag being set unconditionally. +import { hasMetricWithTag } from './utils/datadog'; +import { forceColdStart, invokeLambda } from './utils/lambda'; +import { IDENTIFIER, DEFAULT_DATADOG_INDEXING_WAIT_MS } from '../config'; + +const stackName = `${IDENTIFIER}-durable-cold-start`; + +const INVOCATIONS_METRIC = 'aws.lambda.enhanced.invocations'; + +describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { + let invocationStartTime: number; + let metricsEndTime: number; + + const durableFunctionName = `${stackName}-durable-lambda`; + const nonDurableFunctionName = `${stackName}-non-durable-lambda`; + + beforeAll(async () => { + const functionNames = [durableFunctionName, nonDurableFunctionName]; + + await Promise.all(functionNames.map((fn) => forceColdStart(fn))); + + // Back up the query window by 60s so the metric bucket (which Datadog + // aligns to the rollup interval boundary, often before the invocation) + // falls inside the range we pass to /api/v1/query. + invocationStartTime = Date.now() - 60_000; + + // Durable functions reject unqualified-ARN invokes ("You cannot invoke a + // durable function using an unqualified ARN"); `:$LATEST` satisfies that + // without publishing a version. Metric queries below use the base + // (unqualified) function name, which is unaffected by this qualifier. + await Promise.all([ + invokeLambda(`${durableFunctionName}:$LATEST`), + invokeLambda(nonDurableFunctionName), + ]); + + await new Promise((resolve) => + setTimeout(resolve, DEFAULT_DATADOG_INDEXING_WAIT_MS), + ); + + metricsEndTime = Date.now(); + + console.log('Lambdas invoked and indexing wait complete'); + }, 1800000); + + it('durable function cold-start invocation metric should have durable_function:true', async () => { + const hasTag = await hasMetricWithTag( + INVOCATIONS_METRIC, + durableFunctionName, + 'durable_function:true', + invocationStartTime, + metricsEndTime, + ); + expect(hasTag).toBe(true); + }); + + it('non-durable function cold-start invocation metric should NOT have durable_function:true', async () => { + const hasTag = await hasMetricWithTag( + INVOCATIONS_METRIC, + nonDurableFunctionName, + 'durable_function:true', + invocationStartTime, + metricsEndTime, + ); + expect(hasTag).toBe(false); + }); +}); From ad2f801019d9987e9cc010ef248128e74d48e8ee Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:39:06 -0400 Subject: [PATCH 03/17] test: shorten comments in durable cold-start integration test Co-Authored-By: Claude Sonnet 5 --- .../lib/stacks/durable-cold-start.ts | 7 +------ .../tests/durable-cold-start.test.ts | 15 +++------------ 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/integration-tests/lib/stacks/durable-cold-start.ts b/integration-tests/lib/stacks/durable-cold-start.ts index a40d54de1..84ad87d09 100644 --- a/integration-tests/lib/stacks/durable-cold-start.ts +++ b/integration-tests/lib/stacks/durable-cold-start.ts @@ -10,12 +10,7 @@ import { defaultNodeRuntime, } from '../util'; -// Durable-function cold-start metric tag test. -// The Lambda Telemetry API only reports a `platform.initStart` runtimeVersion -// containing "DurableFunction" for functions actually configured with -// `durableConfig`, so this stack deploys one durable and one non-durable -// function to exercise bottlecap's `durable_function:true` tagging on the -// cold-start `aws.lambda.enhanced.invocations` metric end-to-end. +// Deploys a durable and a non-durable function to test the cold-start durable_function tag. export class DurableColdStart extends cdk.Stack { constructor(scope: Construct, id: string, props: cdk.StackProps) { super(scope, id, props); diff --git a/integration-tests/tests/durable-cold-start.test.ts b/integration-tests/tests/durable-cold-start.test.ts index 93e21727f..ba943fed7 100644 --- a/integration-tests/tests/durable-cold-start.test.ts +++ b/integration-tests/tests/durable-cold-start.test.ts @@ -1,8 +1,4 @@ -// Verifies bottlecap holds the cold-start `aws.lambda.enhanced.invocations` -// metric until the Telemetry API's `platform.initStart` event reports the -// runtime, so the metric carries `durable_function:true` for a real -// durable-configured Lambda function (#1301). A plain (non-durable) function -// is invoked alongside as a guard against the tag being set unconditionally. +// Verifies the cold-start invocations metric has durable_function:true only for a durable function. import { hasMetricWithTag } from './utils/datadog'; import { forceColdStart, invokeLambda } from './utils/lambda'; import { IDENTIFIER, DEFAULT_DATADOG_INDEXING_WAIT_MS } from '../config'; @@ -23,15 +19,10 @@ describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { await Promise.all(functionNames.map((fn) => forceColdStart(fn))); - // Back up the query window by 60s so the metric bucket (which Datadog - // aligns to the rollup interval boundary, often before the invocation) - // falls inside the range we pass to /api/v1/query. + // Back up 60s so the metric's rollup bucket falls inside the query range. invocationStartTime = Date.now() - 60_000; - // Durable functions reject unqualified-ARN invokes ("You cannot invoke a - // durable function using an unqualified ARN"); `:$LATEST` satisfies that - // without publishing a version. Metric queries below use the base - // (unqualified) function name, which is unaffected by this qualifier. + // Durable functions reject unqualified-ARN invokes; `:$LATEST` avoids publishing a version. await Promise.all([ invokeLambda(`${durableFunctionName}:$LATEST`), invokeLambda(nonDurableFunctionName), From 1e22255e7f70310176267d9520485785f0b0ea67 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:43:19 -0400 Subject: [PATCH 04/17] refactor: shorten comments around pending_first_invocation_metric Co-Authored-By: Claude Sonnet 5 --- .../src/lifecycle/invocation/processor.rs | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 6552b04d5..cbeed3af2 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -111,16 +111,8 @@ pub struct Processor { /// on `platform.report`. This flag ensures whichever event arrives first wins and the other is skipped, /// preventing double counting. init_duration_metric_emitted: bool, - /// Timestamp of the invocation metric for the sandbox's first (cold start) invocation in - /// On-Demand mode, held back from `enhanced_metrics` until `platform.initStart` is processed. - /// - /// The Extensions API's `/next` poll (which drives `on_invoke_event`) has no artificial - /// delay, while `platform.initStart` is delivered through the buffered Telemetry API. In - /// On-Demand mode the Invoke event consistently reaches the processor first, so emitting the - /// invocation metric immediately would race `set_durable_function_tag` and miss the - /// `durable_function:true` tag on invocation #1. Holding the metric here until - /// `on_platform_init_start` (or the next invocation, or shutdown) flushes it ensures the tag - /// is applied before the metric is counted. + /// Timestamp of the On-Demand cold-start invocation metric, held back until + /// `platform.initStart` sets the durable tag so it's not missed on invocation #1. pending_first_invocation_metric: Option, } @@ -172,14 +164,10 @@ impl Processor { /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { - // Flush any invocation metric held back from a previous cold start. This is a fallback - // for the case where `platform.initStart` never arrived (or arrived very late) before - // this next invocation; see `pending_first_invocation_metric` for why it's held back. + // Fallback flush in case `platform.initStart` never arrived before this invocation. self.flush_pending_first_invocation_metric(); - // In On-Demand mode, the very first invocation of a cold sandbox races - // `platform.initStart` (see `pending_first_invocation_metric`); hold its invocation - // metric back instead of emitting it inline below. + // See `pending_first_invocation_metric`. let is_on_demand_cold_start = !self.aws_config.is_managed_instance_mode() && self.context_buffer.is_empty(); @@ -247,8 +235,7 @@ impl Processor { .add_enhanced_metric_data(&request_id, enhanced_metric_offsets); } - // Increment the invocation metric, unless it's an On-Demand cold start, in which case - // it's held back until `platform.initStart` (or a fallback) flushes it. + // Hold back the metric for an On-Demand cold start; see `pending_first_invocation_metric`. if is_on_demand_cold_start { self.pending_first_invocation_metric = Some(timestamp); } else { @@ -285,8 +272,7 @@ impl Processor { } } - /// Emits the invocation metric held back by `on_invoke_event` for an On-Demand cold start, - /// if one is pending. See `pending_first_invocation_metric` for why it's held back. + /// Flushes the metric held back by `on_invoke_event`, if any. fn flush_pending_first_invocation_metric(&mut self) { if let Some(timestamp) = self.pending_first_invocation_metric.take() { self.enhanced_metrics.increment_invocation_metric(timestamp); @@ -375,8 +361,7 @@ impl Processor { { self.enhanced_metrics.set_durable_function_tag(); } - // Flush the held-back invocation metric now that the durable_function tag (if any) has - // been set, so invocation #1 is counted with the correct tag. + // Flush the held-back metric now that the durable tag (if any) has been set. self.flush_pending_first_invocation_metric(); let start_time: i64 = SystemTime::from(time) @@ -1030,8 +1015,7 @@ impl Processor { } pub fn on_shutdown_event(&mut self) { - // Don't drop the invocation metric if the sandbox shuts down before - // `platform.initStart` (or a second invocation) had a chance to flush it. + // Fallback flush in case the sandbox shuts down before it's otherwise flushed. self.flush_pending_first_invocation_metric(); let now = std::time::SystemTime::now() From e7646caabb41edd8b0bca753c25f1b5c62b7ce37 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:56:09 -0400 Subject: [PATCH 05/17] refactor: drop on_invoke_event fallback flush, rename pending metric field platform.initStart not arriving before the next invocation is not a realistic scenario, so drop that fallback flush and the test covering it. Renamed pending_first_invocation_metric to pending_first_invocation_metric_timestamp to reflect that it stores a timestamp, and clarified "cold-start invocation metric in On-Demand mode" comments. Co-Authored-By: Claude Sonnet 5 --- .../src/lifecycle/invocation/processor.rs | 42 ++++++------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index cbeed3af2..2a8c637e9 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -111,9 +111,9 @@ pub struct Processor { /// on `platform.report`. This flag ensures whichever event arrives first wins and the other is skipped, /// preventing double counting. init_duration_metric_emitted: bool, - /// Timestamp of the On-Demand cold-start invocation metric, held back until + /// Timestamp of the cold-start invocation metric in On-Demand mode, held back until /// `platform.initStart` sets the durable tag so it's not missed on invocation #1. - pending_first_invocation_metric: Option, + pending_first_invocation_metric_timestamp: Option, } impl Processor { @@ -157,17 +157,14 @@ impl Processor { durable_context_tx, restore_time: None, init_duration_metric_emitted: false, - pending_first_invocation_metric: None, + pending_first_invocation_metric_timestamp: None, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { - // Fallback flush in case `platform.initStart` never arrived before this invocation. - self.flush_pending_first_invocation_metric(); - - // See `pending_first_invocation_metric`. + // See `pending_first_invocation_metric_timestamp`. let is_on_demand_cold_start = !self.aws_config.is_managed_instance_mode() && self.context_buffer.is_empty(); @@ -235,9 +232,9 @@ impl Processor { .add_enhanced_metric_data(&request_id, enhanced_metric_offsets); } - // Hold back the metric for an On-Demand cold start; see `pending_first_invocation_metric`. + // Hold back the metric for a cold start in On-Demand mode; see `pending_first_invocation_metric_timestamp`. if is_on_demand_cold_start { - self.pending_first_invocation_metric = Some(timestamp); + self.pending_first_invocation_metric_timestamp = Some(timestamp); } else { self.enhanced_metrics.increment_invocation_metric(timestamp); } @@ -274,7 +271,7 @@ impl Processor { /// Flushes the metric held back by `on_invoke_event`, if any. fn flush_pending_first_invocation_metric(&mut self) { - if let Some(timestamp) = self.pending_first_invocation_metric.take() { + if let Some(timestamp) = self.pending_first_invocation_metric_timestamp.take() { self.enhanced_metrics.increment_invocation_metric(timestamp); } } @@ -2131,7 +2128,9 @@ mod tests { // `platform.initStart`. The invocation metric must not be emitted yet. processor.on_invoke_event("req-1".to_string()); assert!( - processor.pending_first_invocation_metric.is_some(), + processor + .pending_first_invocation_metric_timestamp + .is_some(), "Expected the cold start invocation metric to be held back" ); @@ -2149,7 +2148,9 @@ mod tests { ); assert!( - processor.pending_first_invocation_metric.is_none(), + processor + .pending_first_invocation_metric_timestamp + .is_none(), "Expected the held-back invocation metric to be flushed by on_platform_init_start" ); @@ -2178,23 +2179,6 @@ mod tests { ); } - #[tokio::test] - async fn test_on_invoke_event_flushes_pending_metric_if_init_start_never_arrives() { - let mut processor = setup(); - - // First invocation of a cold sandbox: metric held back. - processor.on_invoke_event("req-1".to_string()); - assert!(processor.pending_first_invocation_metric.is_some()); - - // A second invocation arrives without platform.initStart ever showing up. The held-back - // metric from req-1 must still be flushed (as a fallback), not lost. - processor.on_invoke_event("req-2".to_string()); - assert!( - processor.pending_first_invocation_metric.is_none(), - "Expected the pending metric to be flushed as a fallback on the next invocation" - ); - } - #[tokio::test] async fn test_init_duration_emitted_from_init_report() { let mut processor = setup(); From 35ee072fcd350f3a18531dfd17b2f768ed011ceb Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:05:09 -0400 Subject: [PATCH 06/17] fix(metrics): don't hold back invocation metric if initStart already ran If the Invoke event reaches the processor after platform.initStart (the reverse of the usual On-Demand race), on_invoke_event would still see an empty context_buffer and hold the metric back a second time, with nothing left to flush it until shutdown. Track whether platform.initStart already ran and skip the hold-back in that case. Co-Authored-By: Claude Sonnet 5 --- .../src/lifecycle/invocation/processor.rs | 81 ++++++++++++++++++- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 2a8c637e9..db5e0cca0 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -63,6 +63,7 @@ pub const DATADOG_INVOCATION_ERROR_KEY: &str = "x-datadog-invocation-error"; const DATADOG_SPAN_ID_KEY: &str = "x-datadog-span-id"; const TAG_SAMPLING_PRIORITY: &str = "_sampling_priority_v1"; +#[allow(clippy::struct_excessive_bools)] pub struct Processor { /// Buffer containing context of the previous 5 invocations context_buffer: ContextBuffer, @@ -114,6 +115,14 @@ pub struct Processor { /// Timestamp of the cold-start invocation metric in On-Demand mode, held back until /// `platform.initStart` sets the durable tag so it's not missed on invocation #1. pending_first_invocation_metric_timestamp: Option, + /// Whether `platform.initStart` has already been processed for this sandbox. + /// + /// Covers the case where the Invoke event reaches the processor after `platform.initStart` + /// (the reverse of the usual On-Demand race). Without this check, `on_invoke_event` would see + /// an empty `context_buffer` (init start doesn't insert a context in On-Demand mode) and hold + /// the metric back again, even though the durable tag is already resolved — and nothing would + /// flush it until `on_shutdown_event`, since `platform.initStart` won't fire again. + platform_init_start_processed: bool, } impl Processor { @@ -158,15 +167,17 @@ impl Processor { restore_time: None, init_duration_metric_emitted: false, pending_first_invocation_metric_timestamp: None, + platform_init_start_processed: false, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { - // See `pending_first_invocation_metric_timestamp`. - let is_on_demand_cold_start = - !self.aws_config.is_managed_instance_mode() && self.context_buffer.is_empty(); + // See `pending_first_invocation_metric_timestamp` and `platform_init_start_processed`. + let is_on_demand_cold_start = !self.aws_config.is_managed_instance_mode() + && self.context_buffer.is_empty() + && !self.platform_init_start_processed; // In Managed Instance mode, if awaiting the first invocation after init, find and update the empty context created on init start if self.aws_config.is_managed_instance_mode() && self.awaiting_first_invocation { @@ -352,6 +363,8 @@ impl Processor { /// This is used to create a cold start span, since this telemetry event does not /// provide a `request_id`, we try to guess which invocation is the cold start. pub fn on_platform_init_start(&mut self, time: DateTime, runtime_version: Option) { + self.platform_init_start_processed = true; + if runtime_version .as_deref() .is_some_and(|rv| rv.contains("DurableFunction")) @@ -2179,6 +2192,68 @@ mod tests { ); } + #[tokio::test] + async fn test_on_invoke_event_does_not_hold_metric_if_init_start_already_processed() { + let mut processor = setup(); + + // `platform.initStart` reaches the processor first (reverse of the usual On-Demand + // race). It resolves the durable tag but finds no context to attach a cold start span + // to yet, since On-Demand mode only looks up an existing context here. + processor.on_platform_init_start( + Utc::now(), + Some("python:3.14.DurableFunction.v6".to_string()), + ); + assert!(processor.platform_init_start_processed); + assert!( + processor + .pending_first_invocation_metric_timestamp + .is_none(), + "Nothing should be pending yet, since no invocation has happened" + ); + + let now: i64 = std::time::UNIX_EPOCH + .elapsed() + .expect("unable to poll clock, unrecoverable") + .as_secs() + .try_into() + .unwrap_or_default(); + + // The Invoke event arrives afterwards. Without `platform_init_start_processed`, this + // would see an empty context_buffer and hold the metric back again — with nothing left + // to flush it until shutdown, since `platform.initStart` won't fire a second time. + processor.on_invoke_event("req-1".to_string()); + assert!( + processor + .pending_first_invocation_metric_timestamp + .is_none(), + "Expected the invocation metric to be emitted immediately, not held back" + ); + + let runtime = processor + .runtime + .clone() + .expect("runtime should have been resolved by on_invoke_event"); + let ts = (now / 10) * 10; + let expected_tags = dogstatsd::metric::SortedTags::parse(&format!( + "cold_start:true,durable_function:true,runtime:{runtime}" + )) + .ok(); + let entry = processor + .enhanced_metrics + .aggr_handle + .get_entry_by_id( + crate::metrics::enhanced::constants::INVOCATIONS_METRIC.into(), + expected_tags, + ts, + ) + .await + .unwrap(); + assert!( + entry.is_some(), + "Expected the immediately-emitted invocation metric to carry the durable_function:true tag" + ); + } + #[tokio::test] async fn test_init_duration_emitted_from_init_report() { let mut processor = setup(); From d39d15ed9e407639b8283c929bce804f4ce44094 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:09:32 -0400 Subject: [PATCH 07/17] Revert "fix(metrics): don't hold back invocation metric if initStart already ran" This reverts commit 35ee072fcd350f3a18531dfd17b2f768ed011ceb. --- .../src/lifecycle/invocation/processor.rs | 81 +------------------ 1 file changed, 3 insertions(+), 78 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index db5e0cca0..2a8c637e9 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -63,7 +63,6 @@ pub const DATADOG_INVOCATION_ERROR_KEY: &str = "x-datadog-invocation-error"; const DATADOG_SPAN_ID_KEY: &str = "x-datadog-span-id"; const TAG_SAMPLING_PRIORITY: &str = "_sampling_priority_v1"; -#[allow(clippy::struct_excessive_bools)] pub struct Processor { /// Buffer containing context of the previous 5 invocations context_buffer: ContextBuffer, @@ -115,14 +114,6 @@ pub struct Processor { /// Timestamp of the cold-start invocation metric in On-Demand mode, held back until /// `platform.initStart` sets the durable tag so it's not missed on invocation #1. pending_first_invocation_metric_timestamp: Option, - /// Whether `platform.initStart` has already been processed for this sandbox. - /// - /// Covers the case where the Invoke event reaches the processor after `platform.initStart` - /// (the reverse of the usual On-Demand race). Without this check, `on_invoke_event` would see - /// an empty `context_buffer` (init start doesn't insert a context in On-Demand mode) and hold - /// the metric back again, even though the durable tag is already resolved — and nothing would - /// flush it until `on_shutdown_event`, since `platform.initStart` won't fire again. - platform_init_start_processed: bool, } impl Processor { @@ -167,17 +158,15 @@ impl Processor { restore_time: None, init_duration_metric_emitted: false, pending_first_invocation_metric_timestamp: None, - platform_init_start_processed: false, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { - // See `pending_first_invocation_metric_timestamp` and `platform_init_start_processed`. - let is_on_demand_cold_start = !self.aws_config.is_managed_instance_mode() - && self.context_buffer.is_empty() - && !self.platform_init_start_processed; + // See `pending_first_invocation_metric_timestamp`. + let is_on_demand_cold_start = + !self.aws_config.is_managed_instance_mode() && self.context_buffer.is_empty(); // In Managed Instance mode, if awaiting the first invocation after init, find and update the empty context created on init start if self.aws_config.is_managed_instance_mode() && self.awaiting_first_invocation { @@ -363,8 +352,6 @@ impl Processor { /// This is used to create a cold start span, since this telemetry event does not /// provide a `request_id`, we try to guess which invocation is the cold start. pub fn on_platform_init_start(&mut self, time: DateTime, runtime_version: Option) { - self.platform_init_start_processed = true; - if runtime_version .as_deref() .is_some_and(|rv| rv.contains("DurableFunction")) @@ -2192,68 +2179,6 @@ mod tests { ); } - #[tokio::test] - async fn test_on_invoke_event_does_not_hold_metric_if_init_start_already_processed() { - let mut processor = setup(); - - // `platform.initStart` reaches the processor first (reverse of the usual On-Demand - // race). It resolves the durable tag but finds no context to attach a cold start span - // to yet, since On-Demand mode only looks up an existing context here. - processor.on_platform_init_start( - Utc::now(), - Some("python:3.14.DurableFunction.v6".to_string()), - ); - assert!(processor.platform_init_start_processed); - assert!( - processor - .pending_first_invocation_metric_timestamp - .is_none(), - "Nothing should be pending yet, since no invocation has happened" - ); - - let now: i64 = std::time::UNIX_EPOCH - .elapsed() - .expect("unable to poll clock, unrecoverable") - .as_secs() - .try_into() - .unwrap_or_default(); - - // The Invoke event arrives afterwards. Without `platform_init_start_processed`, this - // would see an empty context_buffer and hold the metric back again — with nothing left - // to flush it until shutdown, since `platform.initStart` won't fire a second time. - processor.on_invoke_event("req-1".to_string()); - assert!( - processor - .pending_first_invocation_metric_timestamp - .is_none(), - "Expected the invocation metric to be emitted immediately, not held back" - ); - - let runtime = processor - .runtime - .clone() - .expect("runtime should have been resolved by on_invoke_event"); - let ts = (now / 10) * 10; - let expected_tags = dogstatsd::metric::SortedTags::parse(&format!( - "cold_start:true,durable_function:true,runtime:{runtime}" - )) - .ok(); - let entry = processor - .enhanced_metrics - .aggr_handle - .get_entry_by_id( - crate::metrics::enhanced::constants::INVOCATIONS_METRIC.into(), - expected_tags, - ts, - ) - .await - .unwrap(); - assert!( - entry.is_some(), - "Expected the immediately-emitted invocation metric to carry the durable_function:true tag" - ); - } - #[tokio::test] async fn test_init_duration_emitted_from_init_report() { let mut processor = setup(); From 418cf992e93aac2f9cd930905fcbbbd5df8e4a99 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:11:16 -0400 Subject: [PATCH 08/17] docs: note that platform.initStart is assumed to follow Invoke Co-Authored-By: Claude Sonnet 5 --- bottlecap/src/lifecycle/invocation/processor.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 2a8c637e9..4501d1e49 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -113,6 +113,9 @@ pub struct Processor { init_duration_metric_emitted: bool, /// Timestamp of the cold-start invocation metric in On-Demand mode, held back until /// `platform.initStart` sets the durable tag so it's not missed on invocation #1. + /// + /// This assumes `platform.initStart` always arrives after the Invoke event, which is what + /// we've observed in production. pending_first_invocation_metric_timestamp: Option, } From ae2719bb72d6e7059e78f103d5cfdfd0ad10b45d Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:22:28 -0400 Subject: [PATCH 09/17] test: assert durable_function tag persists on a warm invocation too Invoke both functions a second time after the cold start and check the tag in a separate time window, to confirm the tag isn't only set on invocation #1 but persists across warm invocations. Co-Authored-By: Claude Sonnet 5 --- .../tests/durable-cold-start.test.ts | 74 +++++++++++++------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/integration-tests/tests/durable-cold-start.test.ts b/integration-tests/tests/durable-cold-start.test.ts index ba943fed7..0df9af154 100644 --- a/integration-tests/tests/durable-cold-start.test.ts +++ b/integration-tests/tests/durable-cold-start.test.ts @@ -1,4 +1,5 @@ -// Verifies the cold-start invocations metric has durable_function:true only for a durable function. +// Verifies the invocations metric has durable_function:true only for a durable function, on +// both the cold-start invocation and a subsequent warm invocation. import { hasMetricWithTag } from './utils/datadog'; import { forceColdStart, invokeLambda } from './utils/lambda'; import { IDENTIFIER, DEFAULT_DATADOG_INDEXING_WAIT_MS } from '../config'; @@ -8,42 +9,51 @@ const stackName = `${IDENTIFIER}-durable-cold-start`; const INVOCATIONS_METRIC = 'aws.lambda.enhanced.invocations'; describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { - let invocationStartTime: number; - let metricsEndTime: number; + let coldStartWindowStart: number; + let coldStartWindowEnd: number; + let warmInvokeWindowStart: number; + let warmInvokeWindowEnd: number; const durableFunctionName = `${stackName}-durable-lambda`; const nonDurableFunctionName = `${stackName}-non-durable-lambda`; - beforeAll(async () => { - const functionNames = [durableFunctionName, nonDurableFunctionName]; - - await Promise.all(functionNames.map((fn) => forceColdStart(fn))); - - // Back up 60s so the metric's rollup bucket falls inside the query range. - invocationStartTime = Date.now() - 60_000; - - // Durable functions reject unqualified-ARN invokes; `:$LATEST` avoids publishing a version. - await Promise.all([ + // Durable functions reject unqualified-ARN invokes; `:$LATEST` avoids publishing a version. + const invokeBoth = () => + Promise.all([ invokeLambda(`${durableFunctionName}:$LATEST`), invokeLambda(nonDurableFunctionName), ]); - await new Promise((resolve) => + const waitForIndexing = () => + new Promise((resolve) => setTimeout(resolve, DEFAULT_DATADOG_INDEXING_WAIT_MS), ); - metricsEndTime = Date.now(); + beforeAll(async () => { + const functionNames = [durableFunctionName, nonDurableFunctionName]; + await Promise.all(functionNames.map((fn) => forceColdStart(fn))); + + // Back up 60s so the metric's rollup bucket falls inside the query range. + coldStartWindowStart = Date.now() - 60_000; + await invokeBoth(); // invocation #1 (cold start) + await waitForIndexing(); + coldStartWindowEnd = Date.now(); - console.log('Lambdas invoked and indexing wait complete'); - }, 1800000); + warmInvokeWindowStart = Date.now() - 60_000; + await invokeBoth(); // invocation #2 (warm) + await waitForIndexing(); + warmInvokeWindowEnd = Date.now(); + + console.log('Lambdas invoked twice and indexing waits complete'); + }, 3600000); it('durable function cold-start invocation metric should have durable_function:true', async () => { const hasTag = await hasMetricWithTag( INVOCATIONS_METRIC, durableFunctionName, 'durable_function:true', - invocationStartTime, - metricsEndTime, + coldStartWindowStart, + coldStartWindowEnd, ); expect(hasTag).toBe(true); }); @@ -53,8 +63,30 @@ describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { INVOCATIONS_METRIC, nonDurableFunctionName, 'durable_function:true', - invocationStartTime, - metricsEndTime, + coldStartWindowStart, + coldStartWindowEnd, + ); + expect(hasTag).toBe(false); + }); + + it('durable function warm invocation metric should still have durable_function:true', async () => { + const hasTag = await hasMetricWithTag( + INVOCATIONS_METRIC, + durableFunctionName, + 'durable_function:true', + warmInvokeWindowStart, + warmInvokeWindowEnd, + ); + expect(hasTag).toBe(true); + }); + + it('non-durable function warm invocation metric should still NOT have durable_function:true', async () => { + const hasTag = await hasMetricWithTag( + INVOCATIONS_METRIC, + nonDurableFunctionName, + 'durable_function:true', + warmInvokeWindowStart, + warmInvokeWindowEnd, ); expect(hasTag).toBe(false); }); From d12559f9b76ecb8054783cfdc51372c51560e698 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:28:31 -0400 Subject: [PATCH 10/17] test: invoke twice back-to-back with a single indexing wait Replaces the two-window/two-5-min-indexing-wait design with one window and one wait, comparing tagged vs. total invocation counts (new getMetricCountWithTag helper) instead of checking existence per window. Co-Authored-By: Claude Sonnet 5 --- .../tests/durable-cold-start.test.ts | 88 ++++++++----------- integration-tests/tests/utils/datadog.ts | 34 +++++++ 2 files changed, 72 insertions(+), 50 deletions(-) diff --git a/integration-tests/tests/durable-cold-start.test.ts b/integration-tests/tests/durable-cold-start.test.ts index 0df9af154..5a9f48a85 100644 --- a/integration-tests/tests/durable-cold-start.test.ts +++ b/integration-tests/tests/durable-cold-start.test.ts @@ -1,6 +1,6 @@ -// Verifies the invocations metric has durable_function:true only for a durable function, on -// both the cold-start invocation and a subsequent warm invocation. -import { hasMetricWithTag } from './utils/datadog'; +// Verifies the invocations metric has durable_function:true only for a durable function, and +// that the tag persists across a second (warm) invocation instead of only being set on cold start. +import { getMetricCount, getMetricCountWithTag } from './utils/datadog'; import { forceColdStart, invokeLambda } from './utils/lambda'; import { IDENTIFIER, DEFAULT_DATADOG_INDEXING_WAIT_MS } from '../config'; @@ -9,10 +9,8 @@ const stackName = `${IDENTIFIER}-durable-cold-start`; const INVOCATIONS_METRIC = 'aws.lambda.enhanced.invocations'; describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { - let coldStartWindowStart: number; - let coldStartWindowEnd: number; - let warmInvokeWindowStart: number; - let warmInvokeWindowEnd: number; + let windowStart: number; + let windowEnd: number; const durableFunctionName = `${stackName}-durable-lambda`; const nonDurableFunctionName = `${stackName}-non-durable-lambda`; @@ -24,70 +22,60 @@ describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { invokeLambda(nonDurableFunctionName), ]); - const waitForIndexing = () => - new Promise((resolve) => - setTimeout(resolve, DEFAULT_DATADOG_INDEXING_WAIT_MS), - ); - beforeAll(async () => { const functionNames = [durableFunctionName, nonDurableFunctionName]; await Promise.all(functionNames.map((fn) => forceColdStart(fn))); // Back up 60s so the metric's rollup bucket falls inside the query range. - coldStartWindowStart = Date.now() - 60_000; - await invokeBoth(); // invocation #1 (cold start) - await waitForIndexing(); - coldStartWindowEnd = Date.now(); + windowStart = Date.now() - 60_000; + + // Invoke twice back-to-back (cold start, then warm) and wait for indexing once, rather than + // waiting after each invocation. Whether the tag was set on both invocations (rather than + // only the first) is checked below by comparing tagged vs. total invocation counts. + await invokeBoth(); + await invokeBoth(); - warmInvokeWindowStart = Date.now() - 60_000; - await invokeBoth(); // invocation #2 (warm) - await waitForIndexing(); - warmInvokeWindowEnd = Date.now(); + await new Promise((resolve) => + setTimeout(resolve, DEFAULT_DATADOG_INDEXING_WAIT_MS), + ); + windowEnd = Date.now(); - console.log('Lambdas invoked twice and indexing waits complete'); - }, 3600000); + console.log('Lambdas invoked twice and indexing wait complete'); + }, 1800000); - it('durable function cold-start invocation metric should have durable_function:true', async () => { - const hasTag = await hasMetricWithTag( + it('durable function invocation metric should have durable_function:true on every invocation', async () => { + const totalCount = await getMetricCount( INVOCATIONS_METRIC, durableFunctionName, - 'durable_function:true', - coldStartWindowStart, - coldStartWindowEnd, + windowStart, + windowEnd, ); - expect(hasTag).toBe(true); - }); - - it('non-durable function cold-start invocation metric should NOT have durable_function:true', async () => { - const hasTag = await hasMetricWithTag( + const taggedCount = await getMetricCountWithTag( INVOCATIONS_METRIC, - nonDurableFunctionName, + durableFunctionName, 'durable_function:true', - coldStartWindowStart, - coldStartWindowEnd, + windowStart, + windowEnd, ); - expect(hasTag).toBe(false); + expect(totalCount).toBeGreaterThanOrEqual(2); + expect(taggedCount).toBe(totalCount); }); - it('durable function warm invocation metric should still have durable_function:true', async () => { - const hasTag = await hasMetricWithTag( + it('non-durable function invocation metric should NOT have durable_function:true on any invocation', async () => { + const totalCount = await getMetricCount( INVOCATIONS_METRIC, - durableFunctionName, - 'durable_function:true', - warmInvokeWindowStart, - warmInvokeWindowEnd, + nonDurableFunctionName, + windowStart, + windowEnd, ); - expect(hasTag).toBe(true); - }); - - it('non-durable function warm invocation metric should still NOT have durable_function:true', async () => { - const hasTag = await hasMetricWithTag( + const taggedCount = await getMetricCountWithTag( INVOCATIONS_METRIC, nonDurableFunctionName, 'durable_function:true', - warmInvokeWindowStart, - warmInvokeWindowEnd, + windowStart, + windowEnd, ); - expect(hasTag).toBe(false); + expect(totalCount).toBeGreaterThanOrEqual(2); + expect(taggedCount).toBe(0); }); }); diff --git a/integration-tests/tests/utils/datadog.ts b/integration-tests/tests/utils/datadog.ts index f8d2bcb31..cd72d1fed 100644 --- a/integration-tests/tests/utils/datadog.ts +++ b/integration-tests/tests/utils/datadog.ts @@ -370,6 +370,40 @@ export async function getMetricCount( return pointlist.reduce((acc, [, value]) => acc + (value || 0), 0); } +/** + * Same as `getMetricCount`, but filtered by an additional tag (e.g. "durable_function:true"). + * Used to check whether every emission of a metric over a window carried a given tag, by + * comparing against the untagged `getMetricCount` total for the same window. + */ +export async function getMetricCountWithTag( + metricName: string, + functionName: string, + tagFilter: string, + fromTime: number, + toTime: number, +): Promise { + const baseFunctionName = getServiceName(functionName).toLowerCase(); + const query = `sum:${metricName}{functionname:${baseFunctionName},${tagFilter}}.as_count()`; + + console.log(`Querying metric count with tag filter: ${query}`); + + const response = await requestWithRetry(() => datadogClient.get('/api/v1/query', { + params: { + query, + from: Math.floor(fromTime / 1000), + to: Math.floor(toTime / 1000), + }, + }), query); + + const series = response.data.series || []; + if (series.length === 0) { + return 0; + } + + const pointlist: [number, number][] = series[0].pointlist || []; + return pointlist.reduce((acc, [, value]) => acc + (value || 0), 0); +} + async function getMetrics( metricName: string, functionName: string, From 53bd001e49871b8b91a3a274652c5e74f6ef0aff Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:32:38 -0400 Subject: [PATCH 11/17] test: assert exact invocation count; docs: note against over-defensive checks Each function is invoked exactly twice, so assert toBe(2) instead of a loose lower bound. Also add a guideline to AGENTS.md against unnecessary defensive checks/tests. Co-Authored-By: Claude Sonnet 5 --- AGENTS.md | 4 ++++ integration-tests/tests/durable-cold-start.test.ts | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bebb7c640..ca7823624 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,10 @@ The Datadog Lambda Extension ("Bottlecap") is an AWS Lambda Extension written in All Rust code lives under `bottlecap/`. Run these from that directory unless noted. +## Code Style + +- Do not be overly defensive and add unnecessary checks or tests unless you have a good reason. + ## Pull Requests - When creating a PR, follow the PR template at `.github/pull_request_template.md`. diff --git a/integration-tests/tests/durable-cold-start.test.ts b/integration-tests/tests/durable-cold-start.test.ts index 5a9f48a85..21d2d866e 100644 --- a/integration-tests/tests/durable-cold-start.test.ts +++ b/integration-tests/tests/durable-cold-start.test.ts @@ -57,7 +57,7 @@ describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { windowStart, windowEnd, ); - expect(totalCount).toBeGreaterThanOrEqual(2); + expect(totalCount).toBe(2); expect(taggedCount).toBe(totalCount); }); @@ -75,7 +75,7 @@ describe('Durable Function Cold-Start Metric Tag Integration Tests', () => { windowStart, windowEnd, ); - expect(totalCount).toBeGreaterThanOrEqual(2); + expect(totalCount).toBe(2); expect(taggedCount).toBe(0); }); }); From 4272990448bcdaa29bc1d0f7dc05c21c6b9c34da Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:16:32 -0400 Subject: [PATCH 12/17] fix: track first On-Demand invocation with a dedicated flag, not buffer emptiness context_buffer.is_empty() is also true between warm invocations once platform.report is processed (an unordered, separate path from on_invoke_event), so it could wrongly re-trigger the cold-start hold-back on later invocations and drop their metric. Track it explicitly instead. Also fix a test that recomputed `now` after on_invoke_event to derive the expected metric bucket, which could mismatch the actual timestamp if the call straddled a 10s boundary; capture the held-back timestamp instead. Co-Authored-By: Claude Sonnet 5 --- .../src/lifecycle/invocation/processor.rs | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 4501d1e49..dc31af8ec 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -63,6 +63,7 @@ pub const DATADOG_INVOCATION_ERROR_KEY: &str = "x-datadog-invocation-error"; const DATADOG_SPAN_ID_KEY: &str = "x-datadog-span-id"; const TAG_SAMPLING_PRIORITY: &str = "_sampling_priority_v1"; +#[allow(clippy::struct_excessive_bools)] pub struct Processor { /// Buffer containing context of the previous 5 invocations context_buffer: ContextBuffer, @@ -117,6 +118,13 @@ pub struct Processor { /// This assumes `platform.initStart` always arrives after the Invoke event, which is what /// we've observed in production. pending_first_invocation_metric_timestamp: Option, + /// Whether the first On-Demand invocation has already been seen. + /// + /// Used instead of `context_buffer.is_empty()` to detect the first invocation, since the + /// buffer is also emptied between warm invocations once `platform.report` is processed + /// (an unordered, separate path from `on_invoke_event`), which would otherwise make later + /// invocations look like cold starts too and hold back their metric indefinitely. + first_on_demand_invocation_seen: bool, } impl Processor { @@ -161,15 +169,19 @@ impl Processor { restore_time: None, init_duration_metric_emitted: false, pending_first_invocation_metric_timestamp: None, + first_on_demand_invocation_seen: false, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { - // See `pending_first_invocation_metric_timestamp`. + // See `pending_first_invocation_metric_timestamp` and `first_on_demand_invocation_seen`. let is_on_demand_cold_start = - !self.aws_config.is_managed_instance_mode() && self.context_buffer.is_empty(); + !self.aws_config.is_managed_instance_mode() && !self.first_on_demand_invocation_seen; + if is_on_demand_cold_start { + self.first_on_demand_invocation_seen = true; + } // In Managed Instance mode, if awaiting the first invocation after init, find and update the empty context created on init start if self.aws_config.is_managed_instance_mode() && self.awaiting_first_invocation { @@ -2130,19 +2142,9 @@ mod tests { // Simulate the On-Demand race: the Invoke event reaches the processor before // `platform.initStart`. The invocation metric must not be emitted yet. processor.on_invoke_event("req-1".to_string()); - assert!( - processor - .pending_first_invocation_metric_timestamp - .is_some(), - "Expected the cold start invocation metric to be held back" - ); - - let now: i64 = std::time::UNIX_EPOCH - .elapsed() - .expect("unable to poll clock, unrecoverable") - .as_secs() - .try_into() - .unwrap_or_default(); + let held_back_timestamp = processor + .pending_first_invocation_metric_timestamp + .expect("Expected the cold start invocation metric to be held back"); // `platform.initStart` arrives afterwards and reports a durable runtime. processor.on_platform_init_start( @@ -2161,7 +2163,7 @@ mod tests { .runtime .clone() .expect("runtime should have been resolved by on_invoke_event"); - let ts = (now / 10) * 10; + let ts = (held_back_timestamp / 10) * 10; let expected_tags = dogstatsd::metric::SortedTags::parse(&format!( "cold_start:true,durable_function:true,runtime:{runtime}" )) From 5b7ba90687822240b546d8ad9e2159b4a2408e77 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:42:30 -0400 Subject: [PATCH 13/17] fix: emit cold-start invocation metric immediately if initStart already processed Handles both orderings of the Invoke event and platform.initStart. If initStart has already resolved the durable tag by the time invocation #1 arrives, there's nothing to wait for, so emit the metric right away instead of holding it back (which previously would only be flushed at shutdown, since platform.initStart doesn't fire again). Co-Authored-By: Claude Sonnet 5 --- .../src/lifecycle/invocation/processor.rs | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index dc31af8ec..8eb7972e2 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -115,8 +115,8 @@ pub struct Processor { /// Timestamp of the cold-start invocation metric in On-Demand mode, held back until /// `platform.initStart` sets the durable tag so it's not missed on invocation #1. /// - /// This assumes `platform.initStart` always arrives after the Invoke event, which is what - /// we've observed in production. + /// Only used when `platform.initStart` hasn't already been processed by the time invocation + /// #1 arrives; see `platform_init_start_processed`. pending_first_invocation_metric_timestamp: Option, /// Whether the first On-Demand invocation has already been seen. /// @@ -125,6 +125,14 @@ pub struct Processor { /// (an unordered, separate path from `on_invoke_event`), which would otherwise make later /// invocations look like cold starts too and hold back their metric indefinitely. first_on_demand_invocation_seen: bool, + /// Whether `platform.initStart` has already been processed. + /// + /// We've observed `platform.initStart` arrive after the Invoke event in production (the + /// Extensions API's `next()` call is unbuffered, while Telemetry API delivery lags), which + /// is why invocation #1's metric is normally held back. But that's not a hard ordering + /// guarantee, so if `platform.initStart` has already set the durable tag by the time + /// invocation #1 arrives, this flag lets us skip the hold-back and emit immediately. + platform_init_start_processed: bool, } impl Processor { @@ -170,6 +178,7 @@ impl Processor { init_duration_metric_emitted: false, pending_first_invocation_metric_timestamp: None, first_on_demand_invocation_seen: false, + platform_init_start_processed: false, } } @@ -182,6 +191,9 @@ impl Processor { if is_on_demand_cold_start { self.first_on_demand_invocation_seen = true; } + // Only hold back if `platform.initStart` hasn't already resolved the durable tag. + let should_hold_back_invocation_metric = + is_on_demand_cold_start && !self.platform_init_start_processed; // In Managed Instance mode, if awaiting the first invocation after init, find and update the empty context created on init start if self.aws_config.is_managed_instance_mode() && self.awaiting_first_invocation { @@ -248,7 +260,7 @@ impl Processor { } // Hold back the metric for a cold start in On-Demand mode; see `pending_first_invocation_metric_timestamp`. - if is_on_demand_cold_start { + if should_hold_back_invocation_metric { self.pending_first_invocation_metric_timestamp = Some(timestamp); } else { self.enhanced_metrics.increment_invocation_metric(timestamp); @@ -373,6 +385,7 @@ impl Processor { { self.enhanced_metrics.set_durable_function_tag(); } + self.platform_init_start_processed = true; // Flush the held-back metric now that the durable tag (if any) has been set. self.flush_pending_first_invocation_metric(); @@ -2184,6 +2197,58 @@ mod tests { ); } + #[tokio::test] + async fn test_on_invoke_event_does_not_hold_metric_if_init_start_already_processed() { + let mut processor = setup(); + + // Simulate the reverse order: `platform.initStart` arrives before the Invoke event + // and reports a durable runtime. + processor.on_platform_init_start( + Utc::now(), + Some("python:3.14.DurableFunction.v6".to_string()), + ); + + // The durable tag is already known, so the metric must be emitted immediately + // instead of being held back. + let now: i64 = std::time::UNIX_EPOCH + .elapsed() + .expect("unable to poll clock, unrecoverable") + .as_secs() + .try_into() + .unwrap_or_default(); + processor.on_invoke_event("req-1".to_string()); + assert!( + processor + .pending_first_invocation_metric_timestamp + .is_none(), + "Expected the invocation metric to be emitted immediately, not held back" + ); + + let runtime = processor + .runtime + .clone() + .expect("runtime should have been resolved by on_invoke_event"); + let ts = (now / 10) * 10; + let expected_tags = dogstatsd::metric::SortedTags::parse(&format!( + "cold_start:true,durable_function:true,runtime:{runtime}" + )) + .ok(); + let entry = processor + .enhanced_metrics + .aggr_handle + .get_entry_by_id( + crate::metrics::enhanced::constants::INVOCATIONS_METRIC.into(), + expected_tags, + ts, + ) + .await + .unwrap(); + assert!( + entry.is_some(), + "Expected the invocation metric to carry the durable_function:true tag" + ); + } + #[tokio::test] async fn test_init_duration_emitted_from_init_report() { let mut processor = setup(); From cd882a38d2b3c1788bbc2ac13bca396e99b00237 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:51:00 -0400 Subject: [PATCH 14/17] refactor: collapse first-invocation metric tracking into a single enum Replaces pending_first_invocation_metric_timestamp, first_on_demand_invocation_seen, and platform_init_start_processed with one FirstInvocationMetric enum (Pending/InitStartSeen/HeldBack/Resolved), making the state machine explicit instead of spread across three separate fields. Also drops the struct_excessive_bools allow, since the struct is back to 3 bool fields. Co-Authored-By: Claude Sonnet 5 --- .../src/lifecycle/invocation/processor.rs | 101 ++++++++++-------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 8eb7972e2..69040d6a6 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -63,7 +63,6 @@ pub const DATADOG_INVOCATION_ERROR_KEY: &str = "x-datadog-invocation-error"; const DATADOG_SPAN_ID_KEY: &str = "x-datadog-span-id"; const TAG_SAMPLING_PRIORITY: &str = "_sampling_priority_v1"; -#[allow(clippy::struct_excessive_bools)] pub struct Processor { /// Buffer containing context of the previous 5 invocations context_buffer: ContextBuffer, @@ -112,27 +111,32 @@ pub struct Processor { /// on `platform.report`. This flag ensures whichever event arrives first wins and the other is skipped, /// preventing double counting. init_duration_metric_emitted: bool, - /// Timestamp of the cold-start invocation metric in On-Demand mode, held back until - /// `platform.initStart` sets the durable tag so it's not missed on invocation #1. + /// How the cold-start invocation metric in On-Demand mode has been resolved relative to + /// `platform.initStart`, since either the first invocation or `platform.initStart` can + /// arrive first (the Extensions API's `next()` call is unbuffered, while Telemetry API + /// delivery lags, so we've observed the invocation arriving first in production, but that's + /// not a hard ordering guarantee). /// - /// Only used when `platform.initStart` hasn't already been processed by the time invocation - /// #1 arrives; see `platform_init_start_processed`. - pending_first_invocation_metric_timestamp: Option, - /// Whether the first On-Demand invocation has already been seen. - /// - /// Used instead of `context_buffer.is_empty()` to detect the first invocation, since the - /// buffer is also emptied between warm invocations once `platform.report` is processed - /// (an unordered, separate path from `on_invoke_event`), which would otherwise make later - /// invocations look like cold starts too and hold back their metric indefinitely. - first_on_demand_invocation_seen: bool, - /// Whether `platform.initStart` has already been processed. - /// - /// We've observed `platform.initStart` arrive after the Invoke event in production (the - /// Extensions API's `next()` call is unbuffered, while Telemetry API delivery lags), which - /// is why invocation #1's metric is normally held back. But that's not a hard ordering - /// guarantee, so if `platform.initStart` has already set the durable tag by the time - /// invocation #1 arrives, this flag lets us skip the hold-back and emit immediately. - platform_init_start_processed: bool, + /// Also replaces `context_buffer.is_empty()` for detecting the first invocation: the buffer + /// is emptied between warm invocations too, once `platform.report` is processed (an + /// unordered, separate path from `on_invoke_event`), so it can't be used to tell "first + /// invocation" from "buffer momentarily empty between warm invocations." + first_invocation_metric: FirstInvocationMetric, +} + +/// See `Processor::first_invocation_metric`. +enum FirstInvocationMetric { + /// Neither the first invocation nor `platform.initStart` has been seen yet. + Pending, + /// `platform.initStart` arrived first, so the durable tag is already known; the first + /// invocation's metric can be emitted immediately once it arrives. + InitStartSeen, + /// The first invocation arrived before `platform.initStart`; its metric is held back until + /// `platform.initStart` flushes it. + HeldBack(i64), + /// The first invocation's metric has been emitted, either immediately or after being held + /// back. Nothing left to track. + Resolved, } impl Processor { @@ -176,24 +180,23 @@ impl Processor { durable_context_tx, restore_time: None, init_duration_metric_emitted: false, - pending_first_invocation_metric_timestamp: None, - first_on_demand_invocation_seen: false, - platform_init_start_processed: false, + first_invocation_metric: FirstInvocationMetric::Pending, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { - // See `pending_first_invocation_metric_timestamp` and `first_on_demand_invocation_seen`. - let is_on_demand_cold_start = - !self.aws_config.is_managed_instance_mode() && !self.first_on_demand_invocation_seen; - if is_on_demand_cold_start { - self.first_on_demand_invocation_seen = true; - } + // See `first_invocation_metric`. + let is_first_on_demand_invocation = !self.aws_config.is_managed_instance_mode() + && matches!( + self.first_invocation_metric, + FirstInvocationMetric::Pending | FirstInvocationMetric::InitStartSeen + ); // Only hold back if `platform.initStart` hasn't already resolved the durable tag. let should_hold_back_invocation_metric = - is_on_demand_cold_start && !self.platform_init_start_processed; + matches!(self.first_invocation_metric, FirstInvocationMetric::Pending) + && is_first_on_demand_invocation; // In Managed Instance mode, if awaiting the first invocation after init, find and update the empty context created on init start if self.aws_config.is_managed_instance_mode() && self.awaiting_first_invocation { @@ -259,11 +262,14 @@ impl Processor { .add_enhanced_metric_data(&request_id, enhanced_metric_offsets); } - // Hold back the metric for a cold start in On-Demand mode; see `pending_first_invocation_metric_timestamp`. + // Hold back the metric for a cold start in On-Demand mode; see `first_invocation_metric`. if should_hold_back_invocation_metric { - self.pending_first_invocation_metric_timestamp = Some(timestamp); + self.first_invocation_metric = FirstInvocationMetric::HeldBack(timestamp); } else { self.enhanced_metrics.increment_invocation_metric(timestamp); + if is_first_on_demand_invocation { + self.first_invocation_metric = FirstInvocationMetric::Resolved; + } } self.enhanced_metrics.set_invoked_received(); @@ -298,8 +304,9 @@ impl Processor { /// Flushes the metric held back by `on_invoke_event`, if any. fn flush_pending_first_invocation_metric(&mut self) { - if let Some(timestamp) = self.pending_first_invocation_metric_timestamp.take() { + if let FirstInvocationMetric::HeldBack(timestamp) = self.first_invocation_metric { self.enhanced_metrics.increment_invocation_metric(timestamp); + self.first_invocation_metric = FirstInvocationMetric::Resolved; } } @@ -385,7 +392,9 @@ impl Processor { { self.enhanced_metrics.set_durable_function_tag(); } - self.platform_init_start_processed = true; + if matches!(self.first_invocation_metric, FirstInvocationMetric::Pending) { + self.first_invocation_metric = FirstInvocationMetric::InitStartSeen; + } // Flush the held-back metric now that the durable tag (if any) has been set. self.flush_pending_first_invocation_metric(); @@ -2155,9 +2164,11 @@ mod tests { // Simulate the On-Demand race: the Invoke event reaches the processor before // `platform.initStart`. The invocation metric must not be emitted yet. processor.on_invoke_event("req-1".to_string()); - let held_back_timestamp = processor - .pending_first_invocation_metric_timestamp - .expect("Expected the cold start invocation metric to be held back"); + let FirstInvocationMetric::HeldBack(held_back_timestamp) = + processor.first_invocation_metric + else { + panic!("Expected the cold start invocation metric to be held back") + }; // `platform.initStart` arrives afterwards and reports a durable runtime. processor.on_platform_init_start( @@ -2166,9 +2177,10 @@ mod tests { ); assert!( - processor - .pending_first_invocation_metric_timestamp - .is_none(), + matches!( + processor.first_invocation_metric, + FirstInvocationMetric::Resolved + ), "Expected the held-back invocation metric to be flushed by on_platform_init_start" ); @@ -2218,9 +2230,10 @@ mod tests { .unwrap_or_default(); processor.on_invoke_event("req-1".to_string()); assert!( - processor - .pending_first_invocation_metric_timestamp - .is_none(), + matches!( + processor.first_invocation_metric, + FirstInvocationMetric::Resolved + ), "Expected the invocation metric to be emitted immediately, not held back" ); From decffd1e8af266d50d56d5632185f46d39178306 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:56:10 -0400 Subject: [PATCH 15/17] refactor: rename first_invocation_metric to first_invocation_metric_status Field/enum name now reflects that it tracks a status, not the metric value itself. Co-Authored-By: Claude Sonnet 5 --- .../src/lifecycle/invocation/processor.rs | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 69040d6a6..68e5d586b 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -121,11 +121,11 @@ pub struct Processor { /// is emptied between warm invocations too, once `platform.report` is processed (an /// unordered, separate path from `on_invoke_event`), so it can't be used to tell "first /// invocation" from "buffer momentarily empty between warm invocations." - first_invocation_metric: FirstInvocationMetric, + first_invocation_metric_status: FirstInvocationMetricStatus, } -/// See `Processor::first_invocation_metric`. -enum FirstInvocationMetric { +/// See `Processor::first_invocation_metric_status`. +enum FirstInvocationMetricStatus { /// Neither the first invocation nor `platform.initStart` has been seen yet. Pending, /// `platform.initStart` arrived first, so the durable tag is already known; the first @@ -180,23 +180,24 @@ impl Processor { durable_context_tx, restore_time: None, init_duration_metric_emitted: false, - first_invocation_metric: FirstInvocationMetric::Pending, + first_invocation_metric_status: FirstInvocationMetricStatus::Pending, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// pub fn on_invoke_event(&mut self, request_id: String) { - // See `first_invocation_metric`. + // See `first_invocation_metric_status`. let is_first_on_demand_invocation = !self.aws_config.is_managed_instance_mode() && matches!( - self.first_invocation_metric, - FirstInvocationMetric::Pending | FirstInvocationMetric::InitStartSeen + self.first_invocation_metric_status, + FirstInvocationMetricStatus::Pending | FirstInvocationMetricStatus::InitStartSeen ); // Only hold back if `platform.initStart` hasn't already resolved the durable tag. - let should_hold_back_invocation_metric = - matches!(self.first_invocation_metric, FirstInvocationMetric::Pending) - && is_first_on_demand_invocation; + let should_hold_back_invocation_metric = matches!( + self.first_invocation_metric_status, + FirstInvocationMetricStatus::Pending + ) && is_first_on_demand_invocation; // In Managed Instance mode, if awaiting the first invocation after init, find and update the empty context created on init start if self.aws_config.is_managed_instance_mode() && self.awaiting_first_invocation { @@ -262,13 +263,13 @@ impl Processor { .add_enhanced_metric_data(&request_id, enhanced_metric_offsets); } - // Hold back the metric for a cold start in On-Demand mode; see `first_invocation_metric`. + // Hold back the metric for a cold start in On-Demand mode; see `first_invocation_metric_status`. if should_hold_back_invocation_metric { - self.first_invocation_metric = FirstInvocationMetric::HeldBack(timestamp); + self.first_invocation_metric_status = FirstInvocationMetricStatus::HeldBack(timestamp); } else { self.enhanced_metrics.increment_invocation_metric(timestamp); if is_first_on_demand_invocation { - self.first_invocation_metric = FirstInvocationMetric::Resolved; + self.first_invocation_metric_status = FirstInvocationMetricStatus::Resolved; } } self.enhanced_metrics.set_invoked_received(); @@ -304,9 +305,11 @@ impl Processor { /// Flushes the metric held back by `on_invoke_event`, if any. fn flush_pending_first_invocation_metric(&mut self) { - if let FirstInvocationMetric::HeldBack(timestamp) = self.first_invocation_metric { + if let FirstInvocationMetricStatus::HeldBack(timestamp) = + self.first_invocation_metric_status + { self.enhanced_metrics.increment_invocation_metric(timestamp); - self.first_invocation_metric = FirstInvocationMetric::Resolved; + self.first_invocation_metric_status = FirstInvocationMetricStatus::Resolved; } } @@ -392,8 +395,11 @@ impl Processor { { self.enhanced_metrics.set_durable_function_tag(); } - if matches!(self.first_invocation_metric, FirstInvocationMetric::Pending) { - self.first_invocation_metric = FirstInvocationMetric::InitStartSeen; + if matches!( + self.first_invocation_metric_status, + FirstInvocationMetricStatus::Pending + ) { + self.first_invocation_metric_status = FirstInvocationMetricStatus::InitStartSeen; } // Flush the held-back metric now that the durable tag (if any) has been set. self.flush_pending_first_invocation_metric(); @@ -2164,8 +2170,8 @@ mod tests { // Simulate the On-Demand race: the Invoke event reaches the processor before // `platform.initStart`. The invocation metric must not be emitted yet. processor.on_invoke_event("req-1".to_string()); - let FirstInvocationMetric::HeldBack(held_back_timestamp) = - processor.first_invocation_metric + let FirstInvocationMetricStatus::HeldBack(held_back_timestamp) = + processor.first_invocation_metric_status else { panic!("Expected the cold start invocation metric to be held back") }; @@ -2178,8 +2184,8 @@ mod tests { assert!( matches!( - processor.first_invocation_metric, - FirstInvocationMetric::Resolved + processor.first_invocation_metric_status, + FirstInvocationMetricStatus::Resolved ), "Expected the held-back invocation metric to be flushed by on_platform_init_start" ); @@ -2231,8 +2237,8 @@ mod tests { processor.on_invoke_event("req-1".to_string()); assert!( matches!( - processor.first_invocation_metric, - FirstInvocationMetric::Resolved + processor.first_invocation_metric_status, + FirstInvocationMetricStatus::Resolved ), "Expected the invocation metric to be emitted immediately, not held back" ); From 1636553b6b07a6fb8d7cf8b156a4f2c14306fc67 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:01:41 -0400 Subject: [PATCH 16/17] docs: trim historical rationale from first_invocation_metric_status comment Co-Authored-By: Claude Sonnet 5 --- bottlecap/src/lifecycle/invocation/processor.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 68e5d586b..307484b56 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -116,11 +116,6 @@ pub struct Processor { /// arrive first (the Extensions API's `next()` call is unbuffered, while Telemetry API /// delivery lags, so we've observed the invocation arriving first in production, but that's /// not a hard ordering guarantee). - /// - /// Also replaces `context_buffer.is_empty()` for detecting the first invocation: the buffer - /// is emptied between warm invocations too, once `platform.report` is processed (an - /// unordered, separate path from `on_invoke_event`), so it can't be used to tell "first - /// invocation" from "buffer momentarily empty between warm invocations." first_invocation_metric_status: FirstInvocationMetricStatus, } From cc19dee1397065e9e030d2187d08711c1ae59ef3 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:08:21 -0400 Subject: [PATCH 17/17] docs: add comment conciseness guideline to AGENTS.md Co-Authored-By: Claude Sonnet 5 --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index ca7823624..2e27f380a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ All Rust code lives under `bottlecap/`. Run these from that directory unless not ## Code Style - Do not be overly defensive and add unnecessary checks or tests unless you have a good reason. +- Keep comments concise. Do not explain the history of a change or reference past code unless necessary. ## Pull Requests