From a5967efdc4fed18e9ddab7a106ae67ef7f2f646c Mon Sep 17 00:00:00 2001 From: Ian Hou <45278651+iankhou@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:18:00 -0400 Subject: [PATCH 1/3] feat(toolkit-lib): cdk diagnose surfaces the failure details of CloudFormation Hooks When a CloudFormation Hook (e.g. a Guard Hook or Lambda Hook) fails a resource during a deployment, the stack events only carry a terse 'The following hook(s) failed: [X]'. The actual failure details (Guard rule annotations, remediation messages, Lambda hook reasons) live in the GetHookResult API. cdk deploy already fetches them for its live activity stream, but cdk diagnose showed nothing beyond the terse message. Move hook-error diagnosis into the central CloudFormationStackDiagnoser: - ResourceErrors now correlates failed hook events to the resource errors they cause (resolving the 'FIXME: Check hooks') and records them on a new hookFailures field. - The GetHookResult fetch (including the bootstrap-version permission hint) is extracted into a shared fetchHookResultDetails helper, used by the deploy monitor's live enrichment, the diagnoser's change-set hook path, and the new stack-event hook path. - The diagnoser fetches hook failure details for stack-event errors and attaches them as additional diagnostic context. This is gated behind a new fetchHookFailureDetails option, enabled only by the diagnose action: during a deployment the activity stream already shows the details live, so deploy output is unchanged. Also fixes a malformed permissions warning (stray newline and quote) that was carried over from the monitor code. --- .../lib/api/diagnosing/stack-diagnoser.ts | 68 +++++-- .../api/stack-events/hook-result-details.ts | 69 +++++++ .../lib/api/stack-events/resource-errors.ts | 83 +++++++- .../stack-events/stack-activity-monitor.ts | 51 +---- .../toolkit-lib/lib/toolkit/toolkit.ts | 1 + .../api/diagnosing/stack-diagnoser.test.ts | 179 ++++++++++++++++++ .../api/stack-events/resource-errors.test.ts | 133 +++++++++++++ .../stack-activity-monitor.test.ts | 6 +- 8 files changed, 527 insertions(+), 63 deletions(-) create mode 100644 packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts index ae7e4d96f..ca6c8c7e0 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts @@ -10,7 +10,7 @@ import type { EnvironmentResources } from '../environment'; import type { IoHelper } from '../io/private/io-helper'; import type { ISourceTracer } from '../source-tracing/private/source-tracing'; import { PollRange, StackEventPoller } from '../stack-events'; -import { formatHookResultDetails, isFailedHookStatus } from '../stack-events/hook-result-details'; +import { fetchHookResultDetails, formatHookResultDetails, isFailedHookStatus, normalizeHookMessage } from '../stack-events/hook-result-details'; import type { ResourceError, ResourceErrors } from '../stack-events/resource-errors'; import { StackStatus } from '../stack-events/stack-status'; @@ -34,6 +34,18 @@ export interface CloudFormationStackDiagnoserProps { * level and their information is simply not added to the output. */ readonly additionalExplorationSdkProvider?: SdkProvider; + + /** + * Whether to fetch the failure details of hooks that failed resources, via the + * `GetHookResult` API, and attach them to the diagnosis. + * + * During a deployment the activity stream already shows these details live, so + * `cdk deploy` leaves this off to avoid repeating them. `cdk diagnose` has no + * live stream and enables it. + * + * @default false + */ + readonly fetchHookFailureDetails?: boolean; } export type SdkProvider = () => Promise; @@ -399,14 +411,12 @@ export class CloudFormationStackDiagnoser { */ private async _hookResultDetails(hook: HookResultSummary): Promise { if (hook.HookResultId) { - try { - const result = await this.cfn.getHookResult({ HookResultId: hook.HookResultId }); - const details = formatHookResultDetails(result); - if (details) { - return details; - } - } catch (e: any) { - await this.props.ioHelper.defaults.debug(`Could not fetch hook result ${hook.HookResultId}: ${e.message}`); + const details = await fetchHookResultDetails(this.cfn, hook.HookResultId, { + ioHelper: this.props.ioHelper, + envResources: this.props.envResources, + }); + if (details) { + return details; } } return formatHookResultDetails(hook); @@ -425,19 +435,55 @@ export class CloudFormationStackDiagnoser { : this.props.sourceTracer.traceStack(err.stackArn, err.parentStackLogicalIds); // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism - const [sourceTrace, additionalContext] = await Promise.all([ + const [sourceTrace, additionalContext, hookContext] = await Promise.all([ sourceTracePromise, this.investigateResourceBestEffort(err), + this.hookFailureContext(err), ]); + const allContext = [...hookContext, ...additionalContext]; return { ...err, sourceTrace, topLevelStackHierarchicalId: this.props.topLevelStackHierarchicalId, - ...(additionalContext.length > 0 ? { additionalContext } : {}), + ...(allContext.length > 0 ? { additionalContext: allContext } : {}), }; } + /** + * Fetch the failure details of the hooks that caused the given resource error. + * + * The resource's own status reason only names the hooks that failed it (e.g. + * "The following hook(s) failed: [X]"); the actual failure details live in the + * hook results API. Falls back to the hook event's status reason if the fetch + * fails or returns no details. + * + * Only active when `fetchHookFailureDetails` is set (i.e. for `cdk diagnose`); + * during a deployment the activity stream already shows these details. + */ + private async hookFailureContext(err: ResourceError): Promise { + if (!this.props.fetchHookFailureDetails) { + return []; + } + const ret: AdditionalDiagnosticContext[] = []; + for (const hook of err.hookFailures ?? []) { + let details = hook.hookInvocationId + ? await fetchHookResultDetails(this.cfn, hook.hookInvocationId, { + ioHelper: this.props.ioHelper, + envResources: this.props.envResources, + }) + : undefined; + details = details ?? (hook.hookStatusReason ? normalizeHookMessage(hook.hookStatusReason) : undefined); + if (details) { + ret.push({ + source: `CloudFormation Hook (${hook.hookType})`, + messages: details.split('\n').map((line, i) => i === 0 ? `Hook '${hook.hookType}' failed: ${line}` : line), + }); + } + } + return ret; + } + private async investigateResourceBestEffort(err: ResourceError) { const sdk = await this.additionalExplorationSdk(); if (!sdk) { diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts index a29de63fa..7f6875406 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts @@ -1,4 +1,9 @@ +import * as util from 'node:util'; import type { GetHookResultCommandOutput, HookResultSummary } from '@aws-sdk/client-cloudformation'; +import type { ICloudFormationClient } from '../aws-auth/private'; +import { isAccessDeniedError } from '../aws-auth/util'; +import type { EnvironmentResources } from '../environment'; +import type { IoHelper } from '../io/private'; /** * The relevant parts of a hook result, common to `GetHookResult` outputs and @@ -53,6 +58,70 @@ export function isFailedHookStatus(status: string | undefined): boolean { return status === 'HOOK_COMPLETE_FAILED' || status === 'HOOK_FAILED'; } +export interface FetchHookResultDetailsOptions { + /** + * The IoHelper used to warn when the fetch fails. + */ + readonly ioHelper: IoHelper; + + /** + * Environment resources, used to look up the bootstrap toolkit version when + * diagnosing hook result fetch failures caused by missing permissions. + * + * @default - Bootstrap version is not reported in error messages + */ + readonly envResources?: EnvironmentResources; +} + +/** + * Fetch a hook invocation's failure details via the `GetHookResult` API and format + * them into a human-readable string. + * + * For Guard Hooks the details come from the failed annotations; for other hooks + * (e.g. Lambda Hooks) they come from the hook result's own status reason. + * Returns undefined if the fetch fails (emitting a warning, with a bootstrap + * upgrade hint if the failure looks permissions-related) or the result carries + * no failure details. + */ +export async function fetchHookResultDetails( + cfn: ICloudFormationClient, + hookInvocationId: string, + options: FetchHookResultDetailsOptions, +): Promise { + try { + const result = await cfn.getHookResult({ HookResultId: hookInvocationId }); + return formatHookResultDetails(result); + } catch (e: any) { + const errorMessage = e instanceof Error ? e.message : String(e); + + const isPermissionsError = + isAccessDeniedError(e) || + (typeof errorMessage === 'string' && errorMessage.toLowerCase().includes('not authorized to perform: cloudformation:gethookresult')); + + if (isPermissionsError && options.envResources) { + let currentVersion: number | undefined = undefined; + try { + currentVersion = (await options.envResources.lookupToolkit()).version; + } catch { + // ignore errors looking up the bootstrap version + } + + await options.ioHelper.defaults.warn( + `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. ` + + 'Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment ' + + "by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. " + + `Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion ?? 'unknown'}.`, + ); + } else { + await options.ioHelper.defaults.warn( + util.format('Failed to fetch Hook details for invocation %s: %s', hookInvocationId, errorMessage), + ); + } + + return undefined; + } +} + /** * Trims leading/trailing whitespace, collapses all internal whitespace * (including newlines) to a single space, and truncates to `maxChars` diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts index 943216101..8f09e54b7 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts @@ -1,8 +1,33 @@ import type { StackEvent } from '@aws-sdk/client-cloudformation'; +import { isFailedHookStatus } from './hook-result-details'; import type { ResourceEvent } from './stack-event-poller'; import { DeploymentErrorCodes } from '../../toolkit/toolkit-error'; import { isCancellationEvent, isErrorEvent, isRegularResourceEvent } from '../../util/cloudformation'; +/** + * A failed CloudFormation Hook invocation that caused a resource error. + */ +export interface ResourceHookFailure { + /** + * The type name of the hook that failed (e.g. `Private::Guard::TestHook`) + */ + readonly hookType: string; + + /** + * The unique identifier of the hook invocation. + * + * Can be used to fetch the full failure details via the `GetHookResult` API. + */ + readonly hookInvocationId?: string; + + /** + * The status reason reported on the hook's stack event. + * + * This is usually a terse summary; the `GetHookResult` API has richer details. + */ + readonly hookStatusReason?: string; +} + export interface ResourceError { /** * The stack this resource error occurred in @@ -56,6 +81,15 @@ export interface ResourceError { * and early-validation errors. */ readonly timestamp?: Date; + + /** + * CloudFormation Hook failures that caused this resource error, if any. + * + * Hook failures are reported on separate stack events from the resource failure + * itself; they are correlated to the resource error by logical ID and by the + * hook type appearing in the resource's status reason. + */ + readonly hookFailures?: ResourceHookFailure[]; } /** @@ -69,6 +103,16 @@ export class ResourceErrors { */ private readonly _errors: ResourceError[] = []; + /** + * Failed hook invocations seen so far, keyed by stack ARN + logical ID of the + * resource they were invoked on. + * + * Hook failures are reported on their own events (usually with an `IN_PROGRESS` + * resource status), before the resource failure event they cause. We remember + * them here so they can be attached to that resource error when it arrives. + */ + private readonly seenHookFailures = new Map(); + public isEmpty() { return this._errors.length === 0; } @@ -86,16 +130,44 @@ export class ResourceErrors { */ public update(...events: ResourceEvent[]) { for (const event of events) { + this.trackHookFailure(event.event); if (isErrorEvent(event.event)) { // Cancelled is not an interesting failure reason, nor is the stack message (stack // message will just say something like "stack failed to update") if (!isCancellationEvent(event.event) && isRegularResourceEvent(event.event)) { - this._errors.push(errorFromEvent(event)); + this._errors.push(errorFromEvent(event, this.hookFailuresFor(event.event))); } } } } + private trackHookFailure(event: StackEvent) { + if (!isFailedHookStatus(event.HookStatus) || !event.HookType || !event.LogicalResourceId || !event.StackId) { + return; + } + const key = hookFailureKey(event); + const failures = this.seenHookFailures.get(key) ?? []; + failures.push({ + hookType: event.HookType, + hookInvocationId: event.HookInvocationId, + hookStatusReason: event.HookStatusReason, + }); + this.seenHookFailures.set(key, failures); + } + + /** + * Return the hook failures that caused the given resource failure, if any. + * + * A hook failure belongs to a resource failure when it was recorded for the same + * resource, and its hook type is mentioned in the resource's status reason (e.g. + * "The following hook(s) failed: [Private::Guard::TestHook]"). + */ + private hookFailuresFor(event: StackEvent): ResourceHookFailure[] { + const recorded = this.seenHookFailures.get(hookFailureKey(event)) ?? []; + const reason = event.ResourceStatusReason ?? ''; + return recorded.filter((h) => reason.includes(h.hookType)); + } + /** * Take our best guess at the error code of the root cause * @@ -123,9 +195,7 @@ export class ResourceErrors { } } -function errorFromEvent(ev: ResourceEvent): ResourceError { - // FIXME: Check hooks - +function errorFromEvent(ev: ResourceEvent, hookFailures: ResourceHookFailure[]): ResourceError { return { logicalId: ev.event.LogicalResourceId ?? '', message: ev.event.ResourceStatusReason ?? '', @@ -135,9 +205,14 @@ function errorFromEvent(ev: ResourceEvent): ResourceError { errorCode: extractErrorCode(ev.event), physicalId: ev.event.PhysicalResourceId, timestamp: ev.event.Timestamp, + ...(hookFailures.length > 0 ? { hookFailures } : {}), }; } +function hookFailureKey(event: StackEvent): string { + return `${event.StackId ?? ''}|${event.LogicalResourceId ?? ''}`; +} + /** * Extract an error code from the given stack event. * diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts index 28326f8b0..a7b8a2aae 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts @@ -1,13 +1,12 @@ import { randomUUID } from 'node:crypto'; import * as util from 'node:util'; import type { CloudFormationStackArtifact } from '@aws-cdk/cloud-assembly-api'; -import { formatHookResultDetails } from './hook-result-details'; +import { fetchHookResultDetails } from './hook-result-details'; import { StackEventPoller, PollRange } from './stack-event-poller'; import { StackProgressMonitor } from './stack-progress-monitor'; import type { StackActivity } from '../../payloads/stack-activity'; import { stackNameFromArn } from '../../util/cloudformation'; import type { ICloudFormationClient } from '../aws-auth/private'; -import { isAccessDeniedError } from '../aws-auth/util'; import type { EnvironmentResources } from '../environment'; import { IO, type IoHelper } from '../io/private'; import { resourceMetadata } from '../resource-metadata/resource-metadata'; @@ -244,49 +243,6 @@ export class StackActivityMonitor { return resourceMetadata(this.stack, logicalId); } - /** - * Fetches hook failure details via the GetHookResult API and formats them into - * a human-readable string. - * - * For Guard Hooks the details come from the failed annotations; for other hooks - * (e.g. Lambda Hooks) they come from the hook result's own status reason. - * Returns undefined if the fetch fails or the result carries no failure details. - */ - private async fetchHookResultDetails(hookInvocationId: string): Promise { - try { - const result = await this.cfn.getHookResult({ HookResultId: hookInvocationId }); - return formatHookResultDetails(result); - } catch (e: any) { - const errorMessage = e instanceof Error ? e.message : String(e); - - const isPermissionsError = - isAccessDeniedError(e) || - (typeof errorMessage === 'string' && errorMessage.toLowerCase().includes('not authorized to perform: cloudformation:gethookresult')); - - if (isPermissionsError && this.envResources) { - let currentVersion: number | undefined = undefined; - try { - currentVersion = (await this.envResources.lookupToolkit()).version; - } catch { - // ignore errors looking up the bootstrap version - } - - await this.ioHelper.defaults.warn( - util.format( - `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. - 'Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion ?? 'unknown'}.`, - ), - ); - } else { - await this.ioHelper.defaults.warn( - util.format('Failed to fetch Hook details for invocation %s: %s', hookInvocationId, errorMessage), - ); - } - - return undefined; - } - } - /** * Reads all new events from the stack history * @@ -300,7 +256,10 @@ export class StackActivityMonitor { // If this is a failed hook event with an invocation ID, fetch the failure details if (resourceEvent.event.HookInvocationId) { - const details = await this.fetchHookResultDetails(resourceEvent.event.HookInvocationId); + const details = await fetchHookResultDetails(this.cfn, resourceEvent.event.HookInvocationId, { + ioHelper: this.ioHelper, + envResources: this.envResources, + }); if (details) { resourceEvent.event.HookStatusReason = details; } diff --git a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts index 7f9f02132..2d28080ed 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts @@ -648,6 +648,7 @@ export class Toolkit extends CloudAssemblySourceBuilder { ioHelper, topLevelStackHierarchicalId: stack.hierarchicalId, additionalExplorationSdkProvider: () => Promise.resolve(stackEnv.sdk), + fetchHookFailureDetails: true, }); const diagnosis = await diagnoser.diagnoseFromFresh(stack.stackName); diff --git a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts index 6622160a1..41b82ce8a 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts @@ -38,6 +38,16 @@ function makeDiagnoser(topLevelStackHierarchicalId = 'TestStack') { }); } +function makeHookFetchingDiagnoser() { + return new CloudFormationStackDiagnoser({ + sdk, + sourceTracer: fakeTracer, + ioHelper: ioHost.asHelper('diagnose'), + topLevelStackHierarchicalId: 'TestStack', + fetchHookFailureDetails: true, + }); +} + /** * A fake source tracer that records all calls and returns a fixed trace */ @@ -703,6 +713,175 @@ describe('CloudFormationStackDiagnoser', () => { })], }); }); + + test('enriches a resource error with fetched hook failure details', async () => { + const errors = new ResourceErrors(); + errors.update( + { + event: { + EventId: 'evt-hook', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'terse reason', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + { + event: { + EventId: 'evt-fail', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + ); + + mockCloudFormationClient.on(GetHookResultCommand).resolves({ + HookResultId: 'hook-invocation-1', + Status: 'HOOK_COMPLETE_FAILED', + Annotations: [{ + AnnotationName: 'AWS_S3_Bucket_AccessControl', + Status: 'FAILED', + StatusMessage: 'Check was not compliant.', + RemediationMessage: 'AccessControl is deprecated', + }], + } as any); + + const result = await makeHookFetchingDiagnoser().diagnoseFromErrorCollection(errors, { + StackName: 'MyStack', + StackStatus: 'UPDATE_FAILED', + CreationTime: new Date(), + }); + + expect(mockCloudFormationClient).toHaveReceivedCommandWith(GetHookResultCommand, { + HookResultId: 'hook-invocation-1', + }); + assertProblem(result); + const context = result.problems[0].additionalContext ?? []; + expect(context).toEqual([ + expect.objectContaining({ + source: 'CloudFormation Hook (Private::Guard::TestHook)', + messages: expect.arrayContaining([ + expect.stringContaining("Hook 'Private::Guard::TestHook' failed"), + '[AWS_S3_Bucket_AccessControl]', + ]), + }), + ]); + }); + + test('does not fetch hook details when fetchHookFailureDetails is off (deploy)', async () => { + const errors = new ResourceErrors(); + errors.update( + { + event: { + EventId: 'evt-hook', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'terse reason', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + { + event: { + EventId: 'evt-fail', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + ); + + const result = await makeDiagnoser().diagnoseFromErrorCollection(errors, { + StackName: 'MyStack', + StackStatus: 'UPDATE_FAILED', + CreationTime: new Date(), + }); + + expect(mockCloudFormationClient).not.toHaveReceivedCommand(GetHookResultCommand); + assertProblem(result); + expect(result.problems[0].additionalContext).toBeUndefined(); + }); + + test('falls back to the hook status reason when GetHookResult returns no details', async () => { + const errors = new ResourceErrors(); + errors.update( + { + event: { + EventId: 'evt-hook', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'the terse fallback reason', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + { + event: { + EventId: 'evt-fail', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + ); + + mockCloudFormationClient.on(GetHookResultCommand).rejects(new Error('not authorized')); + + const result = await makeHookFetchingDiagnoser().diagnoseFromErrorCollection(errors, { + StackName: 'MyStack', + StackStatus: 'UPDATE_FAILED', + CreationTime: new Date(), + }); + + assertProblem(result); + const context = result.problems[0].additionalContext ?? []; + expect(context).toEqual([ + expect.objectContaining({ + messages: ["Hook 'Private::Guard::TestHook' failed: the terse fallback reason"], + }), + ]); + }); }); describe('CloudTrail investigation wiring', () => { diff --git a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts new file mode 100644 index 000000000..df33295b0 --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts @@ -0,0 +1,133 @@ +import type { StackEvent } from '@aws-sdk/client-cloudformation'; +import type { ResourceEvent } from '../../../lib/api/stack-events'; +import { ResourceErrors } from '../../../lib/api/stack-events/resource-errors'; + +const STACK_ARN = 'arn:aws:cloudformation:us-east-1:123456789012:stack/MyStack/abc'; + +function resourceEvent(event: Partial & Pick): ResourceEvent { + const fullEvent: StackEvent = { + StackId: STACK_ARN, + StackName: 'MyStack', + Timestamp: new Date(), + ...event, + }; + return { + event: fullEvent, + parentStackLogicalIds: [], + isRootStackEvent: false, + }; +} + +describe('hook failure correlation', () => { + test('attaches a preceding hook failure to the resource error it caused', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'Template failed validation', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + }), + ); + + expect(errors.all).toEqual([ + expect.objectContaining({ + logicalId: 'MyBucket', + hookFailures: [ + { + hookType: 'Private::Guard::TestHook', + hookInvocationId: 'hook-invocation-1', + hookStatusReason: 'Template failed validation', + }, + ], + }), + ]); + }); + + test('does not attach a hook failure whose type is not named in the resource status reason', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'Some unrelated failure', + }), + ); + + expect(errors.all[0].hookFailures).toBeUndefined(); + }); + + test('does not correlate a hook failure across different resources', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'OtherResource', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + }), + ); + + expect(errors.all[0].hookFailures).toBeUndefined(); + }); + + test('ignores hook events that did not fail', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_SUCCEEDED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + }), + ); + + expect(errors.all[0].hookFailures).toBeUndefined(); + }); +}); diff --git a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts index cab75bc1e..69e8ea69f 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts @@ -494,8 +494,10 @@ describe('GuardHook GetHookResult fetching', () => { expect(ioHost.notify).toHaveBeenNthCalledWith(2, expect.objectContaining({ level: 'warn', - message: `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. - 'Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion}.`, + message: `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. ` + + 'Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment ' + + "by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. " + + `Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion}.`, }), ); expect(ioHost.notify).toHaveBeenNthCalledWith(3, From 439f5a8527f0b697c415c92518ac69c2f39a0a41 Mon Sep 17 00:00:00 2001 From: Ian Hou <45278651+iankhou@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:37:18 +0000 Subject: [PATCH 2/3] test(cli-integ): cdk diagnose surfaces guard hook failure details Covers the new GetHookResult fetch in the central stack diagnoser: deploys the guard-hook-app fixture's non-compliant stack with --no-rollback, then asserts that cdk diagnose output contains the hook failure details that are only available via the GetHookResult API. --- ...failure-displays-hook-details.integtest.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/diagnose/cdk-diagnose-guard-hook-failure-displays-hook-details.integtest.ts diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/diagnose/cdk-diagnose-guard-hook-failure-displays-hook-details.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/diagnose/cdk-diagnose-guard-hook-failure-displays-hook-details.integtest.ts new file mode 100644 index 000000000..92b843764 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/diagnose/cdk-diagnose-guard-hook-failure-displays-hook-details.integtest.ts @@ -0,0 +1,32 @@ +import { integTest, withSpecificFixture } from '../../../lib'; + +jest.setTimeout(2 * 60 * 60_000); + +integTest( + 'cdk diagnose after guard hook failure displays hook failure details', + withSpecificFixture('guard-hook-app', async (fixture) => { + // Deploy the setup stack which creates the Guard Hook via CloudFormation + await fixture.cdkDeploy('guard-hook-setup'); + + // Attempt to deploy the non-compliant stack; it fails due to the Guard Hook. + // --no-rollback leaves the stack in CREATE_FAILED so diagnose can inspect it. + const deployOutput = await fixture.cdkDeploy('guard-hook-test', { + options: ['--no-rollback'], + allowErrExit: true, + }); + expect(deployOutput).toContain('CREATE_FAILED'); + + // Run cdk diagnose on the failed stack + const diagnoseOutput = await fixture.cdk( + ['--unstable=diagnose', 'diagnose', fixture.fullStackName('guard-hook-test')], + { allowErrExit: true }, + ); + + // diagnose has no live activity stream, so the hook failure details must have + // been fetched via the GetHookResult API and attached to the diagnosis + expect(diagnoseOutput).toContain("Hook 'Private::Guard::TestHook' failed: NonCompliant Rules:"); + expect(diagnoseOutput).toContain('[AWS_S3_Bucket_AccessControl]'); + expect(diagnoseOutput).toContain('• Check was not compliant as property [/Resources/NonCompliantBucket/Properties/AccessControl[L:0,C:91]] existed.'); + expect(diagnoseOutput).toContain('Remediation: AccessControl is deprecated'); + }), +); From 5f6d9c195720f3c7eba92550bfd9739072bb12f4 Mon Sep 17 00:00:00 2001 From: Ian Hou <45278651+iankhou@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:14:11 +0000 Subject: [PATCH 3/3] fix(toolkit-lib): use narrowed problem variant in hook diagnoser tests The Diagnosis wrapper doesn't expose 'problems' directly; use the value returned by assertProblem(), which narrows to the 'problem' variant. --- .../test/api/diagnosing/stack-diagnoser.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts index a5818f9c1..0b9727028 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts @@ -788,8 +788,8 @@ describe('CloudFormationStackDiagnoser', () => { expect(mockCloudFormationClient).toHaveReceivedCommandWith(GetHookResultCommand, { HookResultId: 'hook-invocation-1', }); - assertProblem(result); - const context = result.problems[0].additionalContext ?? []; + const problem = assertProblem(result); + const context = problem.problems[0].additionalContext ?? []; expect(context).toEqual([ expect.objectContaining({ source: 'CloudFormation Hook (Private::Guard::TestHook)', @@ -844,8 +844,8 @@ describe('CloudFormationStackDiagnoser', () => { }); expect(mockCloudFormationClient).not.toHaveReceivedCommand(GetHookResultCommand); - assertProblem(result); - expect(result.problems[0].additionalContext).toBeUndefined(); + const problem = assertProblem(result); + expect(problem.problems[0].additionalContext).toBeUndefined(); }); test('falls back to the hook status reason when GetHookResult returns no details', async () => { @@ -892,8 +892,8 @@ describe('CloudFormationStackDiagnoser', () => { CreationTime: new Date(), }); - assertProblem(result); - const context = result.problems[0].additionalContext ?? []; + const problem = assertProblem(result); + const context = problem.problems[0].additionalContext ?? []; expect(context).toEqual([ expect.objectContaining({ messages: ["Hook 'Private::Guard::TestHook' failed: the terse fallback reason"],