diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/diagnose/cdk-diagnose-guard-hook-failure-displays-hook-details.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/diagnose/cdk-diagnose-guard-hook-failure-displays-hook-details.integtest.ts new file mode 100644 index 000000000..92b843764 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/diagnose/cdk-diagnose-guard-hook-failure-displays-hook-details.integtest.ts @@ -0,0 +1,32 @@ +import { integTest, withSpecificFixture } from '../../../lib'; + +jest.setTimeout(2 * 60 * 60_000); + +integTest( + 'cdk diagnose after guard hook failure displays hook failure details', + withSpecificFixture('guard-hook-app', async (fixture) => { + // Deploy the setup stack which creates the Guard Hook via CloudFormation + await fixture.cdkDeploy('guard-hook-setup'); + + // Attempt to deploy the non-compliant stack; it fails due to the Guard Hook. + // --no-rollback leaves the stack in CREATE_FAILED so diagnose can inspect it. + const deployOutput = await fixture.cdkDeploy('guard-hook-test', { + options: ['--no-rollback'], + allowErrExit: true, + }); + expect(deployOutput).toContain('CREATE_FAILED'); + + // Run cdk diagnose on the failed stack + const diagnoseOutput = await fixture.cdk( + ['--unstable=diagnose', 'diagnose', fixture.fullStackName('guard-hook-test')], + { allowErrExit: true }, + ); + + // diagnose has no live activity stream, so the hook failure details must have + // been fetched via the GetHookResult API and attached to the diagnosis + expect(diagnoseOutput).toContain("Hook 'Private::Guard::TestHook' failed: NonCompliant Rules:"); + expect(diagnoseOutput).toContain('[AWS_S3_Bucket_AccessControl]'); + expect(diagnoseOutput).toContain('• Check was not compliant as property [/Resources/NonCompliantBucket/Properties/AccessControl[L:0,C:91]] existed.'); + expect(diagnoseOutput).toContain('Remediation: AccessControl is deprecated'); + }), +); diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts index e66cd37e7..fc9a82498 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts @@ -12,7 +12,7 @@ import type { EnvironmentResources } from '../environment'; import type { IoHelper } from '../io/private/io-helper'; import type { ISourceTracer } from '../source-tracing/private/source-tracing'; import { PollRange, StackEventPoller } from '../stack-events'; -import { formatHookResultDetails, isFailedHookStatus } from '../stack-events/hook-result-details'; +import { fetchHookResultDetails, formatHookResultDetails, isFailedHookStatus, normalizeHookMessage } from '../stack-events/hook-result-details'; import type { ResourceError, ResourceErrors } from '../stack-events/resource-errors'; import { StackStatus } from '../stack-events/stack-status'; @@ -36,6 +36,18 @@ export interface CloudFormationStackDiagnoserProps { * level and their information is simply not added to the output. */ readonly additionalExplorationSdkProvider?: SdkProvider; + + /** + * Whether to fetch the failure details of hooks that failed resources, via the + * `GetHookResult` API, and attach them to the diagnosis. + * + * During a deployment the activity stream already shows these details live, so + * `cdk deploy` leaves this off to avoid repeating them. `cdk diagnose` has no + * live stream and enables it. + * + * @default false + */ + readonly fetchHookFailureDetails?: boolean; } export type SdkProvider = () => Promise; @@ -417,14 +429,12 @@ export class CloudFormationStackDiagnoser { */ private async _hookResultDetails(hook: HookResultSummary): Promise { if (hook.HookResultId) { - try { - const result = await this.cfn.getHookResult({ HookResultId: hook.HookResultId }); - const details = formatHookResultDetails(result); - if (details) { - return details; - } - } catch (e: any) { - await this.props.ioHelper.defaults.debug(`Could not fetch hook result ${hook.HookResultId}: ${e.message}`); + const details = await fetchHookResultDetails(this.cfn, hook.HookResultId, { + ioHelper: this.props.ioHelper, + envResources: this.props.envResources, + }); + if (details) { + return details; } } return formatHookResultDetails(hook); @@ -443,19 +453,55 @@ export class CloudFormationStackDiagnoser { : this.props.sourceTracer.traceStack(err.stackArn, err.parentStackLogicalIds); // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism - const [sourceTrace, additionalContext] = await Promise.all([ + const [sourceTrace, additionalContext, hookContext] = await Promise.all([ sourceTracePromise, this.investigateResourceBestEffort(err), + this.hookFailureContext(err), ]); + const allContext = [...hookContext, ...additionalContext]; return { ...err, sourceTrace, topLevelStackHierarchicalId: this.props.topLevelStackHierarchicalId, - ...(additionalContext.length > 0 ? { additionalContext } : {}), + ...(allContext.length > 0 ? { additionalContext: allContext } : {}), }; } + /** + * Fetch the failure details of the hooks that caused the given resource error. + * + * The resource's own status reason only names the hooks that failed it (e.g. + * "The following hook(s) failed: [X]"); the actual failure details live in the + * hook results API. Falls back to the hook event's status reason if the fetch + * fails or returns no details. + * + * Only active when `fetchHookFailureDetails` is set (i.e. for `cdk diagnose`); + * during a deployment the activity stream already shows these details. + */ + private async hookFailureContext(err: ResourceError): Promise { + if (!this.props.fetchHookFailureDetails) { + return []; + } + const ret: AdditionalDiagnosticContext[] = []; + for (const hook of err.hookFailures ?? []) { + let details = hook.hookInvocationId + ? await fetchHookResultDetails(this.cfn, hook.hookInvocationId, { + ioHelper: this.props.ioHelper, + envResources: this.props.envResources, + }) + : undefined; + details = details ?? (hook.hookStatusReason ? normalizeHookMessage(hook.hookStatusReason) : undefined); + if (details) { + ret.push({ + source: `CloudFormation Hook (${hook.hookType})`, + messages: details.split('\n').map((line, i) => i === 0 ? `Hook '${hook.hookType}' failed: ${line}` : line), + }); + } + } + return ret; + } + private async investigateResourceBestEffort(err: ResourceError) { const sdk = await this.additionalExplorationSdk(); if (!sdk) { diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts index a29de63fa..7f6875406 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/hook-result-details.ts @@ -1,4 +1,9 @@ +import * as util from 'node:util'; import type { GetHookResultCommandOutput, HookResultSummary } from '@aws-sdk/client-cloudformation'; +import type { ICloudFormationClient } from '../aws-auth/private'; +import { isAccessDeniedError } from '../aws-auth/util'; +import type { EnvironmentResources } from '../environment'; +import type { IoHelper } from '../io/private'; /** * The relevant parts of a hook result, common to `GetHookResult` outputs and @@ -53,6 +58,70 @@ export function isFailedHookStatus(status: string | undefined): boolean { return status === 'HOOK_COMPLETE_FAILED' || status === 'HOOK_FAILED'; } +export interface FetchHookResultDetailsOptions { + /** + * The IoHelper used to warn when the fetch fails. + */ + readonly ioHelper: IoHelper; + + /** + * Environment resources, used to look up the bootstrap toolkit version when + * diagnosing hook result fetch failures caused by missing permissions. + * + * @default - Bootstrap version is not reported in error messages + */ + readonly envResources?: EnvironmentResources; +} + +/** + * Fetch a hook invocation's failure details via the `GetHookResult` API and format + * them into a human-readable string. + * + * For Guard Hooks the details come from the failed annotations; for other hooks + * (e.g. Lambda Hooks) they come from the hook result's own status reason. + * Returns undefined if the fetch fails (emitting a warning, with a bootstrap + * upgrade hint if the failure looks permissions-related) or the result carries + * no failure details. + */ +export async function fetchHookResultDetails( + cfn: ICloudFormationClient, + hookInvocationId: string, + options: FetchHookResultDetailsOptions, +): Promise { + try { + const result = await cfn.getHookResult({ HookResultId: hookInvocationId }); + return formatHookResultDetails(result); + } catch (e: any) { + const errorMessage = e instanceof Error ? e.message : String(e); + + const isPermissionsError = + isAccessDeniedError(e) || + (typeof errorMessage === 'string' && errorMessage.toLowerCase().includes('not authorized to perform: cloudformation:gethookresult')); + + if (isPermissionsError && options.envResources) { + let currentVersion: number | undefined = undefined; + try { + currentVersion = (await options.envResources.lookupToolkit()).version; + } catch { + // ignore errors looking up the bootstrap version + } + + await options.ioHelper.defaults.warn( + `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. ` + + 'Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment ' + + "by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. " + + `Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion ?? 'unknown'}.`, + ); + } else { + await options.ioHelper.defaults.warn( + util.format('Failed to fetch Hook details for invocation %s: %s', hookInvocationId, errorMessage), + ); + } + + return undefined; + } +} + /** * Trims leading/trailing whitespace, collapses all internal whitespace * (including newlines) to a single space, and truncates to `maxChars` diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts index 943216101..8f09e54b7 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts @@ -1,8 +1,33 @@ import type { StackEvent } from '@aws-sdk/client-cloudformation'; +import { isFailedHookStatus } from './hook-result-details'; import type { ResourceEvent } from './stack-event-poller'; import { DeploymentErrorCodes } from '../../toolkit/toolkit-error'; import { isCancellationEvent, isErrorEvent, isRegularResourceEvent } from '../../util/cloudformation'; +/** + * A failed CloudFormation Hook invocation that caused a resource error. + */ +export interface ResourceHookFailure { + /** + * The type name of the hook that failed (e.g. `Private::Guard::TestHook`) + */ + readonly hookType: string; + + /** + * The unique identifier of the hook invocation. + * + * Can be used to fetch the full failure details via the `GetHookResult` API. + */ + readonly hookInvocationId?: string; + + /** + * The status reason reported on the hook's stack event. + * + * This is usually a terse summary; the `GetHookResult` API has richer details. + */ + readonly hookStatusReason?: string; +} + export interface ResourceError { /** * The stack this resource error occurred in @@ -56,6 +81,15 @@ export interface ResourceError { * and early-validation errors. */ readonly timestamp?: Date; + + /** + * CloudFormation Hook failures that caused this resource error, if any. + * + * Hook failures are reported on separate stack events from the resource failure + * itself; they are correlated to the resource error by logical ID and by the + * hook type appearing in the resource's status reason. + */ + readonly hookFailures?: ResourceHookFailure[]; } /** @@ -69,6 +103,16 @@ export class ResourceErrors { */ private readonly _errors: ResourceError[] = []; + /** + * Failed hook invocations seen so far, keyed by stack ARN + logical ID of the + * resource they were invoked on. + * + * Hook failures are reported on their own events (usually with an `IN_PROGRESS` + * resource status), before the resource failure event they cause. We remember + * them here so they can be attached to that resource error when it arrives. + */ + private readonly seenHookFailures = new Map(); + public isEmpty() { return this._errors.length === 0; } @@ -86,16 +130,44 @@ export class ResourceErrors { */ public update(...events: ResourceEvent[]) { for (const event of events) { + this.trackHookFailure(event.event); if (isErrorEvent(event.event)) { // Cancelled is not an interesting failure reason, nor is the stack message (stack // message will just say something like "stack failed to update") if (!isCancellationEvent(event.event) && isRegularResourceEvent(event.event)) { - this._errors.push(errorFromEvent(event)); + this._errors.push(errorFromEvent(event, this.hookFailuresFor(event.event))); } } } } + private trackHookFailure(event: StackEvent) { + if (!isFailedHookStatus(event.HookStatus) || !event.HookType || !event.LogicalResourceId || !event.StackId) { + return; + } + const key = hookFailureKey(event); + const failures = this.seenHookFailures.get(key) ?? []; + failures.push({ + hookType: event.HookType, + hookInvocationId: event.HookInvocationId, + hookStatusReason: event.HookStatusReason, + }); + this.seenHookFailures.set(key, failures); + } + + /** + * Return the hook failures that caused the given resource failure, if any. + * + * A hook failure belongs to a resource failure when it was recorded for the same + * resource, and its hook type is mentioned in the resource's status reason (e.g. + * "The following hook(s) failed: [Private::Guard::TestHook]"). + */ + private hookFailuresFor(event: StackEvent): ResourceHookFailure[] { + const recorded = this.seenHookFailures.get(hookFailureKey(event)) ?? []; + const reason = event.ResourceStatusReason ?? ''; + return recorded.filter((h) => reason.includes(h.hookType)); + } + /** * Take our best guess at the error code of the root cause * @@ -123,9 +195,7 @@ export class ResourceErrors { } } -function errorFromEvent(ev: ResourceEvent): ResourceError { - // FIXME: Check hooks - +function errorFromEvent(ev: ResourceEvent, hookFailures: ResourceHookFailure[]): ResourceError { return { logicalId: ev.event.LogicalResourceId ?? '', message: ev.event.ResourceStatusReason ?? '', @@ -135,9 +205,14 @@ function errorFromEvent(ev: ResourceEvent): ResourceError { errorCode: extractErrorCode(ev.event), physicalId: ev.event.PhysicalResourceId, timestamp: ev.event.Timestamp, + ...(hookFailures.length > 0 ? { hookFailures } : {}), }; } +function hookFailureKey(event: StackEvent): string { + return `${event.StackId ?? ''}|${event.LogicalResourceId ?? ''}`; +} + /** * Extract an error code from the given stack event. * diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts index 28326f8b0..a7b8a2aae 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/stack-activity-monitor.ts @@ -1,13 +1,12 @@ import { randomUUID } from 'node:crypto'; import * as util from 'node:util'; import type { CloudFormationStackArtifact } from '@aws-cdk/cloud-assembly-api'; -import { formatHookResultDetails } from './hook-result-details'; +import { fetchHookResultDetails } from './hook-result-details'; import { StackEventPoller, PollRange } from './stack-event-poller'; import { StackProgressMonitor } from './stack-progress-monitor'; import type { StackActivity } from '../../payloads/stack-activity'; import { stackNameFromArn } from '../../util/cloudformation'; import type { ICloudFormationClient } from '../aws-auth/private'; -import { isAccessDeniedError } from '../aws-auth/util'; import type { EnvironmentResources } from '../environment'; import { IO, type IoHelper } from '../io/private'; import { resourceMetadata } from '../resource-metadata/resource-metadata'; @@ -244,49 +243,6 @@ export class StackActivityMonitor { return resourceMetadata(this.stack, logicalId); } - /** - * Fetches hook failure details via the GetHookResult API and formats them into - * a human-readable string. - * - * For Guard Hooks the details come from the failed annotations; for other hooks - * (e.g. Lambda Hooks) they come from the hook result's own status reason. - * Returns undefined if the fetch fails or the result carries no failure details. - */ - private async fetchHookResultDetails(hookInvocationId: string): Promise { - try { - const result = await this.cfn.getHookResult({ HookResultId: hookInvocationId }); - return formatHookResultDetails(result); - } catch (e: any) { - const errorMessage = e instanceof Error ? e.message : String(e); - - const isPermissionsError = - isAccessDeniedError(e) || - (typeof errorMessage === 'string' && errorMessage.toLowerCase().includes('not authorized to perform: cloudformation:gethookresult')); - - if (isPermissionsError && this.envResources) { - let currentVersion: number | undefined = undefined; - try { - currentVersion = (await this.envResources.lookupToolkit()).version; - } catch { - // ignore errors looking up the bootstrap version - } - - await this.ioHelper.defaults.warn( - util.format( - `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. - 'Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion ?? 'unknown'}.`, - ), - ); - } else { - await this.ioHelper.defaults.warn( - util.format('Failed to fetch Hook details for invocation %s: %s', hookInvocationId, errorMessage), - ); - } - - return undefined; - } - } - /** * Reads all new events from the stack history * @@ -300,7 +256,10 @@ export class StackActivityMonitor { // If this is a failed hook event with an invocation ID, fetch the failure details if (resourceEvent.event.HookInvocationId) { - const details = await this.fetchHookResultDetails(resourceEvent.event.HookInvocationId); + const details = await fetchHookResultDetails(this.cfn, resourceEvent.event.HookInvocationId, { + ioHelper: this.ioHelper, + envResources: this.envResources, + }); if (details) { resourceEvent.event.HookStatusReason = details; } diff --git a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts index 48726e89e..912613040 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/toolkit/toolkit.ts @@ -648,6 +648,7 @@ export class Toolkit extends CloudAssemblySourceBuilder { ioHelper, topLevelStackHierarchicalId: stack.hierarchicalId, additionalExplorationSdkProvider: () => Promise.resolve(stackEnv.sdk), + fetchHookFailureDetails: true, }); const diagnosis = await diagnoser.diagnoseFromFresh(stack.stackName); diff --git a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts index 988675249..0b9727028 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts @@ -39,6 +39,16 @@ function makeDiagnoser(topLevelStackHierarchicalId = 'TestStack') { }); } +function makeHookFetchingDiagnoser() { + return new CloudFormationStackDiagnoser({ + sdk, + sourceTracer: fakeTracer, + ioHelper: ioHost.asHelper('diagnose'), + topLevelStackHierarchicalId: 'TestStack', + fetchHookFailureDetails: true, + }); +} + /** * A fake source tracer that records all calls and returns a fixed trace */ @@ -721,6 +731,175 @@ describe('CloudFormationStackDiagnoser', () => { })], }); }); + + test('enriches a resource error with fetched hook failure details', async () => { + const errors = new ResourceErrors(); + errors.update( + { + event: { + EventId: 'evt-hook', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'terse reason', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + { + event: { + EventId: 'evt-fail', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + ); + + mockCloudFormationClient.on(GetHookResultCommand).resolves({ + HookResultId: 'hook-invocation-1', + Status: 'HOOK_COMPLETE_FAILED', + Annotations: [{ + AnnotationName: 'AWS_S3_Bucket_AccessControl', + Status: 'FAILED', + StatusMessage: 'Check was not compliant.', + RemediationMessage: 'AccessControl is deprecated', + }], + } as any); + + const result = await makeHookFetchingDiagnoser().diagnoseFromErrorCollection(errors, { + StackName: 'MyStack', + StackStatus: 'UPDATE_FAILED', + CreationTime: new Date(), + }); + + expect(mockCloudFormationClient).toHaveReceivedCommandWith(GetHookResultCommand, { + HookResultId: 'hook-invocation-1', + }); + const problem = assertProblem(result); + const context = problem.problems[0].additionalContext ?? []; + expect(context).toEqual([ + expect.objectContaining({ + source: 'CloudFormation Hook (Private::Guard::TestHook)', + messages: expect.arrayContaining([ + expect.stringContaining("Hook 'Private::Guard::TestHook' failed"), + '[AWS_S3_Bucket_AccessControl]', + ]), + }), + ]); + }); + + test('does not fetch hook details when fetchHookFailureDetails is off (deploy)', async () => { + const errors = new ResourceErrors(); + errors.update( + { + event: { + EventId: 'evt-hook', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'terse reason', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + { + event: { + EventId: 'evt-fail', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + ); + + const result = await makeDiagnoser().diagnoseFromErrorCollection(errors, { + StackName: 'MyStack', + StackStatus: 'UPDATE_FAILED', + CreationTime: new Date(), + }); + + expect(mockCloudFormationClient).not.toHaveReceivedCommand(GetHookResultCommand); + const problem = assertProblem(result); + expect(problem.problems[0].additionalContext).toBeUndefined(); + }); + + test('falls back to the hook status reason when GetHookResult returns no details', async () => { + const errors = new ResourceErrors(); + errors.update( + { + event: { + EventId: 'evt-hook', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'the terse fallback reason', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + { + event: { + EventId: 'evt-fail', + StackId: 'arn:stack', + StackName: 'MyStack', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + Timestamp: new Date(), + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }, + ); + + mockCloudFormationClient.on(GetHookResultCommand).rejects(new Error('not authorized')); + + const result = await makeHookFetchingDiagnoser().diagnoseFromErrorCollection(errors, { + StackName: 'MyStack', + StackStatus: 'UPDATE_FAILED', + CreationTime: new Date(), + }); + + const problem = assertProblem(result); + const context = problem.problems[0].additionalContext ?? []; + expect(context).toEqual([ + expect.objectContaining({ + messages: ["Hook 'Private::Guard::TestHook' failed: the terse fallback reason"], + }), + ]); + }); }); describe('CloudTrail investigation wiring', () => { diff --git a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts new file mode 100644 index 000000000..df33295b0 --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts @@ -0,0 +1,133 @@ +import type { StackEvent } from '@aws-sdk/client-cloudformation'; +import type { ResourceEvent } from '../../../lib/api/stack-events'; +import { ResourceErrors } from '../../../lib/api/stack-events/resource-errors'; + +const STACK_ARN = 'arn:aws:cloudformation:us-east-1:123456789012:stack/MyStack/abc'; + +function resourceEvent(event: Partial & Pick): ResourceEvent { + const fullEvent: StackEvent = { + StackId: STACK_ARN, + StackName: 'MyStack', + Timestamp: new Date(), + ...event, + }; + return { + event: fullEvent, + parentStackLogicalIds: [], + isRootStackEvent: false, + }; +} + +describe('hook failure correlation', () => { + test('attaches a preceding hook failure to the resource error it caused', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + HookStatusReason: 'Template failed validation', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + }), + ); + + expect(errors.all).toEqual([ + expect.objectContaining({ + logicalId: 'MyBucket', + hookFailures: [ + { + hookType: 'Private::Guard::TestHook', + hookInvocationId: 'hook-invocation-1', + hookStatusReason: 'Template failed validation', + }, + ], + }), + ]); + }); + + test('does not attach a hook failure whose type is not named in the resource status reason', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'Some unrelated failure', + }), + ); + + expect(errors.all[0].hookFailures).toBeUndefined(); + }); + + test('does not correlate a hook failure across different resources', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'OtherResource', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_FAILED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + }), + ); + + expect(errors.all[0].hookFailures).toBeUndefined(); + }); + + test('ignores hook events that did not fail', () => { + const errors = new ResourceErrors(); + + errors.update( + resourceEvent({ + EventId: 'evt-hook', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'UPDATE_IN_PROGRESS', + HookStatus: 'HOOK_COMPLETE_SUCCEEDED', + HookType: 'Private::Guard::TestHook', + HookInvocationId: 'hook-invocation-1', + }), + resourceEvent({ + EventId: 'evt-fail', + LogicalResourceId: 'MyBucket', + ResourceType: 'AWS::S3::Bucket', + ResourceStatus: 'CREATE_FAILED', + ResourceStatusReason: 'The following hook(s) failed: [Private::Guard::TestHook]', + }), + ); + + expect(errors.all[0].hookFailures).toBeUndefined(); + }); +}); diff --git a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts index cab75bc1e..69e8ea69f 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/stack-activity-monitor.test.ts @@ -494,8 +494,10 @@ describe('GuardHook GetHookResult fetching', () => { expect(ioHost.notify).toHaveBeenNthCalledWith(2, expect.objectContaining({ level: 'warn', - message: `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. - 'Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion}.`, + message: `Failed to fetch result details for Hook invocation ${hookInvocationId}: ${errorMessage}. ` + + 'Make sure you have permissions to call the GetHookResult API, or re-bootstrap your environment ' + + "by running 'cdk bootstrap' to update the Bootstrap CDK Toolkit stack. " + + `Bootstrap toolkit stack version 31 or later is needed; current version: ${currentVersion}.`, }), ); expect(ioHost.notify).toHaveBeenNthCalledWith(3,