From bc1118652f93f8a2b1929791ab495b7415bf3ebd Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Fri, 31 Jul 2026 11:17:36 +0100 Subject: [PATCH] feat(cli-integ): bootstrap test environments with a random qualifier Instead of bootstrapping every test environment with the fixed default qualifier, each test fixture now bootstraps with its own random qualifier and a per-fixture toolkit stack name. This makes bootstrap bucket names unique across (potentially recycled) test environments, avoiding S3 eventual-consistency collisions on the fixed default bucket name. Apps automatically receive the qualifier: TestFixture keeps an app context dict (seeded with the qualifier) that the app fixtures write into the app's cdk.json. Tests can add more context via addAppContext() or the new nestable withAppContext() block. --- .../cli-integ/lib/with-cdk-app.ts | 143 +++++++++++++++++- .../cdk-assets/asset_helpers.ts | 8 +- .../cdk-assets-uses-profile.integtest.ts | 6 +- ...loy-docker-asset-multi-region.integtest.ts | 10 +- 4 files changed, 150 insertions(+), 17 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 1858f6397..5923a445e 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -56,7 +56,17 @@ export function withSpecificCdkApp( stackNamePrefix, context.output, context.aws, - context.randomString); + context.randomString, + ); + if (context.disableBootstrap) { + // Tests that disable the default bootstrap manage their own bootstrap + // stack (name, qualifier, template) and pass the matching context and + // arguments explicitly. Don't preconfigure a qualifier for them; e.g. + // the legacy bootstrap refuses to run with any qualifier set. + await fixture.removeAppContext('@aws-cdk/core:bootstrapQualifier'); + } else { + await fixture.writeAppContext(); + } await fixture.ecrPublicLogin(); let success = true; @@ -128,6 +138,7 @@ export function withCdkMigrateApp( context.aws, context.randomString, ); + await testFixture.writeAppContext(); let success = true; try { @@ -170,6 +181,34 @@ export function withCDKMigrateFixture(language: string, block: (content: TestFix return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCdkMigrateApp(language, block)), options.aws); } +/** + * Higher order function to add context values to the test app + * + * The context is written to the app's `cdk.json`, merged with the + * framework-provided defaults (like the random bootstrap qualifier) and any + * context added by enclosing `withAppContext` blocks. May be nested any + * number of times; inner blocks override outer blocks on key conflicts. + * + * Example: + * + * ```ts + * integTest('my test', withDefaultFixture(withAppContext({ + * '@aws-cdk/core:newStyleStackSynthesis': 'true', + * }, async (fixture) => { + * // ... + * }))); + * ``` + */ +export function withAppContext( + context: Record, + block: (fixture: TestFixture) => Promise, +): (fixture: TestFixture) => Promise { + return async (fixture: TestFixture) => { + await fixture.addAppContext(context); + await block(fixture); + }; +} + /** * Retry wrapper that executes a test callback up to maxAttempts times * @@ -368,6 +407,7 @@ export class TestFixture extends ShellHelper { public readonly cli: ITestCliSource; public readonly cdkAssets: ITestCliSource; public readonly library: ITestLibrarySource; + private readonly appContext: Record; constructor( public readonly integTestDir: string, @@ -381,12 +421,82 @@ export class TestFixture extends ShellHelper { this.cli = testSource('cli'); this.cdkAssets = testSource('cdkAssets'); this.library = testSource('library'); + + // Every test bootstraps its environment with a random qualifier (instead + // of the fixed default qualifier), so that bootstrap bucket names are + // unique across (potentially recycled) test environments. Putting the + // qualifier into the app context makes sure that: + // + // - Synthesized apps automatically use the matching bootstrap resources + // (`DefaultStackSynthesizer` reads `@aws-cdk/core:bootstrapQualifier`). + // - `cdk bootstrap` invocations that don't pass an explicit `--qualifier` + // default to it as well. + this.appContext = { + '@aws-cdk/core:bootstrapQualifier': this.qualifier, + }; } public log(s: string) { this.output.write(`${s}\n`); } + /** + * Add context values for the test app, and write them to the app's `cdk.json` + * + * Values passed here accumulate with earlier calls, and override the + * framework-provided defaults (like the bootstrap qualifier). Command-line + * `--context` arguments still take precedence over all of these. + */ + public async addAppContext(context: Record, dir?: string) { + Object.assign(this.appContext, context); + await this.writeAppContext(dir); + } + + /** + * Remove a context value for the test app, and update the app's `cdk.json` + * + * Note: this only prevents the key from being written by this fixture; + * a value already present in the app's own `cdk.json` is left alone. + */ + public async removeAppContext(key: string, dir?: string) { + delete this.appContext[key]; + await this.writeAppContext(dir); + } + + /** + * Merge the accumulated app context into the app's `cdk.json` + * + * Also points `toolkitStackName` at the fixture's bootstrap stack, so that + * commands that look up the bootstrap stack by name (e.g. large-template + * deploys and refactors that need the staging bucket) find it without the + * test having to pass `--toolkit-stack-name` everywhere. Command-line + * arguments still take precedence. + * + * Called by the fixture factories as soon as the app directory exists; + * tests normally don't need to call this directly (use `addAppContext` + * or wrap the test block in `withAppContext` instead). + */ + public async writeAppContext(dir?: string) { + const cdkJsonPath = path.join(dir ?? this.integTestDir, 'cdk.json'); + + let cdkJson: any = {}; + try { + cdkJson = JSON.parse(await fs.promises.readFile(cdkJsonPath, { encoding: 'utf-8' })); + } catch (e: any) { + if (e.code !== 'ENOENT') { + throw e; + } + } + + cdkJson.toolkitStackName = this.bootstrapStackName; + cdkJson.context = { + ...cdkJson.context, + ...this.appContext, + }; + + await fs.promises.writeFile(cdkJsonPath, JSON.stringify(cdkJson, undefined, 2), { encoding: 'utf-8' }); + } + /** * Login to the public ECR gallery using the current AWS credentials. * Use this if your test needs to directly pull images outside of a `cdk` or `cdk-assets` command. @@ -662,9 +772,14 @@ export class TestFixture extends ShellHelper { return JSON.parse(fs.readFileSync(templatePath, { encoding: 'utf-8' }).toString()); } + /** + * Look up the ECR repository name of this fixture's bootstrap stack + * + * Expects the environment to have been bootstrapped already with + * `toolkitStackName: fixture.bootstrapStackName` (or via `ensureBootstrapped`, + * which uses that stack name by default). + */ public async bootstrapRepoName(): Promise { - await ensureBootstrapped(this); - const response = await this.aws.cloudFormation.send(new DescribeStacksCommand({})); const stack = (response.Stacks ?? []) @@ -805,13 +920,17 @@ export async function ensureBootstrapped(fixture: TestFixture) { // It doesn't matter for tests: when they want to test something about an actual legacy // bootstrap stack, they'll create a bootstrap stack with a non-default name to test that exact property. const envSpecifier = `aws://${await fixture.aws.account()}/${fixture.aws.region}`; - if (ALREADY_BOOTSTRAPPED_IN_THIS_RUN.has(envSpecifier)) { + // Every fixture bootstraps with its own random qualifier (and stack name), + // so the cache key must include it: another fixture's bootstrap of the same + // environment doesn't help us. + const cacheKey = `${envSpecifier}/${fixture.qualifier}`; + if (ALREADY_BOOTSTRAPPED_IN_THIS_RUN.has(cacheKey)) { return; } if (atmosphereEnabled()) { // when atmosphere is enabled, each test starts with an empty environment - // and needs to deploy the bootstrap stack. in case environments are recylced too quickly, + // and needs to deploy the bootstrap stack. in case environments are recycled too quickly, // cloudformation may think the bootstrap bucket still exists even though it doesnt (because of s3 eventual consistency). // so we retry on the specific error for a while. await bootstrapWithRetryOnBucketExists(envSpecifier, fixture); @@ -822,12 +941,20 @@ export async function ensureBootstrapped(fixture: TestFixture) { // when using the atmosphere service, every test needs to bootstrap // its own environment. if (!atmosphereEnabled()) { - ALREADY_BOOTSTRAPPED_IN_THIS_RUN.add(envSpecifier); + ALREADY_BOOTSTRAPPED_IN_THIS_RUN.add(cacheKey); } } async function doBootstrap(envSpecifier: string, fixture: TestFixture, allowErrExit: boolean) { - return fixture.cdk(['bootstrap', '--bootstrap-kms-key-id', 'AWS_MANAGED_KEY', envSpecifier], { + return fixture.cdk(['bootstrap', + '--bootstrap-kms-key-id', 'AWS_MANAGED_KEY', + // Use a random qualifier and a per-fixture stack name, so that bootstrap + // resource names (in particular the bucket name) are unique across + // (potentially recycled) test environments, and so that concurrent tests + // in the same environment don't fight over a shared bootstrap stack. + '--qualifier', fixture.qualifier, + '--toolkit-stack-name', fixture.bootstrapStackName, + envSpecifier], { modEnv: { // Even for v1, use new bootstrap CDK_NEW_BOOTSTRAP: '1', @@ -842,7 +969,7 @@ async function doBootstrap(envSpecifier: string, fixture: TestFixture, allowErrE async function bootstrapWithRetryOnBucketExists(envSpecifier: string, fixture: TestFixture) { const account = await fixture.aws.account(); const retryAfterSeconds = 30; - const bootstrapBucket = `cdk-hnb659fds-assets-${account}-${fixture.aws.region}`; + const bootstrapBucket = `cdk-${fixture.qualifier}-assets-${account}-${fixture.aws.region}`; // s3 says that a bucket deletion can take up to an hour to be fully visible. // empirically we see that a few minutes is enough though. lets give 10 to be on the safe(r) side. diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/asset_helpers.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/asset_helpers.ts index 3197cdd33..b67ecc439 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/asset_helpers.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/asset_helpers.ts @@ -10,8 +10,8 @@ export async function writeFileAsset(fixture: TestFixture) { for (const toCreate of [relativeAssetFile]) { await fs.writeFile(path.join(fixture.integTestDir, toCreate), 'some asset file'); } - const bucketName = `cdk-hnb659fds-assets-${account}-${region}`; - const assumeRoleArn = `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`; + const bucketName = `cdk-${fixture.qualifier}-assets-${account}-${region}`; + const assumeRoleArn = `arn:\${AWS::Partition}:iam::${account}:role/cdk-${fixture.qualifier}-file-publishing-role-${account}-${region}`; return { relativeAssetFile, @@ -36,8 +36,8 @@ export async function writeDockerAsset(fixture: TestFixture) { const account = await fixture.aws.account(); const region = fixture.aws.region; - const repositoryName = `cdk-hnb659fds-container-assets-${account}-${region}`; - const assumeRoleArn = `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-image-publishing-role-${account}-${region}`; + const repositoryName = `cdk-${fixture.qualifier}-container-assets-${account}-${region}`; + const assumeRoleArn = `arn:\${AWS::Partition}:iam::${account}:role/cdk-${fixture.qualifier}-image-publishing-role-${account}-${region}`; const repositoryDomain = `${account}.dkr.ecr.${region}.amazonaws.com`; return { diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts index 828b10ade..54a6f2d1e 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-assets/cdk-assets-uses-profile.integtest.ts @@ -10,7 +10,7 @@ integTest('cdk-assets uses profile when specified', withDefaultFixture(async (fi const account = await fixture.aws.account(); const region = fixture.aws.region; - const bucketName = `cdk-hnb659fds-assets-${account}-${region}`; + const bucketName = `cdk-${fixture.qualifier}-assets-${account}-${region}`; // Write some asset files. Its important to have more than 1 because cdk-assets // code has some funky state mutations that happens on each asset publishing. @@ -31,7 +31,7 @@ integTest('cdk-assets uses profile when specified', withDefaultFixture(async (fi destinations: { current: { region, - assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`, + assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-${fixture.qualifier}-file-publishing-role-${account}-${region}`, bucketName, objectKey: `test-file1-${Date.now()}.json`, }, @@ -45,7 +45,7 @@ integTest('cdk-assets uses profile when specified', withDefaultFixture(async (fi destinations: { current: { region, - assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-hnb659fds-file-publishing-role-${account}-${region}`, + assumeRoleArn: `arn:\${AWS::Partition}:iam::${account}:role/cdk-${fixture.qualifier}-file-publishing-role-${account}-${region}`, bucketName, objectKey: `test-file2-${Date.now()}.json`, }, diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts index 52e8752e9..ab811b9c0 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts @@ -7,9 +7,15 @@ integTest( const availableRegions = (process.env.AWS_REGIONS ?? 'us-east-1,us-west-2').split(','); const secondaryRegion = availableRegions.find((r) => r !== primaryRegion) ?? 'us-west-2'; - // Bootstrap the secondary region + // Bootstrap the secondary region (with the same random qualifier the app + // is synthesized with, and a unique stack name to avoid clashing with + // other tests bootstrapping this region) const account = await fixture.aws.account(); - await fixture.cdk(['bootstrap', '--bootstrap-kms-key-id', 'AWS_MANAGED_KEY', `aws://${account}/${secondaryRegion}`], { + await fixture.cdk(['bootstrap', + '--bootstrap-kms-key-id', 'AWS_MANAGED_KEY', + '--qualifier', fixture.qualifier, + '--toolkit-stack-name', fixture.bootstrapStackName, + `aws://${account}/${secondaryRegion}`], { modEnv: { CDK_NEW_BOOTSTRAP: '1' }, });