From 3c7848e6b6db0ee45b8a839db6fafdca78e650bc Mon Sep 17 00:00:00 2001 From: Dmitrii Creed Date: Tue, 30 Jun 2026 18:14:24 +0400 Subject: [PATCH] fix(aws): skip STS pre-validation for ambient (OIDC) providers; don't bake creds Re-deploying a stack that was previously deployed with static keys, now under ambient GitHub-OIDC creds, fails at the explicit pulumi-aws provider with 'Invalid credentials configured'. In pulumi-aws v6.83.4 that message means INCOMPLETE creds (not 'no creds' -> that is 'No valid credential sources found'): the eager STS pre-validation chokes on the static->ambient provider transition, even though the runner's ambient env creds are complete (incl. AWS_SESSION_TOKEN, which does reach the plugin) and authorize the real calls. Fix: in ambient mode (empty static keys) keep the provider credential-less so the AWS default chain resolves env creds at call time, and set SkipCredentialsValidation=true to skip the brittle pre-validation. Static keys keep validation on. Shared applyAWSProviderCreds helper (Provider() + cloudtrail region provider) + tests. Deliberately does NOT copy the rotating ambient creds (incl. session token) onto provider inputs: that persists ephemeral creds in the Pulumi checkpoint and diffs the provider every run. Supersedes the first approach after a Codex + Gemini review rejected credential-baking. Signed-off-by: Dmitrii Creed --- .../pulumi/aws/cloudtrail_security_alerts.go | 7 +--- pkg/clouds/pulumi/aws/provider.go | 41 +++++++++++++++---- pkg/clouds/pulumi/aws/provider_test.go | 41 +++++++++++++++++++ 3 files changed, 74 insertions(+), 15 deletions(-) diff --git a/pkg/clouds/pulumi/aws/cloudtrail_security_alerts.go b/pkg/clouds/pulumi/aws/cloudtrail_security_alerts.go index ac3e23ff..701d5019 100644 --- a/pkg/clouds/pulumi/aws/cloudtrail_security_alerts.go +++ b/pkg/clouds/pulumi/aws/cloudtrail_security_alerts.go @@ -613,12 +613,7 @@ func CloudTrailSecurityAlerts(ctx *sdk.Context, stack api.Stack, input api.Resou providerArgs := &sdkAws.ProviderArgs{ Region: sdk.String(cfg.LogGroupRegion), } - if cfg.AccessKey != "" { - providerArgs.AccessKey = sdk.String(cfg.AccessKey) - } - if cfg.SecretAccessKey != "" { - providerArgs.SecretKey = sdk.String(cfg.SecretAccessKey) - } + applyAWSProviderCreds(providerArgs, cfg.AccessKey, cfg.SecretAccessKey) regionProvider, err := sdkAws.NewProvider(ctx, fmt.Sprintf("%s-region-provider", resPrefix), providerArgs) if err != nil { return nil, errors.Wrapf(err, "failed to create region-specific provider for %q", cfg.LogGroupRegion) diff --git a/pkg/clouds/pulumi/aws/provider.go b/pkg/clouds/pulumi/aws/provider.go index 0647fb78..28e039e7 100644 --- a/pkg/clouds/pulumi/aws/provider.go +++ b/pkg/clouds/pulumi/aws/provider.go @@ -64,17 +64,40 @@ func Provider(ctx *sdk.Context, stack api.Stack, input api.ResourceInput, params providerArgs := &sdkAws.ProviderArgs{ Region: sdk.String(pcfg.Region), } - // Pin static creds only when configured; otherwise fall back to the AWS default - // credential chain (OIDC web-identity / instance profile / env). Mirrors the - // guarded handling already in cloudtrail_security_alerts.go. - if pcfg.AccessKey != "" { - providerArgs.AccessKey = sdk.String(pcfg.AccessKey) - } - if pcfg.SecretAccessKey != "" { - providerArgs.SecretKey = sdk.String(pcfg.SecretAccessKey) - } + applyAWSProviderCreds(providerArgs, pcfg.AccessKey, pcfg.SecretAccessKey) provider, err := sdkAws.NewProvider(ctx, input.ToResName(input.Descriptor.Name), providerArgs) return &api.ResourceOutput{ Ref: provider, }, err } + +// applyAWSProviderCreds configures credentials on an explicitly-instantiated +// pulumi-aws provider. +// +// - Static keys present (from SC auth config): pin them, and keep the +// provider's STS pre-validation on (it catches bad static keys early). +// - Otherwise — ambient mode (GitHub OIDC web-identity / instance profile / +// env): leave the provider credential-less so the AWS default credential +// chain resolves creds at call time from the runner environment (incl. +// AWS_SESSION_TOKEN, which DOES reach the plugin process), and skip the +// provider's eager STS pre-validation. +// +// The pre-validation is skipped, not worked around: when a stack that was +// previously deployed with static keys is re-deployed under ambient creds, the +// provider's credential-validation step resolves an incomplete credential set +// (it sees the old static access/secret without a session token) and fails with +// "Invalid credentials configured" — even though the ambient env credentials are +// complete and authorize the real API calls. Skipping validation lets the +// transition proceed; authorization is still enforced by AWS on every call. +// +// We deliberately do NOT copy the ambient env creds (incl. the rotating session +// token) onto the provider inputs: that persists ephemeral credentials in the +// Pulumi checkpoint and produces a provider diff on every run. +func applyAWSProviderCreds(args *sdkAws.ProviderArgs, accessKey, secretAccessKey string) { + if accessKey != "" { + args.AccessKey = sdk.String(accessKey) + args.SecretKey = sdk.String(secretAccessKey) + return + } + args.SkipCredentialsValidation = sdk.Bool(true) +} diff --git a/pkg/clouds/pulumi/aws/provider_test.go b/pkg/clouds/pulumi/aws/provider_test.go index 80347400..67044047 100644 --- a/pkg/clouds/pulumi/aws/provider_test.go +++ b/pkg/clouds/pulumi/aws/provider_test.go @@ -9,6 +9,8 @@ import ( "testing" . "github.com/onsi/gomega" + sdkAws "github.com/pulumi/pulumi-aws/sdk/v6/go/aws" + sdk "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/simple-container-com/api/pkg/api/logger" "github.com/simple-container-com/api/pkg/clouds/aws" @@ -55,3 +57,42 @@ func TestInitStateStore_StaticCredsExported(t *testing.T) { Expect(os.Getenv("AWS_SECRET_ACCESS_KEY")).To(Equal("static-secret")) Expect(os.Getenv("AWS_DEFAULT_REGION")).To(Equal("us-east-1")) } + +// TestApplyAWSProviderCreds_Ambient is the regression guard for the testtmp +// failure: in ambient mode (empty static keys) the explicit provider must be +// left credential-less — so the AWS default chain resolves the runner's env +// creds at call time — with STS pre-validation skipped, so re-deploying a stack +// that previously baked static keys doesn't fail with "Invalid credentials +// configured". Crucially it must NOT bake the (rotating) env creds into inputs. +func TestApplyAWSProviderCreds_Ambient(t *testing.T) { + RegisterTestingT(t) + // Even with ambient creds in the env, they must not be copied onto the args. + t.Setenv("AWS_ACCESS_KEY_ID", "ASIAAMBIENT") + t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-secret") + t.Setenv("AWS_SESSION_TOKEN", "ambient-session") + + args := &sdkAws.ProviderArgs{Region: sdk.String("eu-central-1")} + applyAWSProviderCreds(args, "", "") + + Expect(args.AccessKey).To(BeNil(), "must not bake ambient creds into provider inputs") + Expect(args.SecretKey).To(BeNil()) + Expect(args.Token).To(BeNil(), "must not persist the rotating session token in state") + skip, ok := args.SkipCredentialsValidation.(sdk.Bool) + Expect(ok).To(BeTrue(), "ambient mode must skip the eager STS pre-validation") + Expect(bool(skip)).To(BeTrue()) +} + +// TestApplyAWSProviderCreds_Static confirms static keys are pinned and the +// pre-validation stays ON (it catches bad static keys early). +func TestApplyAWSProviderCreds_Static(t *testing.T) { + RegisterTestingT(t) + args := &sdkAws.ProviderArgs{Region: sdk.String("us-east-1")} + applyAWSProviderCreds(args, "AKIASTATIC", "static-secret") + + ak, _ := args.AccessKey.(sdk.String) + Expect(string(ak)).To(Equal("AKIASTATIC")) + sk, _ := args.SecretKey.(sdk.String) + Expect(string(sk)).To(Equal("static-secret")) + Expect(args.Token).To(BeNil()) + Expect(args.SkipCredentialsValidation).To(BeNil(), "static keys keep validation on") +}