cross account roles cilium

operator/cmd/provider_aws_flags.go

@@ -50,5 +50,8 @@ func (hook *awsFlagsHooks) RegisterProviderFlag(cmd *cobra.Command, vp *viper.Vi
 	flags.String(operatorOption.EC2APIEndpoint, "", "AWS API endpoint for the EC2 service")
 	option.BindEnv(vp, operatorOption.EC2APIEndpoint)
 
+	flags.String(operatorOption.AWSCrossAccountRoleARN, "", "ARN of the IAM role in the dedicated rbx-cilium account for cross-account IPAM")
+	option.BindEnv(vp, operatorOption.AWSCrossAccountRoleARN)
+
 	vp.BindPFlags(flags)
 }

operator/option/config.go

@@ -131,6 +131,14 @@ const (
 	// node
 	AWSUsePrimaryAddress = "aws-use-primary-address"
 
+	// AWSCrossAccountRoleARN is the ARN of the IAM role in the dedicated rbx-cilium account.
+	// When not empty, the Cilium Operator assumes this role to execute ENI lifecycle operations
+	// (create, delete, assign/unassign secondary IPs) in the target account. The local cluster
+	// instance profile continues to handle local instance-level operations (attach, describe instances).
+	// Required to enable cross-account IPAM when managing pod subnets shared via AWS RAM
+	// into the rbx-cilium sandbox account.
+	AWSCrossAccountRoleARN = "aws-cross-account-role"
+
 	// Azure options
 
 	// AzureSubscriptionID is the subscription ID to use when accessing the Azure API
@@ -332,6 +340,10 @@ type OperatorConfig struct {
 	// e.g. "ec2-fips.us-west-1.amazonaws.com" to use a FIPS endpoint in the us-west-1 region.
 	EC2APIEndpoint string
 
+	// AWSCrossAccountRoleARN is the ARN of the IAM role in the rbx-cilium account assumed
+	// for cross-account ENI lifecycle operations.
+	AWSCrossAccountRoleARN string
+
 	// Azure options
 
 	// AzureSubscriptionID is the subscription ID to use when accessing the Azure API
@@ -451,6 +463,7 @@ func (c *OperatorConfig) Populate(logger *slog.Logger, vp *viper.Viper) {
 	c.AWSEnablePrefixDelegation = vp.GetBool(AWSEnablePrefixDelegation)
 	c.AWSUsePrimaryAddress = vp.GetBool(AWSUsePrimaryAddress)
 	c.EC2APIEndpoint = vp.GetString(EC2APIEndpoint)
+	c.AWSCrossAccountRoleARN = vp.GetString(AWSCrossAccountRoleARN)
 	c.ExcessIPReleaseDelay = vp.GetInt(ExcessIPReleaseDelay)
 	c.ENIGarbageCollectionInterval = vp.GetDuration(ENIGarbageCollectionInterval)
 

pkg/aws/ec2/ec2.go

@@ -16,9 +16,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
 
 	"github.com/cilium/cilium/pkg/api/helpers"
 	eniTypes "github.com/cilium/cilium/pkg/aws/eni/types"
@@ -42,22 +44,23 @@ const (
 	// requires looking at the error message to get the actual reason. See SubnetFullErrMsgStr for example.
 	InvalidParameterValueStr = "InvalidParameterValue"
 
-	AssignPrivateIpAddresses        = "AssignPrivateIpAddresses"
-	AssociateAddress                = "AssociateAddress"
-	AttachNetworkInterface          = "AttachNetworkInterface"
-	CreateNetworkInterface          = "CreateNetworkInterface"
-	DeleteNetworkInterface          = "DeleteNetworkInterface"
-	DescribeAddresses               = "DescribeAddresses"
-	DescribeInstances               = "DescribeInstances"
-	DescribeInstanceTypes           = "DescribeInstanceTypes"
-	DescribeNetworkInterfaces       = "DescribeNetworkInterfaces"
-	DescribeSecurityGroups          = "DescribeSecurityGroups"
-	DescribeSubnets                 = "DescribeSubnets"
-	DescribeVpcs                    = "DescribeVpcs"
-	DescribeRouteTables             = "DescribeRouteTables"
-	ModifyNetworkInterface          = "ModifyNetworkInterface"
-	ModifyNetworkInterfaceAttribute = "ModifyNetworkInterfaceAttribute"
-	UnassignPrivateIpAddresses      = "UnassignPrivateIpAddresses"
+	AssignPrivateIpAddresses         = "AssignPrivateIpAddresses"
+	AssociateAddress                 = "AssociateAddress"
+	AttachNetworkInterface           = "AttachNetworkInterface"
+	CreateNetworkInterface           = "CreateNetworkInterface"
+	CreateNetworkInterfacePermission = "CreateNetworkInterfacePermission"
+	DeleteNetworkInterface           = "DeleteNetworkInterface"
+	DescribeAddresses                = "DescribeAddresses"
+	DescribeInstances                = "DescribeInstances"
+	DescribeInstanceTypes            = "DescribeInstanceTypes"
+	DescribeNetworkInterfaces        = "DescribeNetworkInterfaces"
+	DescribeSecurityGroups           = "DescribeSecurityGroups"
+	DescribeSubnets                  = "DescribeSubnets"
+	DescribeVpcs                     = "DescribeVpcs"
+	DescribeRouteTables              = "DescribeRouteTables"
+	ModifyNetworkInterface           = "ModifyNetworkInterface"
+	ModifyNetworkInterfaceAttribute  = "ModifyNetworkInterfaceAttribute"
+	UnassignPrivateIpAddresses       = "UnassignPrivateIpAddresses"
 )
 
 var syslogAttr = []any{logfields.LogSubsys, "ec2"}
@@ -124,6 +127,29 @@ func NewConfig(ctx context.Context) (aws.Config, error) {
 	return cfg, nil
 }
 
+// NewCrossAccountConfig returns an aws.Config that assumes the given IAM role ARN.
+// The base config (with region and retry settings) is reused; maybe this will need a change?
+func NewCrossAccountConfig(ctx context.Context, baseConfig aws.Config, roleARN string) (aws.Config, error) {
+	stsClient := sts.NewFromConfig(baseConfig)
+	creds := stscreds.NewAssumeRoleProvider(stsClient, roleARN)
+	cfg := baseConfig.Copy()
+	cfg.Credentials = aws.NewCredentialsCache(creds)
+	// make a call with it confirm the creds work rather than waiting for object's first call
+	if _, err := cfg.Credentials.Retrieve(ctx); err != nil {
+		return aws.Config{}, fmt.Errorf("unable to assume cross-account role %s: %w", roleARN, err)
+	}
+	return cfg, nil
+}
+
+// GetLocalAccountID returns the AWS account ID of the instance running this process.
+func GetLocalAccountID(ctx context.Context, cfg aws.Config) (string, error) {
+	doc, err := imds.NewFromConfig(cfg).GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
+	if err != nil {
+		return "", fmt.Errorf("unable to retrieve instance identity document: %w", err)
+	}
+	return doc.AccountID, nil
+}
+
 // NewSubnetsFilters transforms a map of tags and values and a slice of subnets
 // into a slice of ec2.Filter adequate to filter AWS subnets.
 func NewSubnetsFilters(tags map[string]string, ids []string) []ec2_types.Filter {
@@ -733,6 +759,24 @@ func (c *Client) CreateNetworkInterface(ctx context.Context, toAllocate int32, s
 	return eni.ID, eni, nil
 }
 
+// CreateNetworkInterfacePermission grants INSTANCE-ATTACH permission on the given ENI to a difft AWS account.
+// This is required before an instance in accountID can attach an ENI owned by a different account.
+func (c *Client) CreateNetworkInterfacePermission(ctx context.Context, eniID string, accountID string) error {
+	input := &ec2.CreateNetworkInterfacePermissionInput{
+		NetworkInterfaceId: aws.String(eniID),
+		AwsAccountId:       aws.String(accountID),
+		Permission:         ec2_types.InterfacePermissionTypeInstanceAttach,
+	}
+
+	// wrap this in a limiter
+	c.limiter.Limit(ctx, CreateNetworkInterfacePermission)
+	// track how long it takes
+	sinceStart := spanstat.Start()
+	_, err := c.ec2Client.CreateNetworkInterfacePermission(ctx, input)
+	c.metricsAPI.ObserveAPICall(CreateNetworkInterfacePermission, deriveStatus(err), sinceStart.Seconds())
+	return err
+}
+
 // DeleteNetworkInterface deletes an ENI with the specified ID
 func (c *Client) DeleteNetworkInterface(ctx context.Context, eniID string) error {
 	input := &ec2.DeleteNetworkInterfaceInput{

pkg/aws/ec2/mock/mock.go

@@ -798,3 +798,7 @@ func (e *API) GetSecurityGroups(ctx context.Context) (types.SecurityGroupMap, er
 func (e *API) GetInstanceTypes(ctx context.Context) ([]ec2_types.InstanceTypeInfo, error) {
 	return e.instanceTypes, nil
 }
+
+func (e *API) CreateNetworkInterfacePermission(_ context.Context, _ string, _ string) error {
+	return nil
+}

pkg/aws/eni/crossaccount.go

@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package eni
+
+import (
+	"context"
+	"log/slog"
+
+	ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+	eniTypes "github.com/cilium/cilium/pkg/aws/eni/types"
+	"github.com/cilium/cilium/pkg/aws/types"
+	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
+	"github.com/cilium/cilium/pkg/logging/logfields"
+)
+
+// CrossAccountEC2Client splits EC2 API calls between two accounts:
+//   - remote: ownwer of the VPC and pod subnets. This handles all ENI lifecycle operations
+//   - local: owner of the EC2 instances. This handles all instance-level operations including attachments
+//
+// After CreateNetworkInterface succeeds on the remote client, a
+// CreateNetworkInterfacePermission call grants the local account INSTANCE-ATTACH
+// access so that AttachNetworkInterface can be called from the local account.
+type CrossAccountEC2Client struct {
+	logger         *slog.Logger
+	local          EC2API
+	remote         EC2API
+	localAccountID string
+}
+
+// NewCrossAccountEC2Client constructs a CrossAccountEC2Client.
+// the localAccountID is needed to setup every CreateNetworkInterfacePermission
+func NewCrossAccountEC2Client(logger *slog.Logger, local, remote EC2API, localAccountID string) *CrossAccountEC2Client {
+	return &CrossAccountEC2Client{
+		logger:         logger,
+		local:          local,
+		remote:         remote,
+		localAccountID: localAccountID,
+	}
+}
+
+// *********************************************************************************
+// --- VPC / Subnet / ENI-owner accouint operations → remote (network account) ---
+// *********************************************************************************
+
+func (c *CrossAccountEC2Client) GetSubnets(ctx context.Context) (ipamTypes.SubnetMap, error) {
+	return c.remote.GetSubnets(ctx)
+}
+
+func (c *CrossAccountEC2Client) GetVpcs(ctx context.Context) (ipamTypes.VirtualNetworkMap, error) {
+	return c.remote.GetVpcs(ctx)
+}
+
+func (c *CrossAccountEC2Client) GetRouteTables(ctx context.Context) (ipamTypes.RouteTableMap, error) {
+	return c.remote.GetRouteTables(ctx)
+}
+
+// Security groups must come from the remote (VPC-owning) account because
+// CreateNetworkInterface executes there and cross-account SG references are rejected by AWS.
+func (c *CrossAccountEC2Client) GetSecurityGroups(ctx context.Context) (types.SecurityGroupMap, error) {
+	return c.remote.GetSecurityGroups(ctx)
+}
+
+func (c *CrossAccountEC2Client) GetDetachedNetworkInterfaces(ctx context.Context, tags ipamTypes.Tags, maxResults int32) ([]string, error) {
+	return c.remote.GetDetachedNetworkInterfaces(ctx, tags, maxResults)
+}
+
+// CreateNetworkInterface creates the ENI in the remote account's subnet, then
+// immediately grants the local account INSTANCE-ATTACH permission so that
+// AttachNetworkInterface (local) can succeed.
+func (c *CrossAccountEC2Client) CreateNetworkInterface(ctx context.Context, toAllocate int32, subnetID, desc string, groups []string, allocatePrefixes bool) (string, *eniTypes.ENI, error) {
+	eniID, eni, err := c.remote.CreateNetworkInterface(ctx, toAllocate, subnetID, desc, groups, allocatePrefixes)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if permErr := c.remote.CreateNetworkInterfacePermission(ctx, eniID, c.localAccountID); permErr != nil {
+		// Permission grant call failed. Delete the orphaned eni and rethrow
+		c.logger.Warn(
+			"Failed to grant cross-account ENI attach permission. Deleting orphaned ENI",
+			logfields.ENI, eniID,
+			logfields.Error, permErr,
+		)
+		if delErr := c.remote.DeleteNetworkInterface(ctx, eniID); delErr != nil {
+			//TODO: maybe make a bigger deal of this
+			c.logger.Warn("Failed to delete orphaned ENI",
+				logfields.ENI, eniID,
+				logfields.Error, delErr,
+			)
+		}
+		return "", nil, permErr
+	}
+
+	return eniID, eni, nil
+}
+
+func (c *CrossAccountEC2Client) CreateNetworkInterfacePermission(ctx context.Context, eniID string, accountID string) error {
+	return c.remote.CreateNetworkInterfacePermission(ctx, eniID, accountID)
+}
+
+func (c *CrossAccountEC2Client) DeleteNetworkInterface(ctx context.Context, eniID string) error {
+	return c.remote.DeleteNetworkInterface(ctx, eniID)
+}
+
+func (c *CrossAccountEC2Client) AssignPrivateIpAddresses(ctx context.Context, eniID string, addresses int32) ([]string, error) {
+	return c.remote.AssignPrivateIpAddresses(ctx, eniID, addresses)
+}
+
+func (c *CrossAccountEC2Client) UnassignPrivateIpAddresses(ctx context.Context, eniID string, addresses []string) error {
+	return c.remote.UnassignPrivateIpAddresses(ctx, eniID, addresses)
+}
+
+func (c *CrossAccountEC2Client) AssignENIPrefixes(ctx context.Context, eniID string, prefixes int32) error {
+	return c.remote.AssignENIPrefixes(ctx, eniID, prefixes)
+}
+
+func (c *CrossAccountEC2Client) UnassignENIPrefixes(ctx context.Context, eniID string, prefixes []string) error {
+	return c.remote.UnassignENIPrefixes(ctx, eniID, prefixes)
+}
+
+// *********************************************************************************
+// --- Instance-owner operations → local ---
+// *********************************************************************************
+
+func (c *CrossAccountEC2Client) GetInstance(ctx context.Context, vpcs ipamTypes.VirtualNetworkMap, subnets ipamTypes.SubnetMap, instanceID string) (*ipamTypes.Instance, error) {
+	return c.local.GetInstance(ctx, vpcs, subnets, instanceID)
+}
+
+func (c *CrossAccountEC2Client) GetInstances(ctx context.Context, vpcs ipamTypes.VirtualNetworkMap, subnets ipamTypes.SubnetMap) (*ipamTypes.InstanceMap, error) {
+	return c.local.GetInstances(ctx, vpcs, subnets)
+}
+
+// Needed so we can get max limits by type
+func (c *CrossAccountEC2Client) GetInstanceTypes(ctx context.Context) ([]ec2_types.InstanceTypeInfo, error) {
+	return c.local.GetInstanceTypes(ctx)
+}
+
+func (c *CrossAccountEC2Client) AttachNetworkInterface(ctx context.Context, index int32, instanceID, eniID string) (string, error) {
+	return c.local.AttachNetworkInterface(ctx, index, instanceID, eniID)
+}
+
+func (c *CrossAccountEC2Client) ModifyNetworkInterface(ctx context.Context, eniID, attachmentID string, deleteOnTermination bool) error {
+	return c.local.ModifyNetworkInterface(ctx, eniID, attachmentID, deleteOnTermination)
+}
+
+func (c *CrossAccountEC2Client) AssociateEIP(ctx context.Context, eniID string, eipTags ipamTypes.Tags) (string, error) {
+	return c.local.AssociateEIP(ctx, eniID, eipTags)
+}