Skip to content

Commit d0798b3

Browse files
pokearupokearu
authored
Adding support for upgrading eks hybrid nodes on EKS clusters (#8719)
* adding support for upgrading eks hybrid nodes on EKS clusters * adding logging for remotenodenetwork and remotepodnetwork both empty use case
1 parent 4b71d03 commit d0798b3

6 files changed

Lines changed: 271 additions & 3 deletions

File tree

‎pkg/actions/cluster/export_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ var (
99
DrainAllNodeGroups = drainAllNodeGroups
1010
)
1111

12+
var UpdateRemoteNetworkConfig = (*OwnedCluster).updateRemoteNetworkConfig
13+
1214
func (c *UnownedCluster) SetNewClientSet(newClientSet func() (kubernetes.Interface, error)) {
1315
c.newClientSet = newClientSet
1416
}

‎pkg/actions/cluster/owned.go‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,12 @@ package cluster
33
import (
44
"context"
55
"fmt"
6+
"strings"
67
"time"
78

9+
"github.com/aws/aws-sdk-go-v2/aws"
10+
awseks "github.com/aws/aws-sdk-go-v2/service/eks"
11+
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
812
"github.com/kris-nova/logger"
913

1014
"github.com/weaveworks/eksctl/pkg/actions/addon"
@@ -63,6 +67,11 @@ func (c *OwnedCluster) Upgrade(ctx context.Context, dryRun bool) error {
6367
return err
6468
}
6569

70+
remoteNetworkUpdateRequired, err := c.updateRemoteNetworkConfig(ctx, dryRun)
71+
if err != nil {
72+
return err
73+
}
74+
6675
stackUpdateRequired, err := c.stackManager.AppendNewClusterStackResource(ctx, false, dryRun)
6776
if err != nil {
6877
return err
@@ -72,10 +81,47 @@ func (c *OwnedCluster) Upgrade(ctx context.Context, dryRun bool) error {
7281
logger.Critical("failed checking nodegroups", err.Error())
7382
}
7483

75-
cmdutils.LogPlanModeWarning(dryRun && (stackUpdateRequired || versionUpdateRequired))
84+
cmdutils.LogPlanModeWarning(dryRun && (stackUpdateRequired || versionUpdateRequired || remoteNetworkUpdateRequired))
7685
return nil
7786
}
7887

88+
func (c *OwnedCluster) updateRemoteNetworkConfig(ctx context.Context, dryRun bool) (bool, error) {
89+
if c.cfg.RemoteNetworkConfig == nil {
90+
return false, nil
91+
}
92+
93+
rnc := c.cfg.RemoteNetworkConfig
94+
req := &ekstypes.RemoteNetworkConfigRequest{}
95+
if rnc.RemoteNodeNetworks != nil {
96+
req.RemoteNodeNetworks = make([]ekstypes.RemoteNodeNetwork, len(rnc.RemoteNodeNetworks))
97+
for i, rn := range rnc.RemoteNodeNetworks {
98+
req.RemoteNodeNetworks[i] = ekstypes.RemoteNodeNetwork{Cidrs: rn.CIDRs}
99+
}
100+
}
101+
if rnc.RemotePodNetworks != nil {
102+
req.RemotePodNetworks = make([]ekstypes.RemotePodNetwork, len(rnc.RemotePodNetworks))
103+
for i, rp := range rnc.RemotePodNetworks {
104+
req.RemotePodNetworks[i] = ekstypes.RemotePodNetwork{Cidrs: rp.CIDRs}
105+
}
106+
}
107+
108+
cmdutils.LogIntendedAction(dryRun, "update remote network config for cluster %q", c.cfg.Metadata.Name)
109+
if !dryRun {
110+
if err := c.ctl.UpdateClusterConfig(ctx, &awseks.UpdateClusterConfigInput{
111+
Name: aws.String(c.cfg.Metadata.Name),
112+
RemoteNetworkConfig: req,
113+
}); err != nil {
114+
if strings.Contains(err.Error(), "No changes detected") {
115+
logger.Info("remote network config is already up-to-date on cluster %q", c.cfg.Metadata.Name)
116+
return false, nil
117+
}
118+
return false, fmt.Errorf("updating remote network config: %w", err)
119+
}
120+
logger.Success("remote network config updated successfully for cluster %q", c.cfg.Metadata.Name)
121+
}
122+
return true, nil
123+
}
124+
79125
func (c *OwnedCluster) Delete(ctx context.Context, _, podEvictionWaitPeriod time.Duration, wait, force, disableNodegroupEviction bool, parallel int) error {
80126
clusterOperable, err := c.ctl.CanOperate(c.cfg)
81127
if err != nil {

‎pkg/actions/cluster/remote_network_config_test.go‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
package cluster_test
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
"github.com/aws/aws-sdk-go-v2/aws"
8+
awseks "github.com/aws/aws-sdk-go-v2/service/eks"
9+
ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
10+
11+
. "github.com/onsi/ginkgo/v2"
12+
. "github.com/onsi/gomega"
13+
"github.com/stretchr/testify/mock"
14+
15+
"github.com/weaveworks/eksctl/pkg/actions/cluster"
16+
"github.com/weaveworks/eksctl/pkg/actions/cluster/mocks"
17+
api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
18+
"github.com/weaveworks/eksctl/pkg/cfn/manager/fakes"
19+
"github.com/weaveworks/eksctl/pkg/eks"
20+
"github.com/weaveworks/eksctl/pkg/testutils"
21+
"github.com/weaveworks/eksctl/pkg/testutils/mockprovider"
22+
)
23+
24+
var _ = Describe("UpdateRemoteNetworkConfig", func() {
25+
var (
26+
clusterName string
27+
p *mockprovider.MockProvider
28+
cfg *api.ClusterConfig
29+
fakeStackManager *fakes.FakeStackManager
30+
ctl *eks.ClusterProvider
31+
ownedCluster *cluster.OwnedCluster
32+
)
33+
34+
BeforeEach(func() {
35+
clusterName = "test-cluster"
36+
p = mockprovider.NewMockProvider()
37+
cfg = api.NewClusterConfig()
38+
cfg.Metadata.Name = clusterName
39+
fakeStackManager = new(fakes.FakeStackManager)
40+
ctl = &eks.ClusterProvider{AWSProvider: p, Status: &eks.ProviderStatus{
41+
ClusterInfo: &eks.ClusterInfo{
42+
Cluster: testutils.NewFakeCluster(clusterName, ekstypes.ClusterStatusActive),
43+
},
44+
}}
45+
ownedCluster = cluster.NewOwnedCluster(cfg, ctl, nil, fakeStackManager, &mocks.AutoModeDeleter{})
46+
})
47+
48+
mockSuccessfulUpdate := func() {
49+
p.MockEKS().On("UpdateClusterConfig", mock.Anything, mock.Anything).
50+
Return(&awseks.UpdateClusterConfigOutput{
51+
Update: &ekstypes.Update{
52+
Id: aws.String("update-123"),
53+
Status: ekstypes.UpdateStatusSuccessful,
54+
},
55+
}, nil)
56+
p.MockEKS().On("DescribeUpdate", mock.Anything, mock.Anything, mock.Anything).
57+
Return(&awseks.DescribeUpdateOutput{
58+
Update: &ekstypes.Update{
59+
Id: aws.String("update-123"),
60+
Status: ekstypes.UpdateStatusSuccessful,
61+
},
62+
}, nil)
63+
}
64+
65+
It("skips when remoteNetworkConfig is nil", func() {
66+
cfg.RemoteNetworkConfig = nil
67+
updated, err := cluster.UpdateRemoteNetworkConfig(ownedCluster, context.Background(), false)
68+
Expect(err).NotTo(HaveOccurred())
69+
Expect(updated).To(BeFalse())
70+
p.MockEKS().AssertNotCalled(GinkgoT(), "UpdateClusterConfig", mock.Anything, mock.Anything)
71+
})
72+
73+
It("calls UpdateClusterConfig with correct input for node and pod networks", func() {
74+
cfg.RemoteNetworkConfig = &api.RemoteNetworkConfig{
75+
RemoteNodeNetworks: []*api.RemoteNetwork{
76+
{CIDRs: []string{"10.80.0.0/16", "10.81.0.0/16"}},
77+
},
78+
RemotePodNetworks: []*api.RemoteNetwork{
79+
{CIDRs: []string{"10.90.0.0/16"}},
80+
},
81+
}
82+
mockSuccessfulUpdate()
83+
84+
updated, err := cluster.UpdateRemoteNetworkConfig(ownedCluster, context.Background(), false)
85+
Expect(err).NotTo(HaveOccurred())
86+
Expect(updated).To(BeTrue())
87+
88+
calls := p.MockEKS().Calls
89+
var updateCall mock.Call
90+
for _, call := range calls {
91+
if call.Method == "UpdateClusterConfig" {
92+
updateCall = call
93+
break
94+
}
95+
}
96+
input := updateCall.Arguments[1].(*awseks.UpdateClusterConfigInput)
97+
Expect(*input.Name).To(Equal(clusterName))
98+
Expect(input.RemoteNetworkConfig.RemoteNodeNetworks).To(HaveLen(1))
99+
Expect(input.RemoteNetworkConfig.RemoteNodeNetworks[0].Cidrs).To(Equal([]string{"10.80.0.0/16", "10.81.0.0/16"}))
100+
Expect(input.RemoteNetworkConfig.RemotePodNetworks).To(HaveLen(1))
101+
Expect(input.RemoteNetworkConfig.RemotePodNetworks[0].Cidrs).To(Equal([]string{"10.90.0.0/16"}))
102+
})
103+
104+
It("passes nil for omitted networks (no defaulting)", func() {
105+
cfg.RemoteNetworkConfig = &api.RemoteNetworkConfig{
106+
RemoteNodeNetworks: []*api.RemoteNetwork{
107+
{CIDRs: []string{"10.80.0.0/16"}},
108+
},
109+
}
110+
mockSuccessfulUpdate()
111+
112+
updated, err := cluster.UpdateRemoteNetworkConfig(ownedCluster, context.Background(), false)
113+
Expect(err).NotTo(HaveOccurred())
114+
Expect(updated).To(BeTrue())
115+
116+
calls := p.MockEKS().Calls
117+
var updateCall mock.Call
118+
for _, call := range calls {
119+
if call.Method == "UpdateClusterConfig" {
120+
updateCall = call
121+
break
122+
}
123+
}
124+
input := updateCall.Arguments[1].(*awseks.UpdateClusterConfigInput)
125+
Expect(input.RemoteNetworkConfig.RemoteNodeNetworks).To(HaveLen(1))
126+
Expect(input.RemoteNetworkConfig.RemotePodNetworks).To(BeNil())
127+
})
128+
129+
It("sends empty lists for remove-all case", func() {
130+
cfg.RemoteNetworkConfig = &api.RemoteNetworkConfig{
131+
RemoteNodeNetworks: []*api.RemoteNetwork{},
132+
RemotePodNetworks: []*api.RemoteNetwork{},
133+
}
134+
mockSuccessfulUpdate()
135+
136+
updated, err := cluster.UpdateRemoteNetworkConfig(ownedCluster, context.Background(), false)
137+
Expect(err).NotTo(HaveOccurred())
138+
Expect(updated).To(BeTrue())
139+
140+
calls := p.MockEKS().Calls
141+
var updateCall mock.Call
142+
for _, call := range calls {
143+
if call.Method == "UpdateClusterConfig" {
144+
updateCall = call
145+
break
146+
}
147+
}
148+
input := updateCall.Arguments[1].(*awseks.UpdateClusterConfigInput)
149+
Expect(input.RemoteNetworkConfig.RemoteNodeNetworks).To(BeEmpty())
150+
Expect(input.RemoteNetworkConfig.RemoteNodeNetworks).NotTo(BeNil())
151+
Expect(input.RemoteNetworkConfig.RemotePodNetworks).To(BeEmpty())
152+
Expect(input.RemoteNetworkConfig.RemotePodNetworks).NotTo(BeNil())
153+
})
154+
155+
It("treats 'No changes detected' as success", func() {
156+
cfg.RemoteNetworkConfig = &api.RemoteNetworkConfig{
157+
RemoteNodeNetworks: []*api.RemoteNetwork{
158+
{CIDRs: []string{"10.80.0.0/16"}},
159+
},
160+
}
161+
p.MockEKS().On("UpdateClusterConfig", mock.Anything, mock.Anything).
162+
Return(nil, fmt.Errorf("operation error EKS: UpdateClusterConfig, https response error StatusCode: 400, InvalidParameterException: No changes detected for remoteNetworkConfig"))
163+
164+
updated, err := cluster.UpdateRemoteNetworkConfig(ownedCluster, context.Background(), false)
165+
Expect(err).NotTo(HaveOccurred())
166+
Expect(updated).To(BeFalse())
167+
})
168+
169+
It("propagates other API errors", func() {
170+
cfg.RemoteNetworkConfig = &api.RemoteNetworkConfig{
171+
RemoteNodeNetworks: []*api.RemoteNetwork{
172+
{CIDRs: []string{"10.80.0.0/16"}},
173+
},
174+
}
175+
p.MockEKS().On("UpdateClusterConfig", mock.Anything, mock.Anything).
176+
Return(nil, fmt.Errorf("operation error EKS: UpdateClusterConfig, https response error StatusCode: 400, InvalidParameterException: Only one remoteNodeNetwork is allowed"))
177+
178+
_, err := cluster.UpdateRemoteNetworkConfig(ownedCluster, context.Background(), false)
179+
Expect(err).To(MatchError(ContainSubstring("Only one remoteNodeNetwork is allowed")))
180+
})
181+
182+
It("does not call API in dry-run mode", func() {
183+
cfg.RemoteNetworkConfig = &api.RemoteNetworkConfig{
184+
RemoteNodeNetworks: []*api.RemoteNetwork{
185+
{CIDRs: []string{"10.80.0.0/16"}},
186+
},
187+
}
188+
189+
updated, err := cluster.UpdateRemoteNetworkConfig(ownedCluster, context.Background(), true)
190+
Expect(err).NotTo(HaveOccurred())
191+
Expect(updated).To(BeTrue())
192+
p.MockEKS().AssertNotCalled(GinkgoT(), "UpdateClusterConfig", mock.Anything, mock.Anything)
193+
})
194+
})

‎pkg/apis/eksctl.io/v1alpha5/validation.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,12 @@ func (c *ClusterConfig) validateRemoteNetworkingConfig() error {
105105
}
106106

107107
if len(rnc.RemoteNodeNetworks) == 0 {
108+
// Both lists being explicitly empty is valid for upgrades (removes all remote networks).
109+
// For creates, this is a no-op since HasRemoteNetworkingConfigured() gates the CFN builder.
110+
if rnc.RemotePodNetworks != nil && len(rnc.RemotePodNetworks) == 0 {
111+
logger.Warning("remoteNetworkConfig has empty remoteNodeNetworks and remotePodNetworks; this will remove all remote networks on upgrade, or be ignored on create")
112+
return nil
113+
}
108114
return setNonEmpty("remoteNetworkConfig.remoteNodeNetworks")
109115
}
110116

‎pkg/apis/eksctl.io/v1alpha5/validation_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,6 +1035,14 @@ var _ = Describe("ClusterConfig validation", func() {
10351035
},
10361036
expectedErr: "remoteNetworkConfig.remoteNodeNetworks must be set and non-empty",
10371037
}),
1038+
Entry("remoteNodeNetworks is empty with nil remotePodNetworks", remoteNetworkConfigEntry{
1039+
overrideConfig: func(cc *api.ClusterConfig) {
1040+
cc.RemoteNetworkConfig = &api.RemoteNetworkConfig{
1041+
RemoteNodeNetworks: []*api.RemoteNetwork{},
1042+
}
1043+
},
1044+
expectedErr: "remoteNetworkConfig.remoteNodeNetworks must be set and non-empty",
1045+
}),
10381046
Entry("both vpcGatewayID and pre-existing VPC are set", remoteNetworkConfigEntry{
10391047
overrideConfig: func(cc *api.ClusterConfig) {
10401048
cc.VPC.ID = "vpc-1234"
@@ -1066,6 +1074,18 @@ var _ = Describe("ClusterConfig validation", func() {
10661074
}),
10671075
)
10681076

1077+
It("should allow both remoteNodeNetworks and remotePodNetworks to be empty for updates", func() {
1078+
cfg := api.NewClusterConfig()
1079+
api.SetClusterConfigDefaults(cfg)
1080+
api.SetClusterEndpointAccessDefaults(cfg.VPC)
1081+
cfg.RemoteNetworkConfig = &api.RemoteNetworkConfig{
1082+
RemoteNodeNetworks: []*api.RemoteNetwork{},
1083+
RemotePodNetworks: []*api.RemoteNetwork{},
1084+
}
1085+
err := api.ValidateClusterConfig(cfg)
1086+
Expect(err).NotTo(HaveOccurred())
1087+
})
1088+
10691089
Describe("network config", func() {
10701090
var (
10711091
cfg *api.ClusterConfig

‎userdocs/src/usage/hybrid-nodes.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
AWS EKS introduces Hybrid Nodes, a new feature that enables you to run on-premises and edge applications on customer-managed infrastructure with the same AWS EKS clusters, features, and tools you use in the AWS Cloud. AWS EKS Hybird Nodes brings an AWS-managed Kubernetes experience to on-premises environments for customers to simplify and standardize how you run applications across on-premises, edge and cloud environments. Read more at [EKS Hybrid Nodes][eks-hybrid-nodes].
66

7-
To facilitate support for this feature, eksctl introduces a new top-level field called `remoteNetworkConfig`. Any Hybrid Nodes related configuration shall be set up via this field, as part of the config file; there are no CLI flags counterparts. Additionally, at launch, any remote network config can only be set up during cluster creation and cannot be updated afterwards. This means, you won't be able to update existing clusters to use Hybrid Nodes.
7+
To facilitate support for this feature, eksctl introduces a new top-level field called `remoteNetworkConfig`. Any Hybrid Nodes related configuration shall be set up via this field, as part of the config file; there are no CLI flags counterparts. You can enable or update Hybrid Nodes on existing clusters by adding `remoteNetworkConfig` to your config file and running `eksctl upgrade cluster`.
88

99
The `remoteNetworkConfig` section of the config file allows you to setup the two core areas when it comes to joining remote nodes to you EKS clusters: **networking** and **credentials**.
1010

@@ -96,4 +96,4 @@ Container Networking Interface (CNI): The AWS VPC CNI can’t be used with hybri
9696
- [Launch Announcement][launch-announcement]
9797

9898
[eks-hybrid-nodes]: https://docs.aws.amazon.com/eks/latest/userguide/hybrid-nodes-overview.html
99-
[launch-announcement]: https://aws.amazon.com/about-aws/whats-new/2024/12/amazon-eks-hybrid-nodes
99+
[launch-announcement]: https://aws.amazon.com/about-aws/whats-new/2024/12/amazon-eks-hybrid-nodes

0 commit comments

Comments
 (0)