Add support for custom Bottlerocket AMIs for MNG (#8418)

Author: koooosh

pkg/apis/eksctl.io/v1alpha5/managed_nodegroup_test.go

@@ -84,7 +84,6 @@ var _ = Describe("Managed Nodegroup Validation", func() {
 					AMIFamily: "Bottlerocket",
 				},
 			},
-			errMsg: "cannot set amiFamily to Bottlerocket when using a custom AMI",
 		}),
 		Entry("Custom AMI with overrideBootstrapCommand", &nodeGroupCase{
 			ng: &ManagedNodeGroup{

pkg/apis/eksctl.io/v1alpha5/validation.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -69,8 +70,9 @@ var (
 )
 
 var (
-	SupportedAmazonLinuxImages = supportedAMIFamiliesForOS(IsAmazonLinuxImage)
-	SupportedUbuntuImages      = supportedAMIFamiliesForOS(IsUbuntuImage)
+	SupportedAmazonLinuxImages  = supportedAMIFamiliesForOS(IsAmazonLinuxImage)
+	SupportedBottlerocketImages = supportedAMIFamiliesForOS(IsBottlerocketImage)
+	SupportedUbuntuImages       = supportedAMIFamiliesForOS(IsUbuntuImage)
 )
 
 // NOTE: we don't use k8s.io/apimachinery/pkg/util/sets here to keep API package free of dependencies
@@ -1391,11 +1393,11 @@ func ValidateManagedNodeGroup(index int, ng *ManagedNodeGroup) error {
 		if ng.AMIFamily == "" {
 			return errors.New("when using a custom AMI, amiFamily needs to be explicitly set via config file or via --node-ami-family flag")
 		}
-		if !IsAmazonLinuxImage(ng.AMIFamily) && !IsUbuntuImage(ng.AMIFamily) {
+		if !IsAmazonLinuxImage(ng.AMIFamily) && !IsBottlerocketImage(ng.AMIFamily) && !IsUbuntuImage(ng.AMIFamily) {
 			return fmt.Errorf("cannot set amiFamily to %s when using a custom AMI for managed nodes, only %s are supported", ng.AMIFamily,
-				strings.Join(append(SupportedAmazonLinuxImages, SupportedUbuntuImages...), ", "))
+				strings.Join(slices.Concat(SupportedAmazonLinuxImages, SupportedBottlerocketImages, SupportedUbuntuImages), ", "))
 		}
-		if ng.OverrideBootstrapCommand == nil && ng.AMIFamily != NodeImageFamilyAmazonLinux2023 {
+		if ng.OverrideBootstrapCommand == nil && ng.AMIFamily != NodeImageFamilyAmazonLinux2023 && ng.AMIFamily != NodeImageFamilyBottlerocket {
 			return fmt.Errorf("%[1]s.overrideBootstrapCommand is required when using a custom AMI based on %s (%[1]s.ami)", path, ng.AMIFamily)
 		}
 		notSupportedWithCustomAMIErr := func(field string) error {
@@ -1618,6 +1620,16 @@ func IsAmazonLinuxImage(imageFamily string) bool {
 	}
 }
 
+func IsBottlerocketImage(imageFamily string) bool {
+	switch imageFamily {
+	case NodeImageFamilyBottlerocket:
+		return true
+
+	default:
+		return false
+	}
+}
+
 func IsUbuntuImage(imageFamily string) bool {
 	switch imageFamily {
 	case NodeImageFamilyUbuntuPro2404,

pkg/apis/eksctl.io/v1alpha5/validation_test.go

@@ -2281,9 +2281,7 @@ var _ = Describe("ClusterConfig validation", func() {
 			mng.AMIFamily = api.NodeImageFamilyBottlerocket
 			mng.OverrideBootstrapCommand = nil
 			err = api.ValidateManagedNodeGroup(0, mng)
-			errorMsg := fmt.Sprintf("cannot set amiFamily to %s when using a custom AMI for managed nodes, only %s are supported", mng.AMIFamily,
-				strings.Join(append(api.SupportedAmazonLinuxImages, api.SupportedUbuntuImages...), ", "))
-			Expect(err).To(MatchError(errorMsg))
+			Expect(err).NotTo(HaveOccurred())
 
 			mng.AMIFamily = api.NodeImageFamilyAmazonLinux2023
 			err = api.ValidateManagedNodeGroup(0, mng)

pkg/nodebootstrap/managed_bottlerocket.go

@@ -2,7 +2,6 @@ package nodebootstrap
 
 import (
 	"encoding/base64"
-	"errors"
 	"fmt"
 
 	toml "github.com/pelletier/go-toml"
@@ -47,12 +46,13 @@ func (b *ManagedBottlerocket) UserData() (string, error) {
 		settings.Set(adminContainerEnabledKey, *enableAdminContainer)
 	}
 
-	userData := settings.String()
-	if userData == "" {
-		return "", errors.New("generated unexpected empty TOML user-data from input")
+	// Generate TOML for launch in this NodeGroup.
+	data, err := bottlerocketSettingsTOML(b.clusterConfig, b.ng.NodeGroupBase, settings)
+	if err != nil {
+		return "", err
 	}
 
-	return base64.StdEncoding.EncodeToString([]byte(userData)), nil
+	return base64.StdEncoding.EncodeToString([]byte(data)), nil
 }
 
 // setDerivedSettings configures settings that are derived from top-level nodegroup config

pkg/nodebootstrap/managed_bottlerocket_test.go

@@ -5,6 +5,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	toml "github.com/pelletier/go-toml"
 
 	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
 	"github.com/weaveworks/eksctl/pkg/nodebootstrap"
@@ -15,8 +16,8 @@ var _ = Describe("Managed Bottlerocket", func() {
 	type bottlerocketEntry struct {
 		setFields func(group *api.ManagedNodeGroup)
 
-		expectedErr      string
-		expectedUserData string
+		expectedErr    string
+		verifyUserData func(tree *toml.Tree)
 	}
 
 	DescribeTable("User data", func(e bottlerocketEntry) {
@@ -47,41 +48,37 @@ var _ = Describe("Managed Bottlerocket", func() {
 		Expect(err).NotTo(HaveOccurred())
 		actual, err := base64.StdEncoding.DecodeString(userData)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(string(actual)).To(Equal(e.expectedUserData))
+
+		tree, err := toml.LoadBytes(actual)
+		Expect(err).NotTo(HaveOccurred())
+		if e.verifyUserData != nil {
+			e.verifyUserData(tree)
+		}
 	},
 		Entry("no settings", bottlerocketEntry{
-			expectedUserData: `
-[settings]
-
-  [settings.kubernetes]
-`,
+			verifyUserData: func(tree *toml.Tree) {
+				// Default kubernetes settings include api-server, cluster-certificate, and cluster-name
+				Expect(tree.HasPath([]string{"settings", "kubernetes", "api-server"})).To(BeTrue())
+				Expect(tree.HasPath([]string{"settings", "kubernetes", "cluster-certificate"})).To(BeTrue())
+				Expect(tree.HasPath([]string{"settings", "kubernetes", "cluster-name"})).To(BeTrue())
+			},
 		}),
 		Entry("maxPods set", bottlerocketEntry{
 			setFields: func(ng *api.ManagedNodeGroup) {
 				ng.MaxPodsPerNode = 44
 			},
-			expectedUserData: `
-[settings]
-
-  [settings.kubernetes]
-    max-pods = 44
-`,
+			verifyUserData: func(tree *toml.Tree) {
+				Expect(tree.GetPath([]string{"settings", "kubernetes", "max-pods"})).To(Equal(int64(44)))
+			},
 		}),
 
 		Entry("enableAdminContainer set", bottlerocketEntry{
 			setFields: func(ng *api.ManagedNodeGroup) {
 				ng.Bottlerocket.EnableAdminContainer = api.Enabled()
 			},
-			expectedUserData: `
-[settings]
-
-  [settings.host-containers]
-
-    [settings.host-containers.admin]
-      enabled = true
-
-  [settings.kubernetes]
-`,
+			verifyUserData: func(tree *toml.Tree) {
+				Expect(tree.GetPath([]string{"settings", "host-containers", "admin", "enabled"})).To(BeTrue())
+			},
 		}),
 
 		Entry("host containers enabled", bottlerocketEntry{
@@ -94,16 +91,9 @@ var _ = Describe("Managed Bottlerocket", func() {
 					},
 				}
 			},
-			expectedUserData: `
-[settings]
-
-  [settings.host-containers]
-
-    [settings.host-containers.example]
-      enabled = true
-
-  [settings.kubernetes]
-`,
+			verifyUserData: func(tree *toml.Tree) {
+				Expect(tree.GetPath([]string{"settings", "host-containers", "example", "enabled"})).To(BeTrue())
+			},
 		}),
 
 		Entry("retain user-specified admin container setting", bottlerocketEntry{
@@ -116,16 +106,9 @@ var _ = Describe("Managed Bottlerocket", func() {
 					},
 				}
 			},
-			expectedUserData: `
-[settings]
-
-  [settings.host-containers]
-
-    [settings.host-containers.admin]
-      enabled = true
-
-  [settings.kubernetes]
-`,
+			verifyUserData: func(tree *toml.Tree) {
+				Expect(tree.GetPath([]string{"settings", "host-containers", "admin", "enabled"})).To(BeTrue())
+			},
 		}),
 
 		Entry("labels and taints set", bottlerocketEntry{
@@ -141,12 +124,10 @@ var _ = Describe("Managed Bottlerocket", func() {
 					},
 				}
 			},
-
-			expectedUserData: `
-[settings]
-
-  [settings.kubernetes]
-`,
+			verifyUserData: func(tree *toml.Tree) {
+				Expect(tree.HasPath([]string{"settings", "kubernetes", "node-labels"})).To(BeFalse())
+				Expect(tree.HasPath([]string{"settings", "kubernetes", "node-taints"})).To(BeFalse())
+			},
 		}),
 
 		Entry("preserve dotted keys", bottlerocketEntry{
@@ -155,13 +136,9 @@ var _ = Describe("Managed Bottlerocket", func() {
 					"a.b.c": "value",
 				}
 			},
-
-			expectedUserData: `
-[settings]
-  "a.b.c" = "value"
-
-  [settings.kubernetes]
-`,
+			verifyUserData: func(tree *toml.Tree) {
+				Expect(tree.GetPath([]string{"settings", "a.b.c"})).To(Equal("value"))
+			},
 		}),
 
 		Entry("cluster bootstrap settings set", bottlerocketEntry{

userdocs/src/usage/custom-ami-support.md

@@ -89,7 +89,7 @@ managedNodeGroups:
 The `--node-ami-family` flag can also be used with `eksctl create nodegroup`. `eksctl` requires AMI Family to be explicitly set via config file or via `--node-ami-family` CLI flag, whenever working with a custom AMI.
 
 ???+ note
-    At the moment, EKS managed nodegroups only support the following AMI Families when working with custom AMIs: `AmazonLinux2023`, `AmazonLinux2`, `Ubuntu2004`, `Ubuntu2204` and `Ubuntu2404`
+    At the moment, EKS managed nodegroups only support the following AMI Families when working with custom AMIs: `AmazonLinux2023`, `AmazonLinux2`, `Bottlerocket`, `Ubuntu2004`, `Ubuntu2204` and `Ubuntu2404`
 
 ## Windows custom AMI support
 Only self-managed Windows nodegroups can specify a custom AMI. `amiFamily` should be set to a valid Windows AMI family.