diff --git a/pkg/microservice/aslan/core/common/repository/models/settings.go b/pkg/microservice/aslan/core/common/repository/models/settings.go index f8d3bacbf5..e539a02ca5 100644 --- a/pkg/microservice/aslan/core/common/repository/models/settings.go +++ b/pkg/microservice/aslan/core/common/repository/models/settings.go @@ -30,9 +30,19 @@ type SystemSetting struct { ServerURL string `bson:"server_url" json:"server_url"` WorkflowHook *WorkflowHookSettings `bson:"workflow_hook" json:"workflow_hook"` ReleasePlanHook *ReleasePlanHookSettings `bson:"release_plan_hook" json:"release_plan_hook"` + DindTLSCerts *DindTLSCerts `bson:"dind_tls_certs,omitempty" json:"-"` UpdateTime int64 `bson:"update_time" json:"update_time"` } +type DindTLSCerts struct { + CAPem string `bson:"ca_pem"` + CAKeyPem string `bson:"ca_key_pem"` + ServerCertPem string `bson:"server_cert_pem"` + ServerKeyPem string `bson:"server_key_pem"` + ClientCertPem string `bson:"client_cert_pem"` + ClientKeyPem string `bson:"client_key_pem"` +} + type Theme struct { ThemeType string `bson:"theme_type" json:"theme_type"` CustomTheme *CustomTheme `bson:"custom_theme" json:"custom_theme"` diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/settings.go b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go index 168ec8a65a..6ed8becc51 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/settings.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go @@ -59,6 +59,19 @@ func (c *SystemSettingColl) Get() (*models.SystemSetting, error) { return resp, err } +func (c *SystemSettingColl) GetByID() (*models.SystemSetting, error) { + objectID, err := primitive.ObjectIDFromHex(setting.LocalClusterID) + if err != nil { + return nil, err + } + + query := bson.M{"_id": objectID} + resp := &models.SystemSetting{} + + err = c.FindOne(context.TODO(), query).Decode(resp) + return resp, err +} + func (c *SystemSettingColl) UpdateDefaultLoginSetting(defaultLogin string) error { id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) change := bson.M{"$set": bson.M{ @@ -196,6 +209,26 @@ func (c *SystemSettingColl) UpdateServerURL(serverURL string) error { return err } +func (c *SystemSettingColl) InitDindTLSCertsIfNeeded(certs *models.DindTLSCerts) (bool, error) { + id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) + query := bson.M{ + "_id": id, + "$or": []bson.M{ + {"dind_tls_certs": bson.M{"$exists": false}}, + {"dind_tls_certs": nil}, + }, + } + change := bson.M{"$set": bson.M{ + "dind_tls_certs": certs, + "update_time": time.Now().Unix(), + }} + result, err := c.UpdateOne(context.TODO(), query, change) + if err != nil { + return false, err + } + return result.ModifiedCount > 0, nil +} + func (c *SystemSettingColl) UpdateReleasePlanHookSetting(hookSetting *models.ReleasePlanHookSettings) error { id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) query := bson.M{"_id": id} diff --git a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go new file mode 100644 index 0000000000..6f30601956 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go @@ -0,0 +1,524 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "path" + "reflect" + "sort" + "strings" + "time" + + "go.mongodb.org/mongo-driver/mongo" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/v2/pkg/config" + commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/v2/pkg/setting" + "github.com/koderover/zadig/v2/pkg/types" +) + +const dindTLSCertValidity = 100 * 365 * 24 * time.Hour + +type DindTLSSecretTemplateData struct { + CAPemBase64 string + ServerCertPemBase64 string + ServerKeyPemBase64 string + ClientCertPemBase64 string + ClientKeyPemBase64 string + CertHash string +} + +func EnsureDindTLSCerts() (*commonmodels.DindTLSCerts, error) { + settingColl := commonrepo.NewSystemSettingColl() + settings, err := settingColl.GetByID() + if err == nil { + if dindTLSCertsComplete(settings.DindTLSCerts) { + return settings.DindTLSCerts, nil + } + if settings.DindTLSCerts != nil { + return nil, fmt.Errorf("dind TLS certs are incomplete") + } + } + if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + + certs, err := generateDindTLSCerts() + if err != nil { + return nil, err + } + _, err = settingColl.InitDindTLSCertsIfNeeded(certs) + if err != nil { + return nil, err + } + + settings, err = settingColl.GetByID() + if err != nil { + return nil, err + } + if !dindTLSCertsComplete(settings.DindTLSCerts) { + return nil, fmt.Errorf("dind TLS certs are incomplete") + } + return settings.DindTLSCerts, nil +} + +func EnsureDindTLSSecret(clientset kubernetes.Interface, namespace string) (*commonmodels.DindTLSCerts, error) { + if clientset == nil { + return nil, fmt.Errorf("kubernetes client is nil") + } + + certs, err := EnsureDindTLSCerts() + if err != nil { + return nil, err + } + + existing, getErr := clientset.CoreV1().Secrets(namespace).Get(context.TODO(), types.DindTLSSecretName, metav1.GetOptions{}) + desired := BuildDindTLSSecret(namespace, certs) + if getErr != nil { + if apierrors.IsNotFound(getErr) { + _, err := clientset.CoreV1().Secrets(namespace).Create(context.TODO(), desired, metav1.CreateOptions{}) + return certs, err + } + return nil, getErr + } + + if existing.Type != desired.Type || !reflect.DeepEqual(existing.Data, desired.Data) || !reflect.DeepEqual(existing.Labels, desired.Labels) { + existing.Type = desired.Type + existing.Data = desired.Data + existing.Labels = desired.Labels + _, err := clientset.CoreV1().Secrets(namespace).Update(context.TODO(), existing, metav1.UpdateOptions{}) + if err != nil { + return nil, err + } + } + + return certs, nil +} + +func BuildDindTLSSecret(namespace string, certs *commonmodels.DindTLSCerts) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: types.DindTLSSecretName, + Namespace: namespace, + Labels: DindLabels(), + }, + Type: corev1.SecretTypeOpaque, + Data: DindTLSSecretData(certs), + } +} + +func DindTLSSecretTemplate(certs *commonmodels.DindTLSCerts) DindTLSSecretTemplateData { + data := DindTLSSecretData(certs) + return DindTLSSecretTemplateData{ + CAPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSCACertKey]), + ServerCertPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSServerCertKey]), + ServerKeyPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSServerKeyKey]), + ClientCertPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSClientCertKey]), + ClientKeyPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSClientKeyKey]), + CertHash: DindTLSCertHash(certs), + } +} + +func DindTLSSecretData(certs *commonmodels.DindTLSCerts) map[string][]byte { + if certs == nil { + return map[string][]byte{} + } + return map[string][]byte{ + types.DindTLSCACertKey: []byte(certs.CAPem), + types.DindTLSServerCertKey: []byte(certs.ServerCertPem), + types.DindTLSServerKeyKey: []byte(certs.ServerKeyPem), + types.DindTLSClientCertKey: []byte(certs.ClientCertPem), + types.DindTLSClientKeyKey: []byte(certs.ClientKeyPem), + } +} + +func DindTLSCertHash(certs *commonmodels.DindTLSCerts) string { + data := DindTLSSecretData(certs) + keys := make([]string, 0, len(data)) + for key := range data { + keys = append(keys, key) + } + sort.Strings(keys) + + h := sha256.New() + for _, key := range keys { + h.Write([]byte(key)) + h.Write([]byte{0}) + h.Write(data[key]) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func DindLabels() map[string]string { + return map[string]string{ + "app.kubernetes.io/component": "dind", + "app.kubernetes.io/name": "zadig", + } +} + +func ApplyDindTLSSettings(dindSts *appsv1.StatefulSet, certs *commonmodels.DindTLSCerts) bool { + if dindSts == nil || len(dindSts.Spec.Template.Spec.Containers) == 0 { + return false + } + + modified := false + container := &dindSts.Spec.Template.Spec.Containers[0] + + if !reflect.DeepEqual(container.Args, buildDindTLSArgs(container.Args)) { + container.Args = buildDindTLSArgs(container.Args) + modified = true + } + + ports := ensureDindTLSPort(container.Ports) + if !reflect.DeepEqual(container.Ports, ports) { + container.Ports = ports + modified = true + } + + envs := removeDockerTLSCertDirEnv(container.Env) + if !reflect.DeepEqual(container.Env, envs) { + container.Env = envs + modified = true + } + + volumeMounts := ensureDindTLSVolumeMount(container.VolumeMounts) + if !reflect.DeepEqual(container.VolumeMounts, volumeMounts) { + container.VolumeMounts = volumeMounts + modified = true + } + + volumes := ensureDindTLSVolume(dindSts.Spec.Template.Spec.Volumes) + if !reflect.DeepEqual(dindSts.Spec.Template.Spec.Volumes, volumes) { + dindSts.Spec.Template.Spec.Volumes = volumes + modified = true + } + + if dindSts.Spec.Template.Annotations == nil { + dindSts.Spec.Template.Annotations = map[string]string{} + } + certHash := DindTLSCertHash(certs) + if dindSts.Spec.Template.Annotations[types.DindTLSCertHashAnnotation] != certHash { + dindSts.Spec.Template.Annotations[types.DindTLSCertHashAnnotation] = certHash + modified = true + } + + return modified +} + +func EnsureDindServiceTLS(kclient client.Client, namespace string) error { + return ensureDindServiceTLS(kclient, namespace) +} + +func ensureDindServiceTLS(kclient client.Client, namespace string) error { + service := &corev1.Service{} + if err := kclient.Get(context.TODO(), client.ObjectKey{Name: types.DindStatefulSetName, Namespace: namespace}, service); err != nil { + if apierrors.IsNotFound(err) { + return kclient.Create(context.TODO(), BuildDindService(namespace)) + } + return err + } + + ports := ensureDindTLSServicePort(service.Spec.Ports) + if reflect.DeepEqual(service.Spec.Ports, ports) { + return nil + } + service.Spec.Ports = ports + return kclient.Update(context.TODO(), service) +} + +func BuildDindService(namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: types.DindStatefulSetName, + Namespace: namespace, + Labels: DindLabels(), + }, + Spec: corev1.ServiceSpec{ + Ports: ensureDindTLSServicePort(nil), + ClusterIP: "None", + Selector: DindLabels(), + }, + } +} + +func buildDindTLSArgs(currentArgs []string) []string { + args := []string{ + "--host=unix:///var/run/docker.sock", + fmt.Sprintf("--host=tcp://0.0.0.0:%d", types.DindTLSPort), + "--tlsverify", + fmt.Sprintf("--tlscacert=%s", path.Join(types.DindTLSServerMountPath, types.DindTLSCACertKey)), + fmt.Sprintf("--tlscert=%s", path.Join(types.DindTLSServerMountPath, types.DindTLSServerCertKey)), + fmt.Sprintf("--tlskey=%s", path.Join(types.DindTLSServerMountPath, types.DindTLSServerKeyKey)), + } + + for _, arg := range currentArgs { + if isDindConnectionArg(arg) { + continue + } + args = append(args, arg) + } + + return args +} + +func isDindConnectionArg(arg string) bool { + return strings.HasPrefix(arg, "--host=") || + arg == "--tlsverify" || + strings.HasPrefix(arg, "--tlscacert") || + strings.HasPrefix(arg, "--tlscert") || + strings.HasPrefix(arg, "--tlskey") +} + +func ensureDindTLSPort(ports []corev1.ContainerPort) []corev1.ContainerPort { + resp := make([]corev1.ContainerPort, 0, len(ports)+1) + for _, port := range ports { + if port.ContainerPort == 2375 || port.ContainerPort == types.DindTLSPort { + continue + } + resp = append(resp, port) + } + resp = append(resp, corev1.ContainerPort{ + Protocol: corev1.ProtocolTCP, + ContainerPort: types.DindTLSPort, + }) + return resp +} + +func ensureDindTLSServicePort(ports []corev1.ServicePort) []corev1.ServicePort { + resp := make([]corev1.ServicePort, 0, len(ports)+1) + for _, port := range ports { + if port.Port == 2375 || port.Port == types.DindTLSPort || port.Name == types.DindContainerName || port.Name == types.DindTLSServicePortName { + continue + } + resp = append(resp, port) + } + resp = append(resp, corev1.ServicePort{ + Name: types.DindTLSServicePortName, + Protocol: corev1.ProtocolTCP, + Port: types.DindTLSPort, + TargetPort: intstr.FromInt(types.DindTLSPort), + }) + return resp +} + +func removeDockerTLSCertDirEnv(envs []corev1.EnvVar) []corev1.EnvVar { + resp := make([]corev1.EnvVar, 0, len(envs)) + for _, env := range envs { + if env.Name == "DOCKER_TLS_CERTDIR" { + continue + } + resp = append(resp, env) + } + return resp +} + +func ensureDindTLSVolumeMount(volumeMounts []corev1.VolumeMount) []corev1.VolumeMount { + resp := make([]corev1.VolumeMount, 0, len(volumeMounts)+1) + for _, volumeMount := range volumeMounts { + if volumeMount.Name == types.DindTLSVolumeName { + continue + } + resp = append(resp, volumeMount) + } + resp = append(resp, corev1.VolumeMount{ + Name: types.DindTLSVolumeName, + MountPath: types.DindTLSServerMountPath, + ReadOnly: true, + }) + return resp +} + +func ensureDindTLSVolume(volumes []corev1.Volume) []corev1.Volume { + resp := make([]corev1.Volume, 0, len(volumes)+1) + for _, volume := range volumes { + if volume.Name == types.DindTLSVolumeName { + continue + } + resp = append(resp, volume) + } + resp = append(resp, corev1.Volume{ + Name: types.DindTLSVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: types.DindTLSSecretName, + Items: []corev1.KeyToPath{ + { + Key: types.DindTLSCACertKey, + Path: types.DindTLSCACertKey, + }, + { + Key: types.DindTLSServerCertKey, + Path: types.DindTLSServerCertKey, + }, + { + Key: types.DindTLSServerKeyKey, + Path: types.DindTLSServerKeyKey, + }, + }, + }, + }, + }) + return resp +} + +func dindTLSCertsComplete(certs *commonmodels.DindTLSCerts) bool { + return certs != nil && + certs.CAPem != "" && + certs.CAKeyPem != "" && + certs.ServerCertPem != "" && + certs.ServerKeyPem != "" && + certs.ClientCertPem != "" && + certs.ClientKeyPem != "" +} + +func generateDindTLSCerts() (*commonmodels.DindTLSCerts, error) { + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + caTemplate := certificateTemplate("zadig-dind-ca", true, nil) + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, err + } + + serverKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + serverTemplate := certificateTemplate("zadig-dind-server", false, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}) + serverTemplate.DNSNames = dindTLSServerNames() + serverTemplate.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caTemplate, &serverKey.PublicKey, caKey) + if err != nil { + return nil, err + } + + clientKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + clientTemplate := certificateTemplate("zadig-dind-client", false, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) + clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey) + if err != nil { + return nil, err + } + + return &commonmodels.DindTLSCerts{ + CAPem: string(encodeCert(caDER)), + CAKeyPem: string(encodePrivateKey(caKey)), + ServerCertPem: string(encodeCert(serverDER)), + ServerKeyPem: string(encodePrivateKey(serverKey)), + ClientCertPem: string(encodeCert(clientDER)), + ClientKeyPem: string(encodePrivateKey(clientKey)), + }, nil +} + +func certificateTemplate(commonName string, isCA bool, extKeyUsage []x509.ExtKeyUsage) *x509.Certificate { + now := time.Now() + keyUsage := x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment + if isCA { + keyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign + } + return &x509.Certificate{ + SerialNumber: randomSerialNumber(), + Subject: pkix.Name{CommonName: commonName}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(dindTLSCertValidity), + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsage, + BasicConstraintsValid: true, + IsCA: isCA, + } +} + +func randomSerialNumber() *big.Int { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return big.NewInt(time.Now().UnixNano()) + } + return serialNumber +} + +func dindTLSServerNames() []string { + names := []string{ + "dind", + "*.dind", + "localhost", + } + + for _, namespace := range []string{config.Namespace(), setting.AttachedClusterNamespace} { + namespace = strings.TrimSpace(namespace) + if namespace == "" { + continue + } + names = append(names, + fmt.Sprintf("dind.%s", namespace), + fmt.Sprintf("dind.%s.svc", namespace), + fmt.Sprintf("dind.%s.svc.cluster.local", namespace), + fmt.Sprintf("*.dind.%s", namespace), + fmt.Sprintf("*.dind.%s.svc", namespace), + fmt.Sprintf("*.dind.%s.svc.cluster.local", namespace), + ) + } + + return dedupeStrings(names) +} + +func dedupeStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + ret := make([]string, 0, len(values)) + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + ret = append(ret, value) + } + return ret +} + +func encodeCert(cert []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert}) +} + +func encodePrivateKey(key *rsa.PrivateKey) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) +} diff --git a/pkg/microservice/aslan/core/common/service/kube/service.go b/pkg/microservice/aslan/core/common/service/kube/service.go index f377bdd087..78d072064a 100644 --- a/pkg/microservice/aslan/core/common/service/kube/service.go +++ b/pkg/microservice/aslan/core/common/service/kube/service.go @@ -306,6 +306,11 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment } dindReplicas, dindLimitsCPU, dindLimitsMemory, dindEnablePV, dindSCName, dindStorageSizeInGiB, dindStorageDriver := getDindCfg(cluster) + dindTLSCerts, err := EnsureDindTLSCerts() + if err != nil { + return nil, fmt.Errorf("failed to ensure dind TLS certs: %w", err) + } + dindTLS := DindTLSSecretTemplate(dindTLSCerts) yaml := agentYaml if cluster.AdvancedConfig != nil { @@ -362,6 +367,7 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment DindStorageClassName: dindSCName, DindStorageSizeInGiB: dindStorageSizeInGiB, DindStorageDriver: dindStorageDriver, + DindTLS: dindTLS, ScheduleWorkflow: scheduleWorkflow, EnableIRSA: cluster.AdvancedConfig.EnableIRSA, IRSARoleARN: cluster.AdvancedConfig.IRSARoleARM, @@ -388,6 +394,7 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment DindStorageClassName: dindSCName, DindStorageSizeInGiB: dindStorageSizeInGiB, DindStorageDriver: dindStorageDriver, + DindTLS: dindTLS, EnableIRSA: cluster.AdvancedConfig.EnableIRSA, NodeSelector: cluster.AdvancedConfig.AgentNodeSelector, Toleration: cluster.AdvancedConfig.AgentToleration, @@ -452,6 +459,13 @@ func getDindCfg(cluster *models.K8SCluster) (replicas int, limitsCPU, limitsMemo return } +func ResolveDindNamespace(cluster *models.K8SCluster) string { + if cluster != nil && cluster.Local { + return config.Namespace() + } + return setting.AttachedClusterNamespace +} + // InitializeExternalCluster initialized the resources in the cluster for zadig to run correctly. // if the cluster is of type kubeconfig, we need to create following resource: // Namespace: koderover-agent @@ -516,6 +530,11 @@ func InitializeExternalCluster(clusterID string) error { return errors.Errorf("cluster %s create serviceAccount err: %s", clusterID, err) } + dindTLSCerts, err := EnsureDindTLSSecret(clientset, namespace) + if err != nil { + return fmt.Errorf("failed to ensure dind TLS secret in cluster %q: %s", clusterID, err) + } + // create role binding roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ @@ -540,10 +559,7 @@ func InitializeExternalCluster(clusterID string) error { } log.Infof("cluster %s create role binding successfully", clusterID) - dindLabelMap := map[string]string{ - "app.kubernetes.io/component": "dind", - "app.kubernetes.io/name": "zadig", - } + dindLabelMap := DindLabels() privileged := true @@ -561,6 +577,9 @@ func InitializeExternalCluster(clusterID string) error { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: dindLabelMap, + Annotations: map[string]string{ + types.DindTLSCertHashAnnotation: DindTLSCertHash(dindTLSCerts), + }, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ @@ -577,20 +596,16 @@ func InitializeExternalCluster(clusterID string) error { }, Containers: []corev1.Container{ corev1.Container{ - Name: "dind", + Name: "dind-tls", Image: config.DindImage(), + Args: buildDindTLSArgs(nil), Ports: []corev1.ContainerPort{ { Protocol: "TCP", - ContainerPort: 2375, - }, - }, - Env: []corev1.EnvVar{ - { - Name: "DOCKER_TLS_CERTDIR", - Value: "", + ContainerPort: types.DindTLSPort, }, }, + VolumeMounts: ensureDindTLSVolumeMount(nil), Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(strconv.Itoa(2)), @@ -606,6 +621,7 @@ func InitializeExternalCluster(clusterID string) error { }, }, }, + Volumes: ensureDindTLSVolume(nil), }, }, }, @@ -625,10 +641,10 @@ func InitializeExternalCluster(clusterID string) error { Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { - Name: "dind", + Name: "dind-tls", Protocol: "TCP", - Port: 2375, - TargetPort: intstr.FromInt(2375), + Port: types.DindTLSPort, + TargetPort: intstr.FromInt(types.DindTLSPort), }, }, ClusterIP: "None", @@ -691,6 +707,7 @@ type TemplateSchema struct { DindStorageClassName string DindStorageSizeInGiB int DindStorageDriver string + DindTLS DindTLSSecretTemplateData ScheduleWorkflow bool EnableIRSA bool IRSARoleARN string @@ -1045,6 +1062,24 @@ spec: --- +apiVersion: v1 +kind: Secret +metadata: + name: dind-tls-certs + namespace: {{.Namespace}} + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +type: Opaque +data: + ca.pem: {{.DindTLS.CAPemBase64}} + server-cert.pem: {{.DindTLS.ServerCertPemBase64}} + server-key.pem: {{.DindTLS.ServerKeyPemBase64}} + cert.pem: {{.DindTLS.ClientCertPemBase64}} + key.pem: {{.DindTLS.ClientKeyPemBase64}} + +--- + apiVersion: apps/v1 kind: StatefulSet metadata: @@ -1065,6 +1100,8 @@ spec: labels: app.kubernetes.io/component: dind app.kubernetes.io/name: zadig + annotations: + zadig.koderover.com/dind-tls-cert-hash: {{.DindTLS.CertHash}} spec: affinity: podAntiAffinity: @@ -1072,21 +1109,35 @@ spec: - weight: 100 podAffinityTerm: topologyKey: kubernetes.io/hostname + volumes: + - name: dind-tls-certs + secret: + secretName: dind-tls-certs + items: + - key: ca.pem + path: ca.pem + - key: server-cert.pem + path: server-cert.pem + - key: server-key.pem + path: server-key.pem containers: - name: dind image: {{.DindImage}} - {{- if .DindStorageDriver }} args: + - --host=unix:///var/run/docker.sock + - --host=tcp://0.0.0.0:2376 + - --tlsverify + - --tlscacert=/etc/zadig/dind/tls/ca.pem + - --tlscert=/etc/zadig/dind/tls/server-cert.pem + - --tlskey=/etc/zadig/dind/tls/server-key.pem + {{ if .DindStorageDriver }} - --storage-driver={{.DindStorageDriver}} - {{- end }} - env: - - name: DOCKER_TLS_CERTDIR - value: "" + {{ end }} securityContext: privileged: true ports: - protocol: TCP - containerPort: 2375 + containerPort: 2376 resources: limits: cpu: "4" @@ -1094,10 +1145,15 @@ spec: requests: cpu: 100m memory: 128Mi -{{- if .DindEnablePV }} volumeMounts: + - name: dind-tls-certs + mountPath: /etc/zadig/dind/tls + readOnly: true +{{ if .DindEnablePV }} - name: zadig-docker mountPath: /var/lib/docker +{{ end }} +{{ if .DindEnablePV }} volumeClaimTemplates: - metadata: name: zadig-docker @@ -1107,7 +1163,7 @@ spec: resources: requests: storage: {{.DindStorageSizeInGiB}}Gi -{{- end }} +{{ end }} --- @@ -1121,10 +1177,10 @@ metadata: app.kubernetes.io/name: zadig spec: ports: - - name: dind + - name: dind-tls protocol: TCP - port: 2375 - targetPort: 2375 + port: 2376 + targetPort: 2376 clusterIP: None selector: app.kubernetes.io/component: dind @@ -1190,6 +1246,24 @@ roleRef: --- +apiVersion: v1 +kind: Secret +metadata: + name: dind-tls-certs + namespace: koderover-agent + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +type: Opaque +data: + ca.pem: {{.DindTLS.CAPemBase64}} + server-cert.pem: {{.DindTLS.ServerCertPemBase64}} + server-key.pem: {{.DindTLS.ServerKeyPemBase64}} + cert.pem: {{.DindTLS.ClientCertPemBase64}} + key.pem: {{.DindTLS.ClientKeyPemBase64}} + +--- + apiVersion: apps/v1 kind: StatefulSet metadata: @@ -1210,6 +1284,8 @@ spec: labels: app.kubernetes.io/component: dind app.kubernetes.io/name: zadig + annotations: + zadig.koderover.com/dind-tls-cert-hash: {{.DindTLS.CertHash}} spec: affinity: podAntiAffinity: @@ -1217,21 +1293,35 @@ spec: - weight: 100 podAffinityTerm: topologyKey: kubernetes.io/hostname + volumes: + - name: dind-tls-certs + secret: + secretName: dind-tls-certs + items: + - key: ca.pem + path: ca.pem + - key: server-cert.pem + path: server-cert.pem + - key: server-key.pem + path: server-key.pem containers: - name: dind image: {{.DindImage}} - {{- if .DindStorageDriver }} args: + - --host=unix:///var/run/docker.sock + - --host=tcp://0.0.0.0:2376 + - --tlsverify + - --tlscacert=/etc/zadig/dind/tls/ca.pem + - --tlscert=/etc/zadig/dind/tls/server-cert.pem + - --tlskey=/etc/zadig/dind/tls/server-key.pem + {{ if .DindStorageDriver }} - --storage-driver={{.DindStorageDriver}} - {{- end }} - env: - - name: DOCKER_TLS_CERTDIR - value: "" + {{ end }} securityContext: privileged: true ports: - protocol: TCP - containerPort: 2375 + containerPort: 2376 resources: limits: cpu: {{.DindLimitsCPU}} @@ -1239,10 +1329,15 @@ spec: requests: cpu: 100m memory: 128Mi -{{- if .DindEnablePV }} volumeMounts: + - name: dind-tls-certs + mountPath: /etc/zadig/dind/tls + readOnly: true +{{ if .DindEnablePV }} - name: zadig-docker mountPath: /var/lib/docker +{{ end }} +{{ if .DindEnablePV }} volumeClaimTemplates: - metadata: name: zadig-docker @@ -1252,7 +1347,7 @@ spec: resources: requests: storage: {{.DindStorageSizeInGiB}}Gi -{{- end }} +{{ end }} --- @@ -1266,10 +1361,10 @@ metadata: app.kubernetes.io/name: zadig spec: ports: - - name: dind + - name: dind-tls protocol: TCP - port: 2375 - targetPort: 2375 + port: 2376 + targetPort: 2376 clusterIP: None selector: app.kubernetes.io/component: dind diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go index c18fd03d27..4a28a20a46 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go @@ -750,6 +750,14 @@ func getEnvs(configMapMountDir string, jobTaskSpec *commonmodels.JobTaskFreestyl Name: setting.DockerHost, Value: jobTaskSpec.Properties.DockerHost, }) + ret = append(ret, corev1.EnvVar{ + Name: setting.DockerTLSVerify, + Value: "1", + }) + ret = append(ret, corev1.EnvVar{ + Name: setting.DockerCertPath, + Value: types.DindTLSClientMountPath, + }) } ret = append(ret, corev1.EnvVar{ Name: setting.ENVLogLevel, @@ -783,6 +791,12 @@ func getVolumeMounts(configMapMountDir string, userHostDockerDaemon bool) []core Name: "docker-sock", MountPath: setting.DefaultDockSock, }) + } else { + resp = append(resp, corev1.VolumeMount{ + Name: types.DindTLSVolumeName, + MountPath: types.DindTLSClientMountPath, + ReadOnly: true, + }) } return resp } @@ -828,6 +842,29 @@ func getVolumes(jobName string, userHostDockerDaemon bool) []corev1.Volume { }, }, }) + } else { + resp = append(resp, corev1.Volume{ + Name: types.DindTLSVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: types.DindTLSSecretName, + Items: []corev1.KeyToPath{ + { + Key: types.DindTLSCACertKey, + Path: types.DindTLSCACertKey, + }, + { + Key: types.DindTLSClientCertKey, + Path: types.DindTLSClientCertKey, + }, + { + Key: types.DindTLSClientKeyKey, + Path: types.DindTLSClientKeyKey, + }, + }, + }, + }, + }) } return resp diff --git a/pkg/microservice/aslan/core/multicluster/service/clusters.go b/pkg/microservice/aslan/core/multicluster/service/clusters.go index 6300d8cd2f..4a1af5ef78 100644 --- a/pkg/microservice/aslan/core/multicluster/service/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/service/clusters.go @@ -694,7 +694,7 @@ func UpdateClusterDind(ctx *handler.Context, id string, dindCfg *commonmodels.Di err := commonutil.CheckZadigProfessionalLicense() if err != nil { - if dindCfg.Replicas != 1 { + if dindCfg.Replicas != 0 && dindCfg.Replicas != 1 { return nil, e.ErrLicenseInvalid.AddDesc("") } if dindCfg.Resources != nil { @@ -956,8 +956,27 @@ func UpgradeAgent(id string, logger *zap.SugaredLogger) error { return errors.Errorf("cluster %s update serviceAccount err: %s", id, err) } + if _, err := kube.EnsureDindTLSSecret(clientset, config.Namespace()); err != nil { + return fmt.Errorf("failed to ensure dind TLS secret in local cluster: %s", err) + } + + if err := kube.EnsureDindServiceTLS(kubeClient, config.Namespace()); err != nil { + return fmt.Errorf("failed to ensure dind TLS service in local cluster: %s", err) + } + return UpgradeDind(kubeClient, clusterInfo, config.Namespace()) } else { + clientset, err := clientmanager.NewKubeClientManager().GetKubernetesClientSet(id) + if err != nil { + return err + } + dindNamespace := kube.ResolveDindNamespace(clusterInfo) + if _, err := kube.EnsureDindTLSSecret(clientset, dindNamespace); err != nil { + err = fmt.Errorf("failed to ensure dind TLS secret in cluster %s before applying yaml: %s", clusterInfo.Name, err) + logger.Error(err) + return err + } + // Upgrade attached cluster. yamls, err := s.GetYaml(id, config.HubAgentImage(), serverURL, "/api/hub", true, logger) if err != nil { @@ -993,6 +1012,8 @@ func UpgradeAgent(id string, logger *zap.SugaredLogger) error { } else { err = UpgradeDind(kubeClient, clusterInfo, setting.AttachedClusterNamespace) } + } else if u.GetKind() == "Service" && u.GetName() == types.DindStatefulSetName { + err = kube.EnsureDindServiceTLS(kubeClient, dindNamespace) } else { err = updater.CreateOrPatchUnstructured(u, kubeClient) } @@ -1080,7 +1101,7 @@ func setClusterDind(cluster *commonmodels.K8SCluster) error { return nil } - if cluster.DindCfg.Replicas <= 0 { + if cluster.DindCfg.Replicas < 0 { cluster.DindCfg.Replicas = kube.DefaultDindReplicas } if cluster.DindCfg.Resources == nil { @@ -1188,9 +1209,13 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str } ctx := context.TODO() + dindTLSCerts, err := kube.EnsureDindTLSCerts() + if err != nil { + return err + } // Retry logic for handling concurrent modifications - err := retryOnConflict(func() error { + err = retryOnConflict(func() error { dindSts := &appsv1.StatefulSet{} err := kclient.Get(ctx, client.ObjectKey{ Name: types.DindStatefulSetName, @@ -1206,13 +1231,17 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str return fmt.Errorf("failed to deep copy original dind statefulset, error: %s", err) } - return applyDindUpgrade(kclient, ctx, dindSts, originalSts, cluster, ns) + return applyDindUpgrade(kclient, ctx, dindSts, originalSts, cluster, ns, dindTLSCerts) }) if err != nil { return err } + if err := kube.EnsureDindServiceTLS(kclient, ns); err != nil { + return err + } + // Sync registry configuration after successful update err = commonutil.SyncDinDForRegistries() if err != nil { @@ -1222,7 +1251,7 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str return nil } -func applyDindUpgrade(kclient client.Client, ctx context.Context, dindSts, originalSts *appsv1.StatefulSet, cluster *commonmodels.K8SCluster, ns string) error { +func applyDindUpgrade(kclient client.Client, ctx context.Context, dindSts, originalSts *appsv1.StatefulSet, cluster *commonmodels.K8SCluster, ns string, dindTLSCerts *commonmodels.DindTLSCerts) error { var err error dindSts.Spec.Replicas = util.GetInt32Pointer(int32(cluster.DindCfg.Replicas)) @@ -1365,6 +1394,7 @@ func applyDindUpgrade(kclient client.Client, ctx context.Context, dindSts, origi dindSts.Spec.Template.Spec.Containers[0].Args = finalArgs } + kube.ApplyDindTLSSettings(dindSts, dindTLSCerts) if stsHasImmutableFieldChanged(originalSts, dindSts) { log.Infof("dind has immutable field changed, recreating dind.") diff --git a/pkg/setting/consts.go b/pkg/setting/consts.go index 7c5c104940..d615379629 100644 --- a/pkg/setting/consts.go +++ b/pkg/setting/consts.go @@ -96,6 +96,8 @@ const ( DockerAuthDir = "DOCKER_AUTH_DIR" Path = "PATH" DockerHost = "DOCKER_HOST" + DockerTLSVerify = "DOCKER_TLS_VERIFY" + DockerCertPath = "DOCKER_CERT_PATH" BuildURL = "BUILD_URL" DefaultDockSock = "/var/run/docker.sock" diff --git a/pkg/tool/dockerhost/docker_host.go b/pkg/tool/dockerhost/docker_host.go index b78fe3f282..267db70d58 100644 --- a/pkg/tool/dockerhost/docker_host.go +++ b/pkg/tool/dockerhost/docker_host.go @@ -37,6 +37,7 @@ import ( "github.com/koderover/zadig/v2/pkg/tool/cache" "github.com/koderover/zadig/v2/pkg/tool/clientmanager" "github.com/koderover/zadig/v2/pkg/tool/log" + "github.com/koderover/zadig/v2/pkg/types" ) var ( @@ -151,10 +152,7 @@ func (d *dockerhosts) initClusterInfo(clusterID ClusterID) { } func (d *dockerhosts) getDockerHostsSvc(clusterID ClusterID) []consistent.Member { - ns := config.Namespace() - if string(clusterID) != setting.LocalClusterID { - ns = setting.AttachedClusterNamespace - } + ns := d.getDindNamespace(clusterID) kclient, err := clientmanager.NewKubeClientManager().GetControllerRuntimeClient(string(clusterID)) if err != nil { @@ -180,12 +178,19 @@ func (d *dockerhosts) getDockerHostsSvc(clusterID ClusterID) []consistent.Member return members } +func (d *dockerhosts) getDindNamespace(clusterID ClusterID) string { + if clusterID == "" || string(clusterID) == setting.LocalClusterID { + return config.Namespace() + } + return setting.AttachedClusterNamespace +} + func (d *dockerhosts) getDefaultDockerHosts() []consistent.Member { return []consistent.Member{d.genDindAddr(0)} } func (d *dockerhosts) genDindAddr(idx int) Member { - return Member(fmt.Sprintf("tcp://dind-%d.dind:2375", idx)) + return Member(fmt.Sprintf("tcp://dind-%d.dind:%d", idx, types.DindTLSPort)) } func (d *dockerhosts) Sync() { diff --git a/pkg/types/workload.go b/pkg/types/workload.go index c786f5dfe5..0ddefc7389 100644 --- a/pkg/types/workload.go +++ b/pkg/types/workload.go @@ -36,6 +36,18 @@ const DindStatefulSetName = "dind" const DindContainerName = "dind" const DindMountName = "zadig-docker" const DindMountPath = "/var/lib/docker" +const DindTLSPort = 2376 +const DindTLSServicePortName = "dind-tls" +const DindTLSSecretName = "dind-tls-certs" +const DindTLSVolumeName = "dind-tls-certs" +const DindTLSServerMountPath = "/etc/zadig/dind/tls" +const DindTLSClientMountPath = "/etc/zadig/dind/client" +const DindTLSCertHashAnnotation = "zadig.koderover.com/dind-tls-cert-hash" +const DindTLSCACertKey = "ca.pem" +const DindTLSServerCertKey = "server-cert.pem" +const DindTLSServerKeyKey = "server-key.pem" +const DindTLSClientCertKey = "cert.pem" +const DindTLSClientKeyKey = "key.pem" type KubeResourceKind struct { APIVersion string `yaml:"apiVersion"`