From 3dedfdd16017aff5c7a94888e6c69b1708b8b6de Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Fri, 29 May 2026 16:42:55 +0800 Subject: [PATCH 1/4] dind support tls Signed-off-by: Patrick Zhao --- .../core/common/repository/models/settings.go | 10 + .../common/repository/mongodb/settings.go | 35 ++ .../core/common/service/kube/dind_tls.go | 518 ++++++++++++++++++ .../aslan/core/common/service/kube/service.go | 178 ++++-- .../jobcontroller/kubernetes.go | 37 ++ .../core/multicluster/service/clusters.go | 33 +- pkg/setting/consts.go | 2 + pkg/tool/dockerhost/docker_host.go | 15 +- pkg/types/workload.go | 11 + 9 files changed, 788 insertions(+), 51 deletions(-) create mode 100644 pkg/microservice/aslan/core/common/service/kube/dind_tls.go diff --git a/pkg/microservice/aslan/core/common/repository/models/settings.go b/pkg/microservice/aslan/core/common/repository/models/settings.go index f8d3bacbf5..e539a02ca5 100644 --- a/pkg/microservice/aslan/core/common/repository/models/settings.go +++ b/pkg/microservice/aslan/core/common/repository/models/settings.go @@ -30,9 +30,19 @@ type SystemSetting struct { ServerURL string `bson:"server_url" json:"server_url"` WorkflowHook *WorkflowHookSettings `bson:"workflow_hook" json:"workflow_hook"` ReleasePlanHook *ReleasePlanHookSettings `bson:"release_plan_hook" json:"release_plan_hook"` + DindTLSCerts *DindTLSCerts `bson:"dind_tls_certs,omitempty" json:"-"` UpdateTime int64 `bson:"update_time" json:"update_time"` } +type DindTLSCerts struct { + CAPem string `bson:"ca_pem"` + CAKeyPem string `bson:"ca_key_pem"` + ServerCertPem string `bson:"server_cert_pem"` + ServerKeyPem string `bson:"server_key_pem"` + ClientCertPem string `bson:"client_cert_pem"` + ClientKeyPem string `bson:"client_key_pem"` +} + type Theme struct { ThemeType string `bson:"theme_type" json:"theme_type"` CustomTheme *CustomTheme `bson:"custom_theme" json:"custom_theme"` diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/settings.go b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go index 168ec8a65a..99729e1938 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/settings.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go @@ -196,6 +196,41 @@ func (c *SystemSettingColl) UpdateServerURL(serverURL string) error { return err } +func (c *SystemSettingColl) InitDindTLSCertsIfNeeded(certs *models.DindTLSCerts) (bool, error) { + id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) + query := bson.M{ + "_id": id, + "$or": []bson.M{ + {"dind_tls_certs": bson.M{"$exists": false}}, + {"dind_tls_certs": nil}, + {"dind_tls_certs.ca_pem": bson.M{"$exists": false}}, + {"dind_tls_certs.ca_pem": ""}, + {"dind_tls_certs.ca_key_pem": bson.M{"$exists": false}}, + {"dind_tls_certs.ca_key_pem": ""}, + {"dind_tls_certs.server_cert_pem": bson.M{"$exists": false}}, + {"dind_tls_certs.server_cert_pem": ""}, + {"dind_tls_certs.server_key_pem": bson.M{"$exists": false}}, + {"dind_tls_certs.server_key_pem": ""}, + {"dind_tls_certs.client_cert_pem": bson.M{"$exists": false}}, + {"dind_tls_certs.client_cert_pem": ""}, + {"dind_tls_certs.client_key_pem": bson.M{"$exists": false}}, + {"dind_tls_certs.client_key_pem": ""}, + }, + } + change := bson.M{"$set": bson.M{ + "dind_tls_certs": certs, + "update_time": time.Now().Unix(), + }} + result, err := c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) + if err != nil { + if mongo.IsDuplicateKeyError(err) { + return false, nil + } + return false, err + } + return result.ModifiedCount > 0 || result.UpsertedCount > 0, nil +} + func (c *SystemSettingColl) UpdateReleasePlanHookSetting(hookSetting *models.ReleasePlanHookSettings) error { id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) query := bson.M{"_id": id} diff --git a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go new file mode 100644 index 0000000000..5c65ff29a7 --- /dev/null +++ b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go @@ -0,0 +1,518 @@ +/* +Copyright 2026 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "path" + "reflect" + "sort" + "strings" + "time" + + "go.mongodb.org/mongo-driver/mongo" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/v2/pkg/config" + commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" + "github.com/koderover/zadig/v2/pkg/setting" + "github.com/koderover/zadig/v2/pkg/types" +) + +const dindTLSCertValidity = 10 * 365 * 24 * time.Hour + +type DindTLSSecretTemplateData struct { + CAPemBase64 string + ServerCertPemBase64 string + ServerKeyPemBase64 string + ClientCertPemBase64 string + ClientKeyPemBase64 string + CertHash string +} + +func EnsureDindTLSCerts() (*commonmodels.DindTLSCerts, error) { + settingColl := commonrepo.NewSystemSettingColl() + settings, err := settingColl.Get() + if err == nil && dindTLSCertsComplete(settings.DindTLSCerts) { + return settings.DindTLSCerts, nil + } + if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + + certs, err := generateDindTLSCerts() + if err != nil { + return nil, err + } + created, err := settingColl.InitDindTLSCertsIfNeeded(certs) + if err != nil { + return nil, err + } + if created { + return certs, nil + } + + settings, err = settingColl.Get() + if err != nil { + return nil, err + } + if !dindTLSCertsComplete(settings.DindTLSCerts) { + return nil, fmt.Errorf("dind TLS certs are incomplete") + } + return settings.DindTLSCerts, nil +} + +func EnsureDindTLSSecret(clientset kubernetes.Interface, namespace string) (*commonmodels.DindTLSCerts, error) { + if clientset == nil { + return nil, fmt.Errorf("kubernetes client is nil") + } + + certs, err := EnsureDindTLSCerts() + if err != nil { + return nil, err + } + + existing, getErr := clientset.CoreV1().Secrets(namespace).Get(context.TODO(), types.DindTLSSecretName, metav1.GetOptions{}) + desired := BuildDindTLSSecret(namespace, certs) + if getErr != nil { + if apierrors.IsNotFound(getErr) { + _, err := clientset.CoreV1().Secrets(namespace).Create(context.TODO(), desired, metav1.CreateOptions{}) + return certs, err + } + return nil, getErr + } + + if existing.Type != desired.Type || !reflect.DeepEqual(existing.Data, desired.Data) || !reflect.DeepEqual(existing.Labels, desired.Labels) { + existing.Type = desired.Type + existing.Data = desired.Data + existing.Labels = desired.Labels + _, err := clientset.CoreV1().Secrets(namespace).Update(context.TODO(), existing, metav1.UpdateOptions{}) + if err != nil { + return nil, err + } + } + + return certs, nil +} + +func BuildDindTLSSecret(namespace string, certs *commonmodels.DindTLSCerts) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: types.DindTLSSecretName, + Namespace: namespace, + Labels: DindLabels(), + }, + Type: corev1.SecretTypeOpaque, + Data: DindTLSSecretData(certs), + } +} + +func DindTLSSecretTemplate(certs *commonmodels.DindTLSCerts) DindTLSSecretTemplateData { + data := DindTLSSecretData(certs) + return DindTLSSecretTemplateData{ + CAPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSCACertKey]), + ServerCertPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSServerCertKey]), + ServerKeyPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSServerKeyKey]), + ClientCertPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSClientCertKey]), + ClientKeyPemBase64: base64.StdEncoding.EncodeToString(data[types.DindTLSClientKeyKey]), + CertHash: DindTLSCertHash(certs), + } +} + +func DindTLSSecretData(certs *commonmodels.DindTLSCerts) map[string][]byte { + if certs == nil { + return map[string][]byte{} + } + return map[string][]byte{ + types.DindTLSCACertKey: []byte(certs.CAPem), + types.DindTLSServerCertKey: []byte(certs.ServerCertPem), + types.DindTLSServerKeyKey: []byte(certs.ServerKeyPem), + types.DindTLSClientCertKey: []byte(certs.ClientCertPem), + types.DindTLSClientKeyKey: []byte(certs.ClientKeyPem), + } +} + +func DindTLSCertHash(certs *commonmodels.DindTLSCerts) string { + data := DindTLSSecretData(certs) + keys := make([]string, 0, len(data)) + for key := range data { + keys = append(keys, key) + } + sort.Strings(keys) + + h := sha256.New() + for _, key := range keys { + h.Write([]byte(key)) + h.Write([]byte{0}) + h.Write(data[key]) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func DindLabels() map[string]string { + return map[string]string{ + "app.kubernetes.io/component": "dind", + "app.kubernetes.io/name": "zadig", + } +} + +func ApplyDindTLSSettings(dindSts *appsv1.StatefulSet, certs *commonmodels.DindTLSCerts) bool { + if dindSts == nil || len(dindSts.Spec.Template.Spec.Containers) == 0 { + return false + } + + modified := false + container := &dindSts.Spec.Template.Spec.Containers[0] + + if !reflect.DeepEqual(container.Args, buildDindTLSArgs(container.Args)) { + container.Args = buildDindTLSArgs(container.Args) + modified = true + } + + ports := ensureDindTLSPort(container.Ports) + if !reflect.DeepEqual(container.Ports, ports) { + container.Ports = ports + modified = true + } + + envs := removeDockerTLSCertDirEnv(container.Env) + if !reflect.DeepEqual(container.Env, envs) { + container.Env = envs + modified = true + } + + volumeMounts := ensureDindTLSVolumeMount(container.VolumeMounts) + if !reflect.DeepEqual(container.VolumeMounts, volumeMounts) { + container.VolumeMounts = volumeMounts + modified = true + } + + volumes := ensureDindTLSVolume(dindSts.Spec.Template.Spec.Volumes) + if !reflect.DeepEqual(dindSts.Spec.Template.Spec.Volumes, volumes) { + dindSts.Spec.Template.Spec.Volumes = volumes + modified = true + } + + if dindSts.Spec.Template.Annotations == nil { + dindSts.Spec.Template.Annotations = map[string]string{} + } + certHash := DindTLSCertHash(certs) + if dindSts.Spec.Template.Annotations[types.DindTLSCertHashAnnotation] != certHash { + dindSts.Spec.Template.Annotations[types.DindTLSCertHashAnnotation] = certHash + modified = true + } + + return modified +} + +func EnsureDindServiceTLS(kclient client.Client, namespace string) error { + service := &corev1.Service{} + if err := kclient.Get(context.TODO(), client.ObjectKey{Name: types.DindStatefulSetName, Namespace: namespace}, service); err != nil { + if apierrors.IsNotFound(err) { + return kclient.Create(context.TODO(), BuildDindService(namespace)) + } + return err + } + + ports := ensureDindTLSServicePort(service.Spec.Ports) + if reflect.DeepEqual(service.Spec.Ports, ports) { + return nil + } + service.Spec.Ports = ports + return kclient.Update(context.TODO(), service) +} + +func BuildDindService(namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: types.DindStatefulSetName, + Namespace: namespace, + Labels: DindLabels(), + }, + Spec: corev1.ServiceSpec{ + Ports: ensureDindTLSServicePort(nil), + ClusterIP: "None", + Selector: DindLabels(), + }, + } +} + +func buildDindTLSArgs(currentArgs []string) []string { + args := []string{ + "--host=unix:///var/run/docker.sock", + fmt.Sprintf("--host=tcp://0.0.0.0:%d", types.DindTLSPort), + "--tlsverify", + fmt.Sprintf("--tlscacert=%s", path.Join(types.DindTLSServerMountPath, types.DindTLSCACertKey)), + fmt.Sprintf("--tlscert=%s", path.Join(types.DindTLSServerMountPath, types.DindTLSServerCertKey)), + fmt.Sprintf("--tlskey=%s", path.Join(types.DindTLSServerMountPath, types.DindTLSServerKeyKey)), + } + + for _, arg := range currentArgs { + if isDindConnectionArg(arg) { + continue + } + args = append(args, arg) + } + + return args +} + +func isDindConnectionArg(arg string) bool { + return strings.HasPrefix(arg, "--host=") || + arg == "--tlsverify" || + strings.HasPrefix(arg, "--tlscacert") || + strings.HasPrefix(arg, "--tlscert") || + strings.HasPrefix(arg, "--tlskey") +} + +func ensureDindTLSPort(ports []corev1.ContainerPort) []corev1.ContainerPort { + resp := make([]corev1.ContainerPort, 0, len(ports)+1) + for _, port := range ports { + if port.ContainerPort == 2375 || port.ContainerPort == types.DindTLSPort { + continue + } + resp = append(resp, port) + } + resp = append(resp, corev1.ContainerPort{ + Protocol: corev1.ProtocolTCP, + ContainerPort: types.DindTLSPort, + }) + return resp +} + +func ensureDindTLSServicePort(ports []corev1.ServicePort) []corev1.ServicePort { + resp := make([]corev1.ServicePort, 0, len(ports)+1) + for _, port := range ports { + if port.Port == 2375 || port.Port == types.DindTLSPort || port.Name == types.DindContainerName { + continue + } + resp = append(resp, port) + } + resp = append(resp, corev1.ServicePort{ + Name: types.DindContainerName, + Protocol: corev1.ProtocolTCP, + Port: types.DindTLSPort, + TargetPort: intstr.FromInt(types.DindTLSPort), + }) + return resp +} + +func removeDockerTLSCertDirEnv(envs []corev1.EnvVar) []corev1.EnvVar { + resp := make([]corev1.EnvVar, 0, len(envs)) + for _, env := range envs { + if env.Name == "DOCKER_TLS_CERTDIR" { + continue + } + resp = append(resp, env) + } + return resp +} + +func ensureDindTLSVolumeMount(volumeMounts []corev1.VolumeMount) []corev1.VolumeMount { + resp := make([]corev1.VolumeMount, 0, len(volumeMounts)+1) + for _, volumeMount := range volumeMounts { + if volumeMount.Name == types.DindTLSVolumeName { + continue + } + resp = append(resp, volumeMount) + } + resp = append(resp, corev1.VolumeMount{ + Name: types.DindTLSVolumeName, + MountPath: types.DindTLSServerMountPath, + ReadOnly: true, + }) + return resp +} + +func ensureDindTLSVolume(volumes []corev1.Volume) []corev1.Volume { + resp := make([]corev1.Volume, 0, len(volumes)+1) + for _, volume := range volumes { + if volume.Name == types.DindTLSVolumeName { + continue + } + resp = append(resp, volume) + } + resp = append(resp, corev1.Volume{ + Name: types.DindTLSVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: types.DindTLSSecretName, + Items: []corev1.KeyToPath{ + { + Key: types.DindTLSCACertKey, + Path: types.DindTLSCACertKey, + }, + { + Key: types.DindTLSServerCertKey, + Path: types.DindTLSServerCertKey, + }, + { + Key: types.DindTLSServerKeyKey, + Path: types.DindTLSServerKeyKey, + }, + }, + }, + }, + }) + return resp +} + +func dindTLSCertsComplete(certs *commonmodels.DindTLSCerts) bool { + return certs != nil && + certs.CAPem != "" && + certs.CAKeyPem != "" && + certs.ServerCertPem != "" && + certs.ServerKeyPem != "" && + certs.ClientCertPem != "" && + certs.ClientKeyPem != "" +} + +func generateDindTLSCerts() (*commonmodels.DindTLSCerts, error) { + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + caTemplate := certificateTemplate("zadig-dind-ca", true, nil) + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, err + } + + serverKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + serverTemplate := certificateTemplate("zadig-dind-server", false, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}) + serverTemplate.DNSNames = dindTLSServerNames() + serverTemplate.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caTemplate, &serverKey.PublicKey, caKey) + if err != nil { + return nil, err + } + + clientKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + clientTemplate := certificateTemplate("zadig-dind-client", false, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) + clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, caTemplate, &clientKey.PublicKey, caKey) + if err != nil { + return nil, err + } + + return &commonmodels.DindTLSCerts{ + CAPem: string(encodeCert(caDER)), + CAKeyPem: string(encodePrivateKey(caKey)), + ServerCertPem: string(encodeCert(serverDER)), + ServerKeyPem: string(encodePrivateKey(serverKey)), + ClientCertPem: string(encodeCert(clientDER)), + ClientKeyPem: string(encodePrivateKey(clientKey)), + }, nil +} + +func certificateTemplate(commonName string, isCA bool, extKeyUsage []x509.ExtKeyUsage) *x509.Certificate { + now := time.Now() + keyUsage := x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment + if isCA { + keyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign + } + return &x509.Certificate{ + SerialNumber: randomSerialNumber(), + Subject: pkix.Name{CommonName: commonName}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(dindTLSCertValidity), + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsage, + BasicConstraintsValid: true, + IsCA: isCA, + } +} + +func randomSerialNumber() *big.Int { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return big.NewInt(time.Now().UnixNano()) + } + return serialNumber +} + +func dindTLSServerNames() []string { + names := []string{ + "dind", + "*.dind", + "localhost", + } + + for _, namespace := range []string{config.Namespace(), setting.AttachedClusterNamespace} { + namespace = strings.TrimSpace(namespace) + if namespace == "" { + continue + } + names = append(names, + fmt.Sprintf("dind.%s", namespace), + fmt.Sprintf("dind.%s.svc", namespace), + fmt.Sprintf("dind.%s.svc.cluster.local", namespace), + fmt.Sprintf("*.dind.%s", namespace), + fmt.Sprintf("*.dind.%s.svc", namespace), + fmt.Sprintf("*.dind.%s.svc.cluster.local", namespace), + ) + } + + return dedupeStrings(names) +} + +func dedupeStrings(values []string) []string { + seen := make(map[string]struct{}, len(values)) + ret := make([]string, 0, len(values)) + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + ret = append(ret, value) + } + return ret +} + +func encodeCert(cert []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert}) +} + +func encodePrivateKey(key *rsa.PrivateKey) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) +} diff --git a/pkg/microservice/aslan/core/common/service/kube/service.go b/pkg/microservice/aslan/core/common/service/kube/service.go index f377bdd087..9c66cdcf53 100644 --- a/pkg/microservice/aslan/core/common/service/kube/service.go +++ b/pkg/microservice/aslan/core/common/service/kube/service.go @@ -306,6 +306,11 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment } dindReplicas, dindLimitsCPU, dindLimitsMemory, dindEnablePV, dindSCName, dindStorageSizeInGiB, dindStorageDriver := getDindCfg(cluster) + dindTLSCerts, err := EnsureDindTLSCerts() + if err != nil { + return nil, fmt.Errorf("failed to ensure dind TLS certs: %w", err) + } + dindTLS := DindTLSSecretTemplate(dindTLSCerts) yaml := agentYaml if cluster.AdvancedConfig != nil { @@ -362,6 +367,7 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment DindStorageClassName: dindSCName, DindStorageSizeInGiB: dindStorageSizeInGiB, DindStorageDriver: dindStorageDriver, + DindTLS: dindTLS, ScheduleWorkflow: scheduleWorkflow, EnableIRSA: cluster.AdvancedConfig.EnableIRSA, IRSARoleARN: cluster.AdvancedConfig.IRSARoleARM, @@ -387,15 +393,15 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment DindEnablePV: dindEnablePV, DindStorageClassName: dindSCName, DindStorageSizeInGiB: dindStorageSizeInGiB, - DindStorageDriver: dindStorageDriver, - EnableIRSA: cluster.AdvancedConfig.EnableIRSA, - NodeSelector: cluster.AdvancedConfig.AgentNodeSelector, - Toleration: cluster.AdvancedConfig.AgentToleration, - Affinity: cluster.AdvancedConfig.AgentAffinity, - DisableHostNetwork: cluster.AdvancedConfig.AgentDisableHostNetwork, - IRSARoleARN: cluster.AdvancedConfig.IRSARoleARM, - ImagePullPolicy: configbase.ImagePullPolicy(), - SecretKey: base64.StdEncoding.EncodeToString([]byte(configbase.SecretKey())), + DindStorageDriver: dindStorageDriver, DindTLS: dindTLS, + EnableIRSA: cluster.AdvancedConfig.EnableIRSA, + NodeSelector: cluster.AdvancedConfig.AgentNodeSelector, + Toleration: cluster.AdvancedConfig.AgentToleration, + Affinity: cluster.AdvancedConfig.AgentAffinity, + DisableHostNetwork: cluster.AdvancedConfig.AgentDisableHostNetwork, + IRSARoleARN: cluster.AdvancedConfig.IRSARoleARM, + ImagePullPolicy: configbase.ImagePullPolicy(), + SecretKey: base64.StdEncoding.EncodeToString([]byte(configbase.SecretKey())), }) } @@ -452,6 +458,13 @@ func getDindCfg(cluster *models.K8SCluster) (replicas int, limitsCPU, limitsMemo return } +func ResolveDindNamespace(cluster *models.K8SCluster) string { + if cluster != nil && cluster.Local { + return config.Namespace() + } + return setting.AttachedClusterNamespace +} + // InitializeExternalCluster initialized the resources in the cluster for zadig to run correctly. // if the cluster is of type kubeconfig, we need to create following resource: // Namespace: koderover-agent @@ -516,6 +529,11 @@ func InitializeExternalCluster(clusterID string) error { return errors.Errorf("cluster %s create serviceAccount err: %s", clusterID, err) } + dindTLSCerts, err := EnsureDindTLSSecret(clientset, namespace) + if err != nil { + return fmt.Errorf("failed to ensure dind TLS secret in cluster %q: %s", clusterID, err) + } + // create role binding roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ @@ -540,10 +558,7 @@ func InitializeExternalCluster(clusterID string) error { } log.Infof("cluster %s create role binding successfully", clusterID) - dindLabelMap := map[string]string{ - "app.kubernetes.io/component": "dind", - "app.kubernetes.io/name": "zadig", - } + dindLabelMap := DindLabels() privileged := true @@ -561,6 +576,9 @@ func InitializeExternalCluster(clusterID string) error { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: dindLabelMap, + Annotations: map[string]string{ + types.DindTLSCertHashAnnotation: DindTLSCertHash(dindTLSCerts), + }, }, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{ @@ -579,18 +597,14 @@ func InitializeExternalCluster(clusterID string) error { corev1.Container{ Name: "dind", Image: config.DindImage(), + Args: buildDindTLSArgs(nil), Ports: []corev1.ContainerPort{ { Protocol: "TCP", - ContainerPort: 2375, - }, - }, - Env: []corev1.EnvVar{ - { - Name: "DOCKER_TLS_CERTDIR", - Value: "", + ContainerPort: types.DindTLSPort, }, }, + VolumeMounts: ensureDindTLSVolumeMount(nil), Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(strconv.Itoa(2)), @@ -606,6 +620,7 @@ func InitializeExternalCluster(clusterID string) error { }, }, }, + Volumes: ensureDindTLSVolume(nil), }, }, }, @@ -627,8 +642,8 @@ func InitializeExternalCluster(clusterID string) error { { Name: "dind", Protocol: "TCP", - Port: 2375, - TargetPort: intstr.FromInt(2375), + Port: types.DindTLSPort, + TargetPort: intstr.FromInt(types.DindTLSPort), }, }, ClusterIP: "None", @@ -691,6 +706,7 @@ type TemplateSchema struct { DindStorageClassName string DindStorageSizeInGiB int DindStorageDriver string + DindTLS DindTLSSecretTemplateData ScheduleWorkflow bool EnableIRSA bool IRSARoleARN string @@ -1045,6 +1061,24 @@ spec: --- +apiVersion: v1 +kind: Secret +metadata: + name: dind-tls-certs + namespace: {{.Namespace}} + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +type: Opaque +data: + ca.pem: {{.DindTLS.CAPemBase64}} + server-cert.pem: {{.DindTLS.ServerCertPemBase64}} + server-key.pem: {{.DindTLS.ServerKeyPemBase64}} + cert.pem: {{.DindTLS.ClientCertPemBase64}} + key.pem: {{.DindTLS.ClientKeyPemBase64}} + +--- + apiVersion: apps/v1 kind: StatefulSet metadata: @@ -1065,6 +1099,8 @@ spec: labels: app.kubernetes.io/component: dind app.kubernetes.io/name: zadig + annotations: + zadig.koderover.com/dind-tls-cert-hash: {{.DindTLS.CertHash}} spec: affinity: podAntiAffinity: @@ -1072,21 +1108,35 @@ spec: - weight: 100 podAffinityTerm: topologyKey: kubernetes.io/hostname + volumes: + - name: dind-tls-certs + secret: + secretName: dind-tls-certs + items: + - key: ca.pem + path: ca.pem + - key: server-cert.pem + path: server-cert.pem + - key: server-key.pem + path: server-key.pem containers: - name: dind image: {{.DindImage}} - {{- if .DindStorageDriver }} args: + - --host=unix:///var/run/docker.sock + - --host=tcp://0.0.0.0:2376 + - --tlsverify + - --tlscacert=/etc/zadig/dind/tls/ca.pem + - --tlscert=/etc/zadig/dind/tls/server-cert.pem + - --tlskey=/etc/zadig/dind/tls/server-key.pem + {{ if .DindStorageDriver }} - --storage-driver={{.DindStorageDriver}} - {{- end }} - env: - - name: DOCKER_TLS_CERTDIR - value: "" + {{ end }} securityContext: privileged: true ports: - protocol: TCP - containerPort: 2375 + containerPort: 2376 resources: limits: cpu: "4" @@ -1094,10 +1144,15 @@ spec: requests: cpu: 100m memory: 128Mi -{{- if .DindEnablePV }} volumeMounts: + - name: dind-tls-certs + mountPath: /etc/zadig/dind/tls + readOnly: true +{{ if .DindEnablePV }} - name: zadig-docker mountPath: /var/lib/docker +{{ end }} +{{ if .DindEnablePV }} volumeClaimTemplates: - metadata: name: zadig-docker @@ -1107,7 +1162,7 @@ spec: resources: requests: storage: {{.DindStorageSizeInGiB}}Gi -{{- end }} +{{ end }} --- @@ -1123,8 +1178,8 @@ spec: ports: - name: dind protocol: TCP - port: 2375 - targetPort: 2375 + port: 2376 + targetPort: 2376 clusterIP: None selector: app.kubernetes.io/component: dind @@ -1190,6 +1245,24 @@ roleRef: --- +apiVersion: v1 +kind: Secret +metadata: + name: dind-tls-certs + namespace: koderover-agent + labels: + app.kubernetes.io/component: dind + app.kubernetes.io/name: zadig +type: Opaque +data: + ca.pem: {{.DindTLS.CAPemBase64}} + server-cert.pem: {{.DindTLS.ServerCertPemBase64}} + server-key.pem: {{.DindTLS.ServerKeyPemBase64}} + cert.pem: {{.DindTLS.ClientCertPemBase64}} + key.pem: {{.DindTLS.ClientKeyPemBase64}} + +--- + apiVersion: apps/v1 kind: StatefulSet metadata: @@ -1210,6 +1283,8 @@ spec: labels: app.kubernetes.io/component: dind app.kubernetes.io/name: zadig + annotations: + zadig.koderover.com/dind-tls-cert-hash: {{.DindTLS.CertHash}} spec: affinity: podAntiAffinity: @@ -1217,21 +1292,35 @@ spec: - weight: 100 podAffinityTerm: topologyKey: kubernetes.io/hostname + volumes: + - name: dind-tls-certs + secret: + secretName: dind-tls-certs + items: + - key: ca.pem + path: ca.pem + - key: server-cert.pem + path: server-cert.pem + - key: server-key.pem + path: server-key.pem containers: - name: dind image: {{.DindImage}} - {{- if .DindStorageDriver }} args: + - --host=unix:///var/run/docker.sock + - --host=tcp://0.0.0.0:2376 + - --tlsverify + - --tlscacert=/etc/zadig/dind/tls/ca.pem + - --tlscert=/etc/zadig/dind/tls/server-cert.pem + - --tlskey=/etc/zadig/dind/tls/server-key.pem + {{ if .DindStorageDriver }} - --storage-driver={{.DindStorageDriver}} - {{- end }} - env: - - name: DOCKER_TLS_CERTDIR - value: "" + {{ end }} securityContext: privileged: true ports: - protocol: TCP - containerPort: 2375 + containerPort: 2376 resources: limits: cpu: {{.DindLimitsCPU}} @@ -1239,10 +1328,15 @@ spec: requests: cpu: 100m memory: 128Mi -{{- if .DindEnablePV }} volumeMounts: + - name: dind-tls-certs + mountPath: /etc/zadig/dind/tls + readOnly: true +{{ if .DindEnablePV }} - name: zadig-docker mountPath: /var/lib/docker +{{ end }} +{{ if .DindEnablePV }} volumeClaimTemplates: - metadata: name: zadig-docker @@ -1252,7 +1346,7 @@ spec: resources: requests: storage: {{.DindStorageSizeInGiB}}Gi -{{- end }} +{{ end }} --- @@ -1268,8 +1362,8 @@ spec: ports: - name: dind protocol: TCP - port: 2375 - targetPort: 2375 + port: 2376 + targetPort: 2376 clusterIP: None selector: app.kubernetes.io/component: dind diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go index c18fd03d27..4a28a20a46 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/kubernetes.go @@ -750,6 +750,14 @@ func getEnvs(configMapMountDir string, jobTaskSpec *commonmodels.JobTaskFreestyl Name: setting.DockerHost, Value: jobTaskSpec.Properties.DockerHost, }) + ret = append(ret, corev1.EnvVar{ + Name: setting.DockerTLSVerify, + Value: "1", + }) + ret = append(ret, corev1.EnvVar{ + Name: setting.DockerCertPath, + Value: types.DindTLSClientMountPath, + }) } ret = append(ret, corev1.EnvVar{ Name: setting.ENVLogLevel, @@ -783,6 +791,12 @@ func getVolumeMounts(configMapMountDir string, userHostDockerDaemon bool) []core Name: "docker-sock", MountPath: setting.DefaultDockSock, }) + } else { + resp = append(resp, corev1.VolumeMount{ + Name: types.DindTLSVolumeName, + MountPath: types.DindTLSClientMountPath, + ReadOnly: true, + }) } return resp } @@ -828,6 +842,29 @@ func getVolumes(jobName string, userHostDockerDaemon bool) []corev1.Volume { }, }, }) + } else { + resp = append(resp, corev1.Volume{ + Name: types.DindTLSVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: types.DindTLSSecretName, + Items: []corev1.KeyToPath{ + { + Key: types.DindTLSCACertKey, + Path: types.DindTLSCACertKey, + }, + { + Key: types.DindTLSClientCertKey, + Path: types.DindTLSClientCertKey, + }, + { + Key: types.DindTLSClientKeyKey, + Path: types.DindTLSClientKeyKey, + }, + }, + }, + }, + }) } return resp diff --git a/pkg/microservice/aslan/core/multicluster/service/clusters.go b/pkg/microservice/aslan/core/multicluster/service/clusters.go index 6300d8cd2f..35e783501d 100644 --- a/pkg/microservice/aslan/core/multicluster/service/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/service/clusters.go @@ -956,8 +956,24 @@ func UpgradeAgent(id string, logger *zap.SugaredLogger) error { return errors.Errorf("cluster %s update serviceAccount err: %s", id, err) } + if _, err := kube.EnsureDindTLSSecret(clientset, config.Namespace()); err != nil { + return fmt.Errorf("failed to ensure dind TLS secret in local cluster: %s", err) + } + return UpgradeDind(kubeClient, clusterInfo, config.Namespace()) } else { + clientset, err := clientmanager.NewKubeClientManager().GetKubernetesClientSet(id) + if err != nil { + return err + } + dindNamespace := kube.ResolveDindNamespace(clusterInfo) + if _, err := kube.EnsureDindTLSSecret(clientset, dindNamespace); err != nil { + logger.Warnf("failed to ensure dind TLS secret in cluster %s before applying yaml: %s", clusterInfo.Name, err) + if _, certErr := kube.EnsureDindTLSCerts(); certErr != nil { + return fmt.Errorf("failed to ensure dind TLS certs for cluster %s: %s", clusterInfo.Name, certErr) + } + } + // Upgrade attached cluster. yamls, err := s.GetYaml(id, config.HubAgentImage(), serverURL, "/api/hub", true, logger) if err != nil { @@ -1080,7 +1096,7 @@ func setClusterDind(cluster *commonmodels.K8SCluster) error { return nil } - if cluster.DindCfg.Replicas <= 0 { + if cluster.DindCfg.Replicas < 0 { cluster.DindCfg.Replicas = kube.DefaultDindReplicas } if cluster.DindCfg.Resources == nil { @@ -1188,9 +1204,13 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str } ctx := context.TODO() + dindTLSCerts, err := kube.EnsureDindTLSCerts() + if err != nil { + return err + } // Retry logic for handling concurrent modifications - err := retryOnConflict(func() error { + err = retryOnConflict(func() error { dindSts := &appsv1.StatefulSet{} err := kclient.Get(ctx, client.ObjectKey{ Name: types.DindStatefulSetName, @@ -1206,13 +1226,17 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str return fmt.Errorf("failed to deep copy original dind statefulset, error: %s", err) } - return applyDindUpgrade(kclient, ctx, dindSts, originalSts, cluster, ns) + return applyDindUpgrade(kclient, ctx, dindSts, originalSts, cluster, ns, dindTLSCerts) }) if err != nil { return err } + if err := kube.EnsureDindServiceTLS(kclient, ns); err != nil { + return err + } + // Sync registry configuration after successful update err = commonutil.SyncDinDForRegistries() if err != nil { @@ -1222,7 +1246,7 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str return nil } -func applyDindUpgrade(kclient client.Client, ctx context.Context, dindSts, originalSts *appsv1.StatefulSet, cluster *commonmodels.K8SCluster, ns string) error { +func applyDindUpgrade(kclient client.Client, ctx context.Context, dindSts, originalSts *appsv1.StatefulSet, cluster *commonmodels.K8SCluster, ns string, dindTLSCerts *commonmodels.DindTLSCerts) error { var err error dindSts.Spec.Replicas = util.GetInt32Pointer(int32(cluster.DindCfg.Replicas)) @@ -1365,6 +1389,7 @@ func applyDindUpgrade(kclient client.Client, ctx context.Context, dindSts, origi dindSts.Spec.Template.Spec.Containers[0].Args = finalArgs } + kube.ApplyDindTLSSettings(dindSts, dindTLSCerts) if stsHasImmutableFieldChanged(originalSts, dindSts) { log.Infof("dind has immutable field changed, recreating dind.") diff --git a/pkg/setting/consts.go b/pkg/setting/consts.go index 7c5c104940..d615379629 100644 --- a/pkg/setting/consts.go +++ b/pkg/setting/consts.go @@ -96,6 +96,8 @@ const ( DockerAuthDir = "DOCKER_AUTH_DIR" Path = "PATH" DockerHost = "DOCKER_HOST" + DockerTLSVerify = "DOCKER_TLS_VERIFY" + DockerCertPath = "DOCKER_CERT_PATH" BuildURL = "BUILD_URL" DefaultDockSock = "/var/run/docker.sock" diff --git a/pkg/tool/dockerhost/docker_host.go b/pkg/tool/dockerhost/docker_host.go index b78fe3f282..267db70d58 100644 --- a/pkg/tool/dockerhost/docker_host.go +++ b/pkg/tool/dockerhost/docker_host.go @@ -37,6 +37,7 @@ import ( "github.com/koderover/zadig/v2/pkg/tool/cache" "github.com/koderover/zadig/v2/pkg/tool/clientmanager" "github.com/koderover/zadig/v2/pkg/tool/log" + "github.com/koderover/zadig/v2/pkg/types" ) var ( @@ -151,10 +152,7 @@ func (d *dockerhosts) initClusterInfo(clusterID ClusterID) { } func (d *dockerhosts) getDockerHostsSvc(clusterID ClusterID) []consistent.Member { - ns := config.Namespace() - if string(clusterID) != setting.LocalClusterID { - ns = setting.AttachedClusterNamespace - } + ns := d.getDindNamespace(clusterID) kclient, err := clientmanager.NewKubeClientManager().GetControllerRuntimeClient(string(clusterID)) if err != nil { @@ -180,12 +178,19 @@ func (d *dockerhosts) getDockerHostsSvc(clusterID ClusterID) []consistent.Member return members } +func (d *dockerhosts) getDindNamespace(clusterID ClusterID) string { + if clusterID == "" || string(clusterID) == setting.LocalClusterID { + return config.Namespace() + } + return setting.AttachedClusterNamespace +} + func (d *dockerhosts) getDefaultDockerHosts() []consistent.Member { return []consistent.Member{d.genDindAddr(0)} } func (d *dockerhosts) genDindAddr(idx int) Member { - return Member(fmt.Sprintf("tcp://dind-%d.dind:2375", idx)) + return Member(fmt.Sprintf("tcp://dind-%d.dind:%d", idx, types.DindTLSPort)) } func (d *dockerhosts) Sync() { diff --git a/pkg/types/workload.go b/pkg/types/workload.go index c786f5dfe5..a237885cf7 100644 --- a/pkg/types/workload.go +++ b/pkg/types/workload.go @@ -36,6 +36,17 @@ const DindStatefulSetName = "dind" const DindContainerName = "dind" const DindMountName = "zadig-docker" const DindMountPath = "/var/lib/docker" +const DindTLSPort = 2376 +const DindTLSSecretName = "dind-tls-certs" +const DindTLSVolumeName = "dind-tls-certs" +const DindTLSServerMountPath = "/etc/zadig/dind/tls" +const DindTLSClientMountPath = "/etc/zadig/dind/client" +const DindTLSCertHashAnnotation = "zadig.koderover.com/dind-tls-cert-hash" +const DindTLSCACertKey = "ca.pem" +const DindTLSServerCertKey = "server-cert.pem" +const DindTLSServerKeyKey = "server-key.pem" +const DindTLSClientCertKey = "cert.pem" +const DindTLSClientKeyKey = "key.pem" type KubeResourceKind struct { APIVersion string `yaml:"apiVersion"` From 53631c14ca9ab006cc2fcc97c214193b8bbe8ba6 Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Mon, 1 Jun 2026 09:55:53 +0800 Subject: [PATCH 2/4] change dind svc port name Signed-off-by: Patrick Zhao --- .../core/common/service/kube/dind_tls.go | 24 ++++++++++++++----- .../core/multicluster/service/clusters.go | 14 +++++++++-- pkg/types/workload.go | 1 + 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go index 5c65ff29a7..8d92adaee1 100644 --- a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go +++ b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go @@ -240,15 +240,23 @@ func ApplyDindTLSSettings(dindSts *appsv1.StatefulSet, certs *commonmodels.DindT } func EnsureDindServiceTLS(kclient client.Client, namespace string) error { + return ensureDindServiceTLS(kclient, namespace, types.DindContainerName) +} + +func EnsureLocalDindServiceTLS(kclient client.Client, namespace string) error { + return ensureDindServiceTLS(kclient, namespace, types.DindTLSServicePortName) +} + +func ensureDindServiceTLS(kclient client.Client, namespace, portName string) error { service := &corev1.Service{} if err := kclient.Get(context.TODO(), client.ObjectKey{Name: types.DindStatefulSetName, Namespace: namespace}, service); err != nil { if apierrors.IsNotFound(err) { - return kclient.Create(context.TODO(), BuildDindService(namespace)) + return kclient.Create(context.TODO(), buildDindService(namespace, portName)) } return err } - ports := ensureDindTLSServicePort(service.Spec.Ports) + ports := ensureDindTLSServicePort(service.Spec.Ports, portName) if reflect.DeepEqual(service.Spec.Ports, ports) { return nil } @@ -257,6 +265,10 @@ func EnsureDindServiceTLS(kclient client.Client, namespace string) error { } func BuildDindService(namespace string) *corev1.Service { + return buildDindService(namespace, types.DindContainerName) +} + +func buildDindService(namespace, portName string) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: types.DindStatefulSetName, @@ -264,7 +276,7 @@ func BuildDindService(namespace string) *corev1.Service { Labels: DindLabels(), }, Spec: corev1.ServiceSpec{ - Ports: ensureDindTLSServicePort(nil), + Ports: ensureDindTLSServicePort(nil, portName), ClusterIP: "None", Selector: DindLabels(), }, @@ -314,16 +326,16 @@ func ensureDindTLSPort(ports []corev1.ContainerPort) []corev1.ContainerPort { return resp } -func ensureDindTLSServicePort(ports []corev1.ServicePort) []corev1.ServicePort { +func ensureDindTLSServicePort(ports []corev1.ServicePort, portName string) []corev1.ServicePort { resp := make([]corev1.ServicePort, 0, len(ports)+1) for _, port := range ports { - if port.Port == 2375 || port.Port == types.DindTLSPort || port.Name == types.DindContainerName { + if port.Port == 2375 || port.Port == types.DindTLSPort || port.Name == types.DindContainerName || port.Name == types.DindTLSServicePortName { continue } resp = append(resp, port) } resp = append(resp, corev1.ServicePort{ - Name: types.DindContainerName, + Name: portName, Protocol: corev1.ProtocolTCP, Port: types.DindTLSPort, TargetPort: intstr.FromInt(types.DindTLSPort), diff --git a/pkg/microservice/aslan/core/multicluster/service/clusters.go b/pkg/microservice/aslan/core/multicluster/service/clusters.go index 35e783501d..bdcb140160 100644 --- a/pkg/microservice/aslan/core/multicluster/service/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/service/clusters.go @@ -960,6 +960,10 @@ func UpgradeAgent(id string, logger *zap.SugaredLogger) error { return fmt.Errorf("failed to ensure dind TLS secret in local cluster: %s", err) } + if err := kube.EnsureLocalDindServiceTLS(kubeClient, config.Namespace()); err != nil { + return fmt.Errorf("failed to ensure local dind TLS service: %s", err) + } + return UpgradeDind(kubeClient, clusterInfo, config.Namespace()) } else { clientset, err := clientmanager.NewKubeClientManager().GetKubernetesClientSet(id) @@ -1233,8 +1237,14 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str return err } - if err := kube.EnsureDindServiceTLS(kclient, ns); err != nil { - return err + if cluster.Local { + if err := kube.EnsureLocalDindServiceTLS(kclient, ns); err != nil { + return err + } + } else { + if err := kube.EnsureDindServiceTLS(kclient, ns); err != nil { + return err + } } // Sync registry configuration after successful update diff --git a/pkg/types/workload.go b/pkg/types/workload.go index a237885cf7..0ddefc7389 100644 --- a/pkg/types/workload.go +++ b/pkg/types/workload.go @@ -37,6 +37,7 @@ const DindContainerName = "dind" const DindMountName = "zadig-docker" const DindMountPath = "/var/lib/docker" const DindTLSPort = 2376 +const DindTLSServicePortName = "dind-tls" const DindTLSSecretName = "dind-tls-certs" const DindTLSVolumeName = "dind-tls-certs" const DindTLSServerMountPath = "/etc/zadig/dind/tls" From 7a11f8d92fabc0bfc620248d5a198ffc4fde8b11 Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Mon, 1 Jun 2026 11:15:20 +0800 Subject: [PATCH 3/4] change all dind tls port name Signed-off-by: Patrick Zhao --- .../core/common/service/kube/dind_tls.go | 22 +++++---------- .../aslan/core/common/service/kube/service.go | 27 ++++++++++--------- .../core/multicluster/service/clusters.go | 25 +++++++---------- 3 files changed, 31 insertions(+), 43 deletions(-) diff --git a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go index 8d92adaee1..c3772fca1e 100644 --- a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go +++ b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go @@ -240,23 +240,19 @@ func ApplyDindTLSSettings(dindSts *appsv1.StatefulSet, certs *commonmodels.DindT } func EnsureDindServiceTLS(kclient client.Client, namespace string) error { - return ensureDindServiceTLS(kclient, namespace, types.DindContainerName) + return ensureDindServiceTLS(kclient, namespace) } -func EnsureLocalDindServiceTLS(kclient client.Client, namespace string) error { - return ensureDindServiceTLS(kclient, namespace, types.DindTLSServicePortName) -} - -func ensureDindServiceTLS(kclient client.Client, namespace, portName string) error { +func ensureDindServiceTLS(kclient client.Client, namespace string) error { service := &corev1.Service{} if err := kclient.Get(context.TODO(), client.ObjectKey{Name: types.DindStatefulSetName, Namespace: namespace}, service); err != nil { if apierrors.IsNotFound(err) { - return kclient.Create(context.TODO(), buildDindService(namespace, portName)) + return kclient.Create(context.TODO(), BuildDindService(namespace)) } return err } - ports := ensureDindTLSServicePort(service.Spec.Ports, portName) + ports := ensureDindTLSServicePort(service.Spec.Ports) if reflect.DeepEqual(service.Spec.Ports, ports) { return nil } @@ -265,10 +261,6 @@ func ensureDindServiceTLS(kclient client.Client, namespace, portName string) err } func BuildDindService(namespace string) *corev1.Service { - return buildDindService(namespace, types.DindContainerName) -} - -func buildDindService(namespace, portName string) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: types.DindStatefulSetName, @@ -276,7 +268,7 @@ func buildDindService(namespace, portName string) *corev1.Service { Labels: DindLabels(), }, Spec: corev1.ServiceSpec{ - Ports: ensureDindTLSServicePort(nil, portName), + Ports: ensureDindTLSServicePort(nil), ClusterIP: "None", Selector: DindLabels(), }, @@ -326,7 +318,7 @@ func ensureDindTLSPort(ports []corev1.ContainerPort) []corev1.ContainerPort { return resp } -func ensureDindTLSServicePort(ports []corev1.ServicePort, portName string) []corev1.ServicePort { +func ensureDindTLSServicePort(ports []corev1.ServicePort) []corev1.ServicePort { resp := make([]corev1.ServicePort, 0, len(ports)+1) for _, port := range ports { if port.Port == 2375 || port.Port == types.DindTLSPort || port.Name == types.DindContainerName || port.Name == types.DindTLSServicePortName { @@ -335,7 +327,7 @@ func ensureDindTLSServicePort(ports []corev1.ServicePort, portName string) []cor resp = append(resp, port) } resp = append(resp, corev1.ServicePort{ - Name: portName, + Name: types.DindTLSServicePortName, Protocol: corev1.ProtocolTCP, Port: types.DindTLSPort, TargetPort: intstr.FromInt(types.DindTLSPort), diff --git a/pkg/microservice/aslan/core/common/service/kube/service.go b/pkg/microservice/aslan/core/common/service/kube/service.go index 9c66cdcf53..78d072064a 100644 --- a/pkg/microservice/aslan/core/common/service/kube/service.go +++ b/pkg/microservice/aslan/core/common/service/kube/service.go @@ -393,15 +393,16 @@ func (s *Service) GetYaml(id, agentImage, aslanURL, hubURI string, useDeployment DindEnablePV: dindEnablePV, DindStorageClassName: dindSCName, DindStorageSizeInGiB: dindStorageSizeInGiB, - DindStorageDriver: dindStorageDriver, DindTLS: dindTLS, - EnableIRSA: cluster.AdvancedConfig.EnableIRSA, - NodeSelector: cluster.AdvancedConfig.AgentNodeSelector, - Toleration: cluster.AdvancedConfig.AgentToleration, - Affinity: cluster.AdvancedConfig.AgentAffinity, - DisableHostNetwork: cluster.AdvancedConfig.AgentDisableHostNetwork, - IRSARoleARN: cluster.AdvancedConfig.IRSARoleARM, - ImagePullPolicy: configbase.ImagePullPolicy(), - SecretKey: base64.StdEncoding.EncodeToString([]byte(configbase.SecretKey())), + DindStorageDriver: dindStorageDriver, + DindTLS: dindTLS, + EnableIRSA: cluster.AdvancedConfig.EnableIRSA, + NodeSelector: cluster.AdvancedConfig.AgentNodeSelector, + Toleration: cluster.AdvancedConfig.AgentToleration, + Affinity: cluster.AdvancedConfig.AgentAffinity, + DisableHostNetwork: cluster.AdvancedConfig.AgentDisableHostNetwork, + IRSARoleARN: cluster.AdvancedConfig.IRSARoleARM, + ImagePullPolicy: configbase.ImagePullPolicy(), + SecretKey: base64.StdEncoding.EncodeToString([]byte(configbase.SecretKey())), }) } @@ -595,7 +596,7 @@ func InitializeExternalCluster(clusterID string) error { }, Containers: []corev1.Container{ corev1.Container{ - Name: "dind", + Name: "dind-tls", Image: config.DindImage(), Args: buildDindTLSArgs(nil), Ports: []corev1.ContainerPort{ @@ -640,7 +641,7 @@ func InitializeExternalCluster(clusterID string) error { Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { - Name: "dind", + Name: "dind-tls", Protocol: "TCP", Port: types.DindTLSPort, TargetPort: intstr.FromInt(types.DindTLSPort), @@ -1176,7 +1177,7 @@ metadata: app.kubernetes.io/name: zadig spec: ports: - - name: dind + - name: dind-tls protocol: TCP port: 2376 targetPort: 2376 @@ -1360,7 +1361,7 @@ metadata: app.kubernetes.io/name: zadig spec: ports: - - name: dind + - name: dind-tls protocol: TCP port: 2376 targetPort: 2376 diff --git a/pkg/microservice/aslan/core/multicluster/service/clusters.go b/pkg/microservice/aslan/core/multicluster/service/clusters.go index bdcb140160..4a1af5ef78 100644 --- a/pkg/microservice/aslan/core/multicluster/service/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/service/clusters.go @@ -694,7 +694,7 @@ func UpdateClusterDind(ctx *handler.Context, id string, dindCfg *commonmodels.Di err := commonutil.CheckZadigProfessionalLicense() if err != nil { - if dindCfg.Replicas != 1 { + if dindCfg.Replicas != 0 && dindCfg.Replicas != 1 { return nil, e.ErrLicenseInvalid.AddDesc("") } if dindCfg.Resources != nil { @@ -960,8 +960,8 @@ func UpgradeAgent(id string, logger *zap.SugaredLogger) error { return fmt.Errorf("failed to ensure dind TLS secret in local cluster: %s", err) } - if err := kube.EnsureLocalDindServiceTLS(kubeClient, config.Namespace()); err != nil { - return fmt.Errorf("failed to ensure local dind TLS service: %s", err) + if err := kube.EnsureDindServiceTLS(kubeClient, config.Namespace()); err != nil { + return fmt.Errorf("failed to ensure dind TLS service in local cluster: %s", err) } return UpgradeDind(kubeClient, clusterInfo, config.Namespace()) @@ -972,10 +972,9 @@ func UpgradeAgent(id string, logger *zap.SugaredLogger) error { } dindNamespace := kube.ResolveDindNamespace(clusterInfo) if _, err := kube.EnsureDindTLSSecret(clientset, dindNamespace); err != nil { - logger.Warnf("failed to ensure dind TLS secret in cluster %s before applying yaml: %s", clusterInfo.Name, err) - if _, certErr := kube.EnsureDindTLSCerts(); certErr != nil { - return fmt.Errorf("failed to ensure dind TLS certs for cluster %s: %s", clusterInfo.Name, certErr) - } + err = fmt.Errorf("failed to ensure dind TLS secret in cluster %s before applying yaml: %s", clusterInfo.Name, err) + logger.Error(err) + return err } // Upgrade attached cluster. @@ -1013,6 +1012,8 @@ func UpgradeAgent(id string, logger *zap.SugaredLogger) error { } else { err = UpgradeDind(kubeClient, clusterInfo, setting.AttachedClusterNamespace) } + } else if u.GetKind() == "Service" && u.GetName() == types.DindStatefulSetName { + err = kube.EnsureDindServiceTLS(kubeClient, dindNamespace) } else { err = updater.CreateOrPatchUnstructured(u, kubeClient) } @@ -1237,14 +1238,8 @@ func UpgradeDind(kclient client.Client, cluster *commonmodels.K8SCluster, ns str return err } - if cluster.Local { - if err := kube.EnsureLocalDindServiceTLS(kclient, ns); err != nil { - return err - } - } else { - if err := kube.EnsureDindServiceTLS(kclient, ns); err != nil { - return err - } + if err := kube.EnsureDindServiceTLS(kclient, ns); err != nil { + return err } // Sync registry configuration after successful update From 7beb1d7e41e85d14fc01447fc1c10e53984890c5 Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Tue, 2 Jun 2026 10:43:51 +0800 Subject: [PATCH 4/4] fix concurrent issue Signed-off-by: Patrick Zhao --- .../common/repository/mongodb/settings.go | 32 +++++++++---------- .../core/common/service/kube/dind_tls.go | 20 ++++++------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/settings.go b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go index 99729e1938..6ed8becc51 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/settings.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/settings.go @@ -59,6 +59,19 @@ func (c *SystemSettingColl) Get() (*models.SystemSetting, error) { return resp, err } +func (c *SystemSettingColl) GetByID() (*models.SystemSetting, error) { + objectID, err := primitive.ObjectIDFromHex(setting.LocalClusterID) + if err != nil { + return nil, err + } + + query := bson.M{"_id": objectID} + resp := &models.SystemSetting{} + + err = c.FindOne(context.TODO(), query).Decode(resp) + return resp, err +} + func (c *SystemSettingColl) UpdateDefaultLoginSetting(defaultLogin string) error { id, _ := primitive.ObjectIDFromHex(setting.LocalClusterID) change := bson.M{"$set": bson.M{ @@ -203,32 +216,17 @@ func (c *SystemSettingColl) InitDindTLSCertsIfNeeded(certs *models.DindTLSCerts) "$or": []bson.M{ {"dind_tls_certs": bson.M{"$exists": false}}, {"dind_tls_certs": nil}, - {"dind_tls_certs.ca_pem": bson.M{"$exists": false}}, - {"dind_tls_certs.ca_pem": ""}, - {"dind_tls_certs.ca_key_pem": bson.M{"$exists": false}}, - {"dind_tls_certs.ca_key_pem": ""}, - {"dind_tls_certs.server_cert_pem": bson.M{"$exists": false}}, - {"dind_tls_certs.server_cert_pem": ""}, - {"dind_tls_certs.server_key_pem": bson.M{"$exists": false}}, - {"dind_tls_certs.server_key_pem": ""}, - {"dind_tls_certs.client_cert_pem": bson.M{"$exists": false}}, - {"dind_tls_certs.client_cert_pem": ""}, - {"dind_tls_certs.client_key_pem": bson.M{"$exists": false}}, - {"dind_tls_certs.client_key_pem": ""}, }, } change := bson.M{"$set": bson.M{ "dind_tls_certs": certs, "update_time": time.Now().Unix(), }} - result, err := c.UpdateOne(context.TODO(), query, change, options.Update().SetUpsert(true)) + result, err := c.UpdateOne(context.TODO(), query, change) if err != nil { - if mongo.IsDuplicateKeyError(err) { - return false, nil - } return false, err } - return result.ModifiedCount > 0 || result.UpsertedCount > 0, nil + return result.ModifiedCount > 0, nil } func (c *SystemSettingColl) UpdateReleasePlanHookSetting(hookSetting *models.ReleasePlanHookSettings) error { diff --git a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go index c3772fca1e..6f30601956 100644 --- a/pkg/microservice/aslan/core/common/service/kube/dind_tls.go +++ b/pkg/microservice/aslan/core/common/service/kube/dind_tls.go @@ -52,7 +52,7 @@ import ( "github.com/koderover/zadig/v2/pkg/types" ) -const dindTLSCertValidity = 10 * 365 * 24 * time.Hour +const dindTLSCertValidity = 100 * 365 * 24 * time.Hour type DindTLSSecretTemplateData struct { CAPemBase64 string @@ -65,9 +65,14 @@ type DindTLSSecretTemplateData struct { func EnsureDindTLSCerts() (*commonmodels.DindTLSCerts, error) { settingColl := commonrepo.NewSystemSettingColl() - settings, err := settingColl.Get() - if err == nil && dindTLSCertsComplete(settings.DindTLSCerts) { - return settings.DindTLSCerts, nil + settings, err := settingColl.GetByID() + if err == nil { + if dindTLSCertsComplete(settings.DindTLSCerts) { + return settings.DindTLSCerts, nil + } + if settings.DindTLSCerts != nil { + return nil, fmt.Errorf("dind TLS certs are incomplete") + } } if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { return nil, err @@ -77,15 +82,12 @@ func EnsureDindTLSCerts() (*commonmodels.DindTLSCerts, error) { if err != nil { return nil, err } - created, err := settingColl.InitDindTLSCertsIfNeeded(certs) + _, err = settingColl.InitDindTLSCertsIfNeeded(certs) if err != nil { return nil, err } - if created { - return certs, nil - } - settings, err = settingColl.Get() + settings, err = settingColl.GetByID() if err != nil { return nil, err }