diff --git a/api/go.mod b/api/go.mod index 52357973..1cab220a 100644 --- a/api/go.mod +++ b/api/go.mod @@ -94,3 +94,7 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/lmiccini/lib-common/modules/common v0.0.0-20260624115944-5ec800107a98 + +replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 diff --git a/api/go.sum b/api/go.sum index 83586d9b..a7929332 100644 --- a/api/go.sum +++ b/api/go.sum @@ -64,6 +64,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 h1:CiQFCVrdzGgeEQqt3C39e6s9Vpi1784HIm+LNYm9py4= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260624115944-5ec800107a98 h1:OZx/6wxHDSIIcV0V1Csa+AEJxO/YB+htqUZO6j63uIU= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260624115944-5ec800107a98/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -78,10 +82,6 @@ github.com/onsi/ginkgo/v2 v2.28.2 h1:DTrMfpqxiNUyQ3Y0zhn1n3cOO2euFgQPYIpkWwxVFps github.com/onsi/ginkgo/v2 v2.28.2/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36 h1:nGpBRRuWJbxiH9Vv5ir0TUWmL3XFChvqvXX8We5Lvnc= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:aIc5ECO3dubv265jjUZ66oi56kf5iUt8Y1DWmCPrzOc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:3cyU3HUhCoV7vscqea6ZUbkwxNSAJd1Rwk0P15vsUZw= github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:x9v3qtFDuv8mitRPA6/dTPftl4GODnM4O9242mCoDoA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/go.mod b/go.mod index c11c2e9d..2fac6d83 100644 --- a/go.mod +++ b/go.mod @@ -145,3 +145,9 @@ replace k8s.io/component-base => k8s.io/component-base v0.31.14 //allow-merging replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20250929174222-a0d328fa4dec //allow-merging replace k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20250627150254-e9823e99808e //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/lmiccini/lib-common/modules/common v0.0.0-20260624115944-5ec800107a98 + +replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 + +replace github.com/openstack-k8s-operators/keystone-operator/api => github.com/lmiccini/keystone-operator/api v0.0.0-20260620073840-e242ef5877eb diff --git a/go.sum b/go.sum index 0d9add63..239f36ae 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462 h1:CiQFCVrdzGgeEQqt3C39e6s9Vpi1784HIm+LNYm9py4= +github.com/lmiccini/infra-operator/apis v0.0.0-20260623100659-aca54b995462/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= +github.com/lmiccini/keystone-operator/api v0.0.0-20260620073840-e242ef5877eb h1:7ovQLyFPUfYVyHpgFuvK9WrFnndAVlUHIbTpD5rNeUU= +github.com/lmiccini/keystone-operator/api v0.0.0-20260620073840-e242ef5877eb/go.mod h1:RURtCihykyyH1VBNEZDleD1DseForvb8J6AlmG0WX7k= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260624115944-5ec800107a98 h1:OZx/6wxHDSIIcV0V1Csa+AEJxO/YB+htqUZO6j63uIU= +github.com/lmiccini/lib-common/modules/common v0.0.0-20260624115944-5ec800107a98/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -118,12 +124,6 @@ github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyUt0GEdoAE+r5TXy7YS21yNEo+2U= github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36 h1:nGpBRRuWJbxiH9Vv5ir0TUWmL3XFChvqvXX8We5Lvnc= -github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260618172644-5a4764bdaa36/go.mod h1:fcTuxQ/hzNBPxCf99vbsBt7dgZ3W12gUthaCXSvkPr8= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260618171050-eb14893f440c h1:HLU/y6d4a26JkEOoZF+m7aUqZRWojgx0UY9UBPL4At8= -github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20260618171050-eb14893f440c/go.mod h1:AZ+ScPRFjfxxJ+A8j8Ukp2RVSo5N/IYzf6kCaMWro8w= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:aIc5ECO3dubv265jjUZ66oi56kf5iUt8Y1DWmCPrzOc= -github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:oeIagnkOxEsxluKFcFMW80Lf1rXdV7FT2W+peB6kSE0= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:OVFoNXzinsI0rq8gbegu8TnlDPkO409iyVoWhU4nEdQ= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20260618132757-fe8e60d1d8a6/go.mod h1:7yqbVpg0k0vW+kZks+TMU/cd1ovoejyHfVPWcyGYLHI= github.com/openstack-k8s-operators/lib-common/modules/storage v0.6.1-0.20260618132757-fe8e60d1d8a6 h1:3cyU3HUhCoV7vscqea6ZUbkwxNSAJd1Rwk0P15vsUZw= diff --git a/internal/cinder/const.go b/internal/cinder/const.go index 64be7b3d..c58c1d35 100644 --- a/internal/cinder/const.go +++ b/internal/cinder/const.go @@ -79,6 +79,11 @@ const ( // ACConsumerFinalizer is added to AC secrets that cinder is actively consuming ACConsumerFinalizer = "openstack.org/cinder-ac-consumer" + // TransportConsumerFinalizer is added to transport URL secrets that cinder + // is actively consuming, preventing the infra-operator from cleaning up + // old RabbitMQ users before cinder pods have rolled out new credentials + TransportConsumerFinalizer = "openstack.org/cinder-transport-consumer" + // ShortDuration is a short duration for quick retries ShortDuration = time.Duration(5) * time.Second // NormalDuration is the normal duration for standard retries diff --git a/internal/controller/cinder_controller.go b/internal/controller/cinder_controller.go index d4516e5f..2a4251a0 100644 --- a/internal/controller/cinder_controller.go +++ b/internal/controller/cinder_controller.go @@ -52,6 +52,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/job" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/object" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -448,12 +449,32 @@ func (r *CinderReconciler) reconcileDelete(ctx context.Context, instance *cinder instance.Status.ApplicationCredentialSecret, instance.Spec.Auth.ApplicationCredentialSecret, } { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, secretName, cinder.ACConsumerFinalizer); err != nil { return ctrl.Result{}, err } } + transportSecrets := []string{instance.Status.TransportURLSecret} + for _, tuName := range []string{ + fmt.Sprintf("%s-cinder-transport", instance.Name), + fmt.Sprintf("%s-cinder-notifications-transport", instance.Name), + } { + tu := &rabbitmqv1.TransportURL{} + if err := r.Get(ctx, types.NamespacedName{Name: tuName, Namespace: instance.Namespace}, tu); err == nil { + transportSecrets = append(transportSecrets, tu.Status.SecretName) + } + } + if instance.Status.NotificationsURLSecret != nil { + transportSecrets = append(transportSecrets, *instance.Status.NotificationsURLSecret) + } + for _, secretName := range transportSecrets { + if err := object.RemoveSecretConsumerFinalizer(ctx, helper, instance.Namespace, + secretName, cinder.TransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + } + // Service is deleted so remove the finalizer. controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) @@ -611,9 +632,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) } - instance.Status.TransportURLSecret = transportURL.Status.SecretName - - if instance.Status.TransportURLSecret == "" { + if transportURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.RabbitMqTransportURLReadyCondition, @@ -625,6 +644,19 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + // Set status early for first-time setup so PatchInstance persists it + // even on early returns. During rotation (old != current), the status + // is only updated by FinalizeSecretRotation at end of reconcile. + if instance.Status.TransportURLSecret == "" || + instance.Status.TransportURLSecret == transportURL.Status.SecretName { + instance.Status.TransportURLSecret = transportURL.Status.SecretName + } + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + transportURL.Status.SecretName, cinder.TransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + // end transportURL // @@ -634,15 +666,18 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder // Determine if notifications are enabled by checking NotificationsBus.Cluster // (the webhook defaults this from the deprecated NotificationsBusInstance field) + var notificationBusInstanceURL *rabbitmqv1.TransportURL if instance.Spec.NotificationsBus != nil && instance.Spec.NotificationsBus.Cluster != "" { - // init .Status.NotificationURLSecret - instance.Status.NotificationsURLSecret = ptr.To("") + if instance.Status.NotificationsURLSecret == nil { + instance.Status.NotificationsURLSecret = ptr.To("") + } // Use NotificationsBus config (never fall back to MessagingBus to ensure separation) notificationsRabbitMqConfig := *instance.Spec.NotificationsBus // A separate TransportURL is always created for notifications, // even when using the same cluster as messaging (to allow different vhost/user) - notificationBusInstanceURL, op, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels, true, notificationsRabbitMqConfig) + var op controllerutil.OperationResult + notificationBusInstanceURL, op, err = r.transportURLCreateOrUpdate(ctx, instance, serviceLabels, true, notificationsRabbitMqConfig) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -657,9 +692,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder Log.Info(fmt.Sprintf("NotificationBusInstanceURL %s successfully reconciled - operation: %s", notificationBusInstanceURL.Name, string(op))) } - *instance.Status.NotificationsURLSecret = notificationBusInstanceURL.Status.SecretName - - if instance.Status.NotificationsURLSecret == nil { + if notificationBusInstanceURL.Status.SecretName == "" { Log.Info(fmt.Sprintf("Waiting for NotificationBusInstanceURL %s secret to be created", notificationBusInstanceURL.Name)) instance.Status.Conditions.Set(condition.FalseCondition( condition.NotificationBusInstanceReadyCondition, @@ -669,6 +702,19 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder return cinder.ResultRequeue, nil } + // Set status early for first-time setup so PatchInstance persists it + // even on early returns. During rotation (old != current), the status + // is only updated by FinalizeSecretRotation at end of reconcile. + if instance.Status.NotificationsURLSecret == nil || + *instance.Status.NotificationsURLSecret == notificationBusInstanceURL.Status.SecretName { + instance.Status.NotificationsURLSecret = ptr.To(notificationBusInstanceURL.Status.SecretName) + } + + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, + notificationBusInstanceURL.Status.SecretName, cinder.TransportConsumerFinalizer); err != nil { + return ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue(condition.NotificationBusInstanceReadyCondition, condition.NotificationBusInstanceReadyMessage) } else { // make sure we do not have an entry in the status if @@ -755,7 +801,11 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder // // Create Secrets required as input for the Service and calculate an overall hash of hashes // - err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db) + notificationsURLSecretName := "" + if notificationBusInstanceURL != nil { + notificationsURLSecretName = notificationBusInstanceURL.Status.SecretName + } + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db, transportURL.Status.SecretName, notificationsURLSecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -795,9 +845,8 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder // The old secret's finalizer is removed later (after all services deploy) // so that rapid rotations don't revoke a credential still in use by pods. if instance.Spec.Auth.ApplicationCredentialSecret != "" { - if err := keystonev1.ManageACSecretFinalizer(ctx, helper, instance.Namespace, + if err := object.ManageSecretConsumerFinalizer(ctx, helper, instance.Namespace, instance.Spec.Auth.ApplicationCredentialSecret, - "", cinder.ACConsumerFinalizer); err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -868,7 +917,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder // // deploy cinder-api - cinderAPI, op, err := r.apiDeploymentCreateOrUpdate(ctx, instance) + cinderAPI, op, err := r.apiDeploymentCreateOrUpdate(ctx, instance, transportURL.Status.SecretName, notificationsURLSecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( cinderv1beta1.CinderAPIReadyCondition, @@ -897,7 +946,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder } // deploy cinder-scheduler - cinderScheduler, op, err := r.schedulerDeploymentCreateOrUpdate(ctx, instance) + cinderScheduler, op, err := r.schedulerDeploymentCreateOrUpdate(ctx, instance, transportURL.Status.SecretName, notificationsURLSecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( cinderv1beta1.CinderSchedulerReadyCondition, @@ -935,7 +984,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder // Many OpenStack deployments don't use the cinder-backup service (it's optional), // so there's no need to deploy it unless it's required. if *instance.Spec.CinderBackup.Replicas > 0 && instance.Spec.CinderBackups == nil { - cinderBackup, op, err := r.backupDeploymentCreateOrUpdate(ctx, instance, crName, nil) + cinderBackup, op, err := r.backupDeploymentCreateOrUpdate(ctx, instance, crName, nil, transportURL.Status.SecretName, notificationsURLSecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( cinderv1beta1.CinderBackupReadyCondition, @@ -981,7 +1030,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder backup := (*instance.Spec.CinderBackups)[name] crName := fmt.Sprintf("%s-backup-%s", instance.Name, name) if *backup.Replicas > 0 { - cinderBackup, op, err := r.backupDeploymentCreateOrUpdate(ctx, instance, crName, &backup) + cinderBackup, op, err := r.backupDeploymentCreateOrUpdate(ctx, instance, crName, &backup, transportURL.Status.SecretName, notificationsURLSecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( cinderv1beta1.CinderBackupReadyCondition, @@ -1035,7 +1084,7 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder waitingGenerationMatch := false for _, name := range slices.Sorted(maps.Keys(instance.Spec.CinderVolumes)) { volume := instance.Spec.CinderVolumes[name] - cinderVolume, op, err := r.volumeDeploymentCreateOrUpdate(ctx, instance, name, volume) + cinderVolume, op, err := r.volumeDeploymentCreateOrUpdate(ctx, instance, name, volume, transportURL.Status.SecretName, notificationsURLSecretName) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( cinderv1beta1.CinderVolumeReadyCondition, @@ -1114,25 +1163,67 @@ func (r *CinderReconciler) reconcileNormal(ctx context.Context, instance *cinder Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) - // Manage the old AC secret's finalizer and status tracking. - // On rotation (old != new), only remove the old secret's finalizer after - // all sub-services are ready with the new credentials. This prevents - // premature revocation during rapid rotations. - isRotation := instance.Status.ApplicationCredentialSecret != "" && instance.Status.ApplicationCredentialSecret != instance.Spec.Auth.ApplicationCredentialSecret - - if isRotation { - allServicesReady := instance.Status.Conditions.AllSubConditionIsTrue() - if allServicesReady { - if err := keystonev1.RemoveACSecretConsumerFinalizer(ctx, helper, instance.Namespace, - instance.Status.ApplicationCredentialSecret, cinder.ACConsumerFinalizer); err != nil { - return ctrl.Result{}, err - } - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + rotationPending := instance.Status.TransportURLSecret != "" && + instance.Status.TransportURLSecret != transportURL.Status.SecretName + if notificationBusInstanceURL != nil && instance.Status.NotificationsURLSecret != nil { + rotationPending = rotationPending || + (*instance.Status.NotificationsURLSecret != "" && + *instance.Status.NotificationsURLSecret != notificationBusInstanceURL.Status.SecretName) + } + + result, graceActive, err := object.ManageRotationGracePeriod( + ctx, r.Client, instance, rotationPending, 60*time.Second) + if err != nil { + return ctrl.Result{}, err + } + if graceActive { + return result, nil + } + + guardReady := condition.CredentialRotationGuardReady(true, &instance.Status.Conditions) + + transportSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.TransportURLSecret, + transportURL.Status.SecretName, + cinder.TransportConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.TransportURLSecret = transportSecretName + + if notificationBusInstanceURL != nil { + notifStatusSecret := "" + if instance.Status.NotificationsURLSecret != nil { + notifStatusSecret = *instance.Status.NotificationsURLSecret } - } else { - instance.Status.ApplicationCredentialSecret = instance.Spec.Auth.ApplicationCredentialSecret + notifSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + notifStatusSecret, + notificationBusInstanceURL.Status.SecretName, + cinder.TransportConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.NotificationsURLSecret = ptr.To(notifSecretName) } + acSecretName, err := object.FinalizeSecretRotation( + ctx, helper, instance.Namespace, + instance.Status.ApplicationCredentialSecret, + instance.Spec.Auth.ApplicationCredentialSecret, + cinder.ACConsumerFinalizer, + guardReady, + ) + if err != nil { + return ctrl.Result{}, err + } + instance.Status.ApplicationCredentialSecret = acSecretName + // update the overall status condition if service is ready if instance.IsReady() { instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) @@ -1149,6 +1240,8 @@ func (r *CinderReconciler) generateServiceConfigs( serviceLabels map[string]string, memcached *memcachedv1.Memcached, db *mariadbv1.Database, + transportURLSecretName string, + notificationsURLSecretName string, ) error { // // create Secret required for cinder input @@ -1187,7 +1280,7 @@ func (r *CinderReconciler) generateServiceConfigs( return err } - transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + transportURLSecret, _, err := secret.GetSecret(ctx, h, transportURLSecretName, instance.Namespace) if err != nil { return err } @@ -1260,10 +1353,10 @@ func (r *CinderReconciler) generateServiceConfigs( } var notificationInstanceURLSecret *corev1.Secret - if instance.Status.NotificationsURLSecret != nil { + if notificationsURLSecretName != "" { // A separate TransportURL is always created for notifications (even when using the same cluster) // to allow different vhost/user configuration for isolation, so always use the dedicated secret - notificationInstanceURLSecret, _, err = secret.GetSecret(ctx, h, *instance.Status.NotificationsURLSecret, instance.Namespace) + notificationInstanceURLSecret, _, err = secret.GetSecret(ctx, h, notificationsURLSecretName, instance.Namespace) if err != nil { return err } @@ -1359,13 +1452,13 @@ func (r *CinderReconciler) transportURLCreateOrUpdate( return transportURL, op, err } -func (r *CinderReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder) (*cinderv1beta1.CinderAPI, controllerutil.OperationResult, error) { +func (r *CinderReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder, transportURLSecretName string, notificationsURLSecretName string) (*cinderv1beta1.CinderAPI, controllerutil.OperationResult, error) { cinderAPISpec := cinderv1beta1.CinderAPISpec{ CinderTemplate: instance.Spec.CinderTemplate, CinderAPITemplate: instance.Spec.CinderAPI, ExtraMounts: instance.Spec.ExtraMounts, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), MemcachedInstance: &instance.Spec.MemcachedInstance, APITimeout: instance.Spec.APITimeout, @@ -1391,8 +1484,8 @@ func (r *CinderReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cinderAPISpec - if instance.Spec.NotificationsBus != nil && instance.Spec.NotificationsBus.Cluster != "" { - deployment.Spec.NotificationsURLSecret = *instance.Status.NotificationsURLSecret + if notificationsURLSecretName != "" { + deployment.Spec.NotificationsURLSecret = notificationsURLSecretName } err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) @@ -1406,13 +1499,13 @@ func (r *CinderReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst return deployment, op, err } -func (r *CinderReconciler) schedulerDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder) (*cinderv1beta1.CinderScheduler, controllerutil.OperationResult, error) { +func (r *CinderReconciler) schedulerDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder, transportURLSecretName string, notificationsURLSecretName string) (*cinderv1beta1.CinderScheduler, controllerutil.OperationResult, error) { cinderSchedulerSpec := cinderv1beta1.CinderSchedulerSpec{ CinderTemplate: instance.Spec.CinderTemplate, CinderSchedulerTemplate: instance.Spec.CinderScheduler, ExtraMounts: instance.Spec.ExtraMounts, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), TLS: instance.Spec.CinderAPI.TLS.Ca, MemcachedInstance: &instance.Spec.MemcachedInstance, @@ -1438,8 +1531,8 @@ func (r *CinderReconciler) schedulerDeploymentCreateOrUpdate(ctx context.Context op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cinderSchedulerSpec - if instance.Spec.NotificationsBus != nil && instance.Spec.NotificationsBus.Cluster != "" { - deployment.Spec.NotificationsURLSecret = *instance.Status.NotificationsURLSecret + if notificationsURLSecretName != "" { + deployment.Spec.NotificationsURLSecret = notificationsURLSecretName } err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) @@ -1458,13 +1551,15 @@ func (r *CinderReconciler) backupDeploymentCreateOrUpdate( instance *cinderv1beta1.Cinder, name string, bkpTemplate *cinderv1beta1.CinderBackupTemplate, + transportURLSecretName string, + notificationsURLSecretName string, ) (*cinderv1beta1.CinderBackup, controllerutil.OperationResult, error) { cinderBackupSpec := cinderv1beta1.CinderBackupSpec{ CinderTemplate: instance.Spec.CinderTemplate, ExtraMounts: instance.Spec.ExtraMounts, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), TLS: instance.Spec.CinderAPI.TLS.Ca, MemcachedInstance: &instance.Spec.MemcachedInstance, @@ -1490,8 +1585,8 @@ func (r *CinderReconciler) backupDeploymentCreateOrUpdate( op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cinderBackupSpec - if instance.Spec.NotificationsBus != nil && instance.Spec.NotificationsBus.Cluster != "" { - deployment.Spec.NotificationsURLSecret = *instance.Status.NotificationsURLSecret + if notificationsURLSecretName != "" { + deployment.Spec.NotificationsURLSecret = notificationsURLSecretName } err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) @@ -1537,13 +1632,13 @@ func (r *CinderReconciler) backupCleanupDeployment( return nil } -func (r *CinderReconciler) volumeDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder, name string, volTemplate cinderv1beta1.CinderVolumeTemplate) (*cinderv1beta1.CinderVolume, controllerutil.OperationResult, error) { +func (r *CinderReconciler) volumeDeploymentCreateOrUpdate(ctx context.Context, instance *cinderv1beta1.Cinder, name string, volTemplate cinderv1beta1.CinderVolumeTemplate, transportURLSecretName string, notificationsURLSecretName string) (*cinderv1beta1.CinderVolume, controllerutil.OperationResult, error) { cinderVolumeSpec := cinderv1beta1.CinderVolumeSpec{ CinderTemplate: instance.Spec.CinderTemplate, CinderVolumeTemplate: volTemplate, ExtraMounts: instance.Spec.ExtraMounts, DatabaseHostname: instance.Status.DatabaseHostname, - TransportURLSecret: instance.Status.TransportURLSecret, + TransportURLSecret: transportURLSecretName, ServiceAccount: instance.RbacResourceName(), TLS: instance.Spec.CinderAPI.TLS.Ca, MemcachedInstance: &instance.Spec.MemcachedInstance, @@ -1568,8 +1663,8 @@ func (r *CinderReconciler) volumeDeploymentCreateOrUpdate(ctx context.Context, i op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { deployment.Spec = cinderVolumeSpec - if instance.Spec.NotificationsBus != nil && instance.Spec.NotificationsBus.Cluster != "" { - deployment.Spec.NotificationsURLSecret = *instance.Status.NotificationsURLSecret + if notificationsURLSecretName != "" { + deployment.Spec.NotificationsURLSecret = notificationsURLSecretName } err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) diff --git a/internal/controller/cinderapi_controller.go b/internal/controller/cinderapi_controller.go index 560e1587..475fe3b5 100644 --- a/internal/controller/cinderapi_controller.go +++ b/internal/controller/cinderapi_controller.go @@ -1024,7 +1024,7 @@ func (r *CinderAPIReconciler) reconcileNormal(ctx context.Context, instance *cin return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + if statefulset.IsReady(ssData) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else if *instance.Spec.Replicas > 0 { diff --git a/internal/controller/cinderbackup_controller.go b/internal/controller/cinderbackup_controller.go index 798d02d4..748f98fc 100644 --- a/internal/controller/cinderbackup_controller.go +++ b/internal/controller/cinderbackup_controller.go @@ -679,7 +679,7 @@ func (r *CinderBackupReconciler) reconcileNormal(ctx context.Context, instance * return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + if statefulset.IsReady(ssData) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else if *instance.Spec.Replicas > 0 { instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/internal/controller/cinderscheduler_controller.go b/internal/controller/cinderscheduler_controller.go index b3dac321..4866ab8b 100644 --- a/internal/controller/cinderscheduler_controller.go +++ b/internal/controller/cinderscheduler_controller.go @@ -677,7 +677,7 @@ func (r *CinderSchedulerReconciler) reconcileNormal(ctx context.Context, instanc return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + if statefulset.IsReady(ssData) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else if *instance.Spec.Replicas > 0 { instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/internal/controller/cindervolume_controller.go b/internal/controller/cindervolume_controller.go index 4b7115ac..720d8afc 100644 --- a/internal/controller/cindervolume_controller.go +++ b/internal/controller/cindervolume_controller.go @@ -681,7 +681,7 @@ func (r *CinderVolumeReconciler) reconcileNormal(ctx context.Context, instance * return ctrl.Result{}, err } - if instance.Status.ReadyCount > 0 { + if statefulset.IsReady(ssData) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else if *instance.Spec.Replicas > 0 { instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/test/functional/cinder_controller_test.go b/test/functional/cinder_controller_test.go index 9322fb90..431744f2 100644 --- a/test/functional/cinder_controller_test.go +++ b/test/functional/cinder_controller_test.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "os" + "time" . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports . "github.com/onsi/gomega" //revive:disable:dot-imports @@ -38,7 +39,6 @@ import ( cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" "github.com/openstack-k8s-operators/cinder-operator/internal/cinder" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" - rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" @@ -2430,6 +2430,291 @@ var _ = Describe("Cinder Webhook", func() { }, timeout, interval).Should(Succeed()) }) }) + + When("TransportURL consumer finalizer is managed", func() { + BeforeEach(func() { + DeferCleanup(k8sClient.Delete, ctx, + CreateCinderMessageBusSecret( + cinderTest.Instance.Namespace, + cinderTest.RabbitmqSecretName, + ), + ) + spec := GetDefaultCinderSpec() + delete(spec, "cinderVolume") + spec["cinderVolumes"] = map[string]any{ + "volume1": map[string]any{}, + } + backupSpec := GetDefaultCinderBackupSpec() + backupSpec["replicas"] = 0 + spec["cinderBackup"] = backupSpec + DeferCleanup(th.DeleteInstance, CreateCinder(cinderTest.Instance, spec)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cinderTest.Instance.Namespace, + GetCinder(cinderTest.Instance).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}})) + + acc, accSecret := mariadb.CreateMariaDBAccountAndSecret(cinderTest.Database, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, acc) + DeferCleanup(k8sClient.Delete, ctx, accSecret) + mariadb.CreateMariaDBDatabase(cinderTest.Database.Namespace, cinderTest.Database.Name, mariadbv1.MariaDBDatabaseSpec{}) + DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(cinderTest.Database)) + + DeferCleanup(keystone.DeleteKeystoneAPI, + keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) + + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(cinderTest.Instance.Namespace, MemcachedInstance, memcachedv1.MemcachedSpec{})) + infra.SimulateMemcachedReady(cinderTest.CinderMemcached) + + infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) + mariadb.SimulateMariaDBAccountCompleted(cinderTest.Database) + mariadb.SimulateMariaDBDatabaseCompleted(cinderTest.Database) + th.SimulateJobSuccess(cinderTest.CinderDBSync) + th.SimulateJobSuccess(cinderTest.CinderOnlineDataMigration) + keystone.SimulateKeystoneServiceReady(cinderTest.CinderKeystoneService) + keystone.SimulateKeystoneEndpointReady(cinderTest.CinderKeystoneEndpoint) + }) + + It("should add the consumer finalizer to the transport secret", func() { + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: cinderTest.RabbitmqSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + }) + + It("should remove the consumer finalizer from transport secret on CR deletion", func() { + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: cinderTest.RabbitmqSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + th.DeleteInstance(GetCinder(cinderTest.Instance)) + + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: cinderTest.RabbitmqSecretName, + }) + g.Expect(secret.Finalizers).NotTo( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + }) + + It("should move the finalizer from the old to the new secret on transport rotation", func() { + oldSecretName := cinderTest.RabbitmqSecretName + newSecretName := "rabbitmq-secret-rotated" + + // Wait for the consumer finalizer to be added to the old secret + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Get cinder to fully ready state (jobs already simulated in BeforeEach) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderAPI) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderScheduler) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderVolumes[0]) + Eventually(func(g Gomega) { + c := GetCinder(cinderTest.Instance) + g.Expect(c.Status.Conditions.IsTrue(condition.ReadyCondition)).To(BeTrue()) + g.Expect(c.Status.TransportURLSecret).To(Equal(oldSecretName)) + }, timeout, interval).Should(Succeed()) + + // Create the new rotated secret with DIFFERENT content + newSecret := th.CreateSecret( + types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: newSecretName, + }, + map[string][]byte{ + "transport_url": []byte("rabbit://rotated-user:rotated-pass@rabbitmq/fake"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, newSecret) + + // Simulate transport rotation: update TransportURL status with new secret name + Eventually(func(g Gomega) { + transport := infra.GetTransportURL(cinderTest.CinderTransportURL) + transport.Status.SecretName = newSecretName + g.Expect(k8sClient.Status().Update(ctx, transport)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Verify finalizer is added to the new secret + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: newSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // The old secret's finalizer should NOT be removed yet — sub-CRs + // are re-deploying with new credentials and are not ready + Consistently(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Simulate sub-CRs becoming ready with the new credentials by + // directly updating their status. We bypass the sub-CR controllers + // because envtest has no real StatefulSet controller, causing the + // sub-CR controllers to enter exponential backoff. + Eventually(func(g Gomega) { + for _, name := range []types.NamespacedName{ + cinderTest.CinderAPI, + cinderTest.CinderScheduler, + cinderTest.CinderVolumes[0], + } { + th.SimulateStatefulSetReplicaReady(name) + } + // Also trigger a parent reconcile by touching the Cinder CR + c := GetCinder(cinderTest.Instance) + if c.Annotations == nil { + c.Annotations = map[string]string{} + } + c.Annotations["test-reconcile-trigger"] = fmt.Sprintf("%d", time.Now().UnixNano()) + g.Expect(k8sClient.Update(ctx, c)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Fast-forward the rotation grace period for envtest + Eventually(func(g Gomega) { + c := GetCinder(cinderTest.Instance) + if c.Annotations != nil && c.Annotations["openstack.org/rotation-grace-until"] != "" { + c.Annotations["openstack.org/rotation-grace-until"] = time.Now().Add(-5 * time.Second).Format(time.RFC3339) + g.Expect(k8sClient.Update(ctx, c)).To(Succeed()) + } + }, timeout, interval).Should(Succeed()) + + // Verify old finalizer is removed and status updated + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).NotTo( + ContainElement(cinder.TransportConsumerFinalizer)) + c := GetCinder(cinderTest.Instance) + g.Expect(c.Status.TransportURLSecret).To(Equal(newSecretName)) + }, 10*time.Second, interval).Should(Succeed()) + }) + + It("should hold the finalizer until the last sub-CR is ready", func() { + oldSecretName := cinderTest.RabbitmqSecretName + newSecretName := "rabbitmq-secret-rotated" + + // Get cinder to fully ready state + th.SimulateStatefulSetReplicaReady(cinderTest.CinderAPI) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderScheduler) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderVolumes[0]) + Eventually(func(g Gomega) { + c := GetCinder(cinderTest.Instance) + g.Expect(c.Status.Conditions.IsTrue(condition.ReadyCondition)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + + // Create the new rotated secret + newSecret := th.CreateSecret( + types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: newSecretName, + }, + map[string][]byte{ + "transport_url": []byte("rabbit://rotated-user:rotated-pass@rabbitmq/fake"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, newSecret) + + // Trigger rotation + Eventually(func(g Gomega) { + transport := infra.GetTransportURL(cinderTest.CinderTransportURL) + transport.Status.SecretName = newSecretName + g.Expect(k8sClient.Status().Update(ctx, transport)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Wait for new secret to get the finalizer + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: newSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Simulate only CinderAPI and CinderScheduler ready, but NOT CinderVolume. + // The finalizer on the old secret MUST be held. + Eventually(func(g Gomega) { + th.SimulateStatefulSetReplicaReady(cinderTest.CinderAPI) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderScheduler) + c := GetCinder(cinderTest.Instance) + if c.Annotations == nil { + c.Annotations = map[string]string{} + } + c.Annotations["test-reconcile-trigger"] = fmt.Sprintf("%d", time.Now().UnixNano()) + g.Expect(k8sClient.Update(ctx, c)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Verify old secret's finalizer is still held (CinderVolume not ready) + Consistently(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).To( + ContainElement(cinder.TransportConsumerFinalizer)) + }, timeout, interval).Should(Succeed()) + + // Now simulate all sub-CRs ready including CinderVolume + Eventually(func(g Gomega) { + th.SimulateStatefulSetReplicaReady(cinderTest.CinderAPI) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderScheduler) + th.SimulateStatefulSetReplicaReady(cinderTest.CinderVolumes[0]) + c := GetCinder(cinderTest.Instance) + if c.Annotations == nil { + c.Annotations = map[string]string{} + } + c.Annotations["test-reconcile-trigger"] = fmt.Sprintf("%d", time.Now().UnixNano()) + g.Expect(k8sClient.Update(ctx, c)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + // Fast-forward the rotation grace period for envtest + Eventually(func(g Gomega) { + c := GetCinder(cinderTest.Instance) + if c.Annotations != nil && c.Annotations["openstack.org/rotation-grace-until"] != "" { + c.Annotations["openstack.org/rotation-grace-until"] = time.Now().Add(-5 * time.Second).Format(time.RFC3339) + g.Expect(k8sClient.Update(ctx, c)).To(Succeed()) + } + }, timeout, interval).Should(Succeed()) + + // Now the finalizer should be released + Eventually(func(g Gomega) { + secret := th.GetSecret(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: oldSecretName, + }) + g.Expect(secret.Finalizers).NotTo( + ContainElement(cinder.TransportConsumerFinalizer)) + }, 10*time.Second, interval).Should(Succeed()) + }) + }) }) var _ = Describe("Cinder with RabbitMQ custom vhost and user", func() { @@ -2661,6 +2946,10 @@ var _ = Describe("Cinder with RabbitMQ custom vhost and user", func() { ), ) infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) + infra.SimulateTransportURLReady(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: fmt.Sprintf("%s-cinder-notifications-transport", cinderTest.Instance.Name), + }) DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, cinderTest.MemcachedInstance, memcachedSpec)) infra.SimulateMemcachedReady(cinderTest.CinderMemcached) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) @@ -2713,6 +3002,10 @@ var _ = Describe("Cinder with RabbitMQ custom vhost and user", func() { ), ) infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) + infra.SimulateTransportURLReady(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: fmt.Sprintf("%s-cinder-notifications-transport", cinderTest.Instance.Name), + }) DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, cinderTest.MemcachedInstance, memcachedSpec)) infra.SimulateMemcachedReady(cinderTest.CinderMemcached) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) @@ -2770,6 +3063,10 @@ var _ = Describe("Cinder with RabbitMQ custom vhost and user", func() { ), ) infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) + infra.SimulateTransportURLReady(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: fmt.Sprintf("%s-cinder-notifications-transport", cinderTest.Instance.Name), + }) DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, cinderTest.MemcachedInstance, memcachedSpec)) infra.SimulateMemcachedReady(cinderTest.CinderMemcached) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) @@ -2818,14 +3115,31 @@ var _ = Describe("Cinder with RabbitMQ custom vhost and user", func() { When("Cinder notifications are disabled after creation", func() { BeforeEach(func() { + DeferCleanup(k8sClient.Delete, ctx, + CreateCinderMessageBusSecret( + cinderTest.Instance.Namespace, + cinderTest.RabbitmqSecretName, + ), + ) + DeferCleanup(k8sClient.Delete, ctx, + CreateCinderMessageBusSecret( + cinderTest.Instance.Namespace, + "rabbitmq-notifications-secret", + ), + ) spec := GetDefaultCinderSpec() // Start with notifications enabled spec["notificationsBus"] = map[string]any{ "cluster": "rabbitmq-notifications", } + delete(spec, "cinderVolume") + spec["cinderVolumes"] = map[string]any{ + "volume1": map[string]any{}, + } + backupSpec := GetDefaultCinderBackupSpec() + backupSpec["replicas"] = 0 + spec["cinderBackup"] = backupSpec DeferCleanup(th.DeleteInstance, CreateCinder(cinderTest.Instance, spec)) - DeferCleanup(k8sClient.Delete, ctx, CreateCinderMessageBusSecret(cinderTest.Instance.Namespace, cinderTest.RabbitmqSecretName)) - DeferCleanup(k8sClient.Delete, ctx, CreateCinderMessageBusSecret(cinderTest.Instance.Namespace, "rabbitmq-notifications-secret")) DeferCleanup( mariadb.DeleteDBService, mariadb.CreateDBService( @@ -2836,32 +3150,38 @@ var _ = Describe("Cinder with RabbitMQ custom vhost and user", func() { }, ), ) - infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) + + acc, accSecret := mariadb.CreateMariaDBAccountAndSecret(cinderTest.Database, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, acc) + DeferCleanup(k8sClient.Delete, ctx, accSecret) + mariadb.CreateMariaDBDatabase(cinderTest.Database.Namespace, cinderTest.Database.Name, mariadbv1.MariaDBDatabaseSpec{}) + DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(cinderTest.Database)) + + DeferCleanup(keystone.DeleteKeystoneAPI, + keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, cinderTest.MemcachedInstance, memcachedSpec)) infra.SimulateMemcachedReady(cinderTest.CinderMemcached) - DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(cinderTest.Instance.Namespace)) + + infra.SimulateTransportURLReady(cinderTest.CinderTransportURL) + infra.SimulateTransportURLReady(types.NamespacedName{ + Namespace: cinderTest.Instance.Namespace, + Name: fmt.Sprintf("%s-cinder-notifications-transport", cinderTest.Instance.Name), + }) mariadb.SimulateMariaDBAccountCompleted(cinderTest.Database) mariadb.SimulateMariaDBDatabaseCompleted(cinderTest.Database) + th.SimulateJobSuccess(cinderTest.CinderDBSync) + th.SimulateJobSuccess(cinderTest.CinderOnlineDataMigration) + keystone.SimulateKeystoneServiceReady(cinderTest.CinderKeystoneService) + keystone.SimulateKeystoneEndpointReady(cinderTest.CinderKeystoneEndpoint) }) It("should remove NotificationsURLSecret when notificationsBus is set to empty", func() { - // Wait for the notifications TransportURL to be created and simulate it being ready - notificationsTransportURLName := types.NamespacedName{ - Namespace: cinderTest.Instance.Namespace, - Name: fmt.Sprintf("%s-cinder-notifications-transport", cinderTest.Instance.Name), - } - Eventually(func(g Gomega) { - transportURL := &rabbitmqv1.TransportURL{} - err := k8sClient.Get(ctx, notificationsTransportURLName, transportURL) - g.Expect(err).ToNot(HaveOccurred()) - }, timeout, interval).Should(Succeed()) - infra.SimulateTransportURLReady(notificationsTransportURLName) - // Verify notifications are enabled Eventually(func(g Gomega) { cinder := GetCinder(cinderTest.Instance) g.Expect(cinder.Status.NotificationsURLSecret).ToNot(BeNil()) - g.Expect(*cinder.Status.NotificationsURLSecret).ToNot(Equal(cinder.Status.TransportURLSecret)) + g.Expect(*cinder.Status.NotificationsURLSecret).ToNot(Equal("")) }, timeout, interval).Should(Succeed()) // Now disable notifications by clearing both the deprecated field and the new field