From 28e010f547557ef95f7c982888c56190cdb140b5 Mon Sep 17 00:00:00 2001 From: Arun Maurya Date: Fri, 19 Jun 2026 16:22:29 +0530 Subject: [PATCH 1/2] test(e2e): add multi-operand qualification suite Co-authored-by: Cursor --- test/e2e/issuer_acme_http01_test.go | 6 +- test/e2e/multiple_operands_test.go | 917 ++++++++++++++++++++++++++ test/e2e/trustmanager_helpers_test.go | 7 + test/e2e/trustmanager_test.go | 23 +- test/e2e/utils_test.go | 24 +- test/go.mod | 2 +- 6 files changed, 966 insertions(+), 13 deletions(-) create mode 100644 test/e2e/multiple_operands_test.go diff --git a/test/e2e/issuer_acme_http01_test.go b/test/e2e/issuer_acme_http01_test.go index eac2c54d4..7fb6ac698 100644 --- a/test/e2e/issuer_acme_http01_test.go +++ b/test/e2e/issuer_acme_http01_test.go @@ -265,7 +265,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered } err := wait.PollUntilContextTimeout(ctx, fastPollInterval, lowTimeout, true, func(ctx context.Context) (bool, error) { - pods, err := k8sClientSet.CoreV1().Pods("").List(ctx, metav1.ListOptions{ + pods, err := k8sClientSet.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{ LabelSelector: acmeSolverPodLabel, }) if err != nil { @@ -286,10 +286,6 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered return false, nil // No matching pods yet, keep waiting }) Expect(err).NotTo(HaveOccurred(), "should find ACME HTTP01 solver pods with expected resource configuration") - - By("waiting for certificate to get ready") - err = waitForCertificateReadiness(ctx, secretName, ns.Name) - Expect(err).NotTo(HaveOccurred(), "timeout waiting for certificate to become ready") }) }) diff --git a/test/e2e/multiple_operands_test.go b/test/e2e/multiple_operands_test.go new file mode 100644 index 000000000..bc39e976e --- /dev/null +++ b/test/e2e/multiple_operands_test.go @@ -0,0 +1,917 @@ +//go:build e2e +// +build e2e + +// CM-786 IstioCSR Controller qualification (doc section "IstioCSR Controller specific tests" +// through "Record about cert-manager Operator"): multi-operand install plus sanity and +// qualification steps 5–14 that are automatable in e2e. +package e2e + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "fmt" + "io" + "net/url" + "path/filepath" + "strings" + "sync" + "time" + + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + trustapi "github.com/cert-manager/trust-manager/pkg/apis/trust/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/sync/errgroup" + "sigs.k8s.io/yaml" + + testutils "github.com/openshift/cert-manager-operator/pkg/controller/istiocsr" + "github.com/openshift/cert-manager-operator/api/operator/v1alpha1" + "github.com/openshift/cert-manager-operator/test/library" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" +) + +const ( + istioCSRDeploymentName = "cert-manager-istio-csr" + istioCSRServiceName = "cert-manager-istio-csr" + istioCSRServiceAccountName = "cert-manager-istio-csr" + + bothOperandsFeatureGates = "IstioCSR=true,TrustManager=true" + + cm786QualificationLabelKey = "cm-786-qualification" + cm786QualificationLabelValue = "true" + + cm10TestDNSName = "multi-operand.cm10.test.example" + cm12TestDNSName = "multi-operand.cm12.invalid.example" + + qualificationBundleName = "cm786-ca-bundle-secret" + qualificationBundleSourceCM = "cm786-bundle-source" + qualificationBundleSourceKey = "ca-bundle.crt" + qualificationBundleTargetKey = "ca-bundle.crt" +) + +type operandDeployment struct { + namespace string + name string +} + +var coreOperandDeployments = []operandDeployment{ + {operandNamespace, certmanagerControllerDeployment}, + {operandNamespace, certmanagerWebhookDeployment}, + {operandNamespace, certmanagerCAinjectorDeployment}, + {operandNamespace, trustManagerDeploymentName}, +} + +var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platform:Generic", "Feature:MultipleOperands", "TechPreview", "Skipped:MicroShift"), func() { + var ( + ctx = context.Background() + + clientset *kubernetes.Clientset + originalUnsupportedAddonFeatures string + originalOperatorLogLevel string + istioNS *corev1.Namespace + qualificationBundlePEM string + ) + + BeforeAll(func() { + var err error + clientset, err = kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + if isMicroShiftCluster(ctx, clientset) { + Skip("MicroShift: OLM not available") + } + + trustManagerRemoveStaleOperandServiceAccount(ctx, clientset) + + originalUnsupportedAddonFeatures, err = getSubscriptionEnvVar(ctx, loader, "UNSUPPORTED_ADDON_FEATURES") + Expect(err).NotTo(HaveOccurred()) + + originalOperatorLogLevel, err = getSubscriptionEnvVar(ctx, loader, "OPERATOR_LOG_LEVEL") + Expect(err).NotTo(HaveOccurred()) + + By("CM-786 step 8: enable IstioCSR and TrustManager feature gates") + err = patchSubscriptionWithEnvVars(ctx, loader, map[string]string{ + "UNSUPPORTED_ADDON_FEATURES": bothOperandsFeatureGates, + "OPERATOR_LOG_LEVEL": "4", + }) + Expect(err).NotTo(HaveOccurred()) + + err = waitForDeploymentEnvVarAndRollout(ctx, operatorNamespace, operatorDeploymentName, + "UNSUPPORTED_ADDON_FEATURES", bothOperandsFeatureGates, highTimeout) + Expect(err).NotTo(HaveOccurred()) + + caTweak := func(cert *x509.Certificate) { + cert.IsCA = true + cert.KeyUsage |= x509.KeyUsageCertSign + } + qualificationBundlePEM = testutils.GenerateCertificate("cm786-bundle-ca", []string{"cert-manager-operator-e2e"}, caTweak) + }) + + AfterAll(func() { + expectDeleteClean(trustManagerClient().Delete(ctx, "cluster", metav1.DeleteOptions{}), "TrustManager CR cluster") + deleteTrustManagerDefaultCAPackageConfigMap(ctx) + if istioNS != nil { + expectDeleteClean( + certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNS.Name).Delete(ctx, istioCSRResourceName, metav1.DeleteOptions{}), + fmt.Sprintf("IstioCSR %s/%s", istioNS.Name, istioCSRResourceName), + ) + } + + err := patchSubscriptionWithEnvVars(ctx, loader, map[string]string{ + "UNSUPPORTED_ADDON_FEATURES": originalUnsupportedAddonFeatures, + "OPERATOR_LOG_LEVEL": originalOperatorLogLevel, + }) + Expect(err).NotTo(HaveOccurred()) + + if originalUnsupportedAddonFeatures == "" { + err = waitForDeploymentEnvVarRemovedAndRollout(ctx, operatorNamespace, operatorDeploymentName, "UNSUPPORTED_ADDON_FEATURES", lowTimeout) + } else { + err = waitForDeploymentEnvVarAndRollout(ctx, operatorNamespace, operatorDeploymentName, + "UNSUPPORTED_ADDON_FEATURES", originalUnsupportedAddonFeatures, lowTimeout) + } + Expect(err).NotTo(HaveOccurred()) + }) + + Context("with operands installed concurrently", Ordered, func() { + BeforeAll(func() { + var err error + + istioNS, err = loader.CreateTestingNS("multi-operand-istio", true) + Expect(err).NotTo(HaveOccurred()) + + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), istioNS.Name) + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), istioNS.Name) + err = waitForCertificateReadiness(ctx, "my-selfsigned-ca", istioNS.Name) + Expect(err).NotTo(HaveOccurred()) + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), istioNS.Name) + + ensureMultiOperandCRsAbsent(ctx, istioNS.Name) + + istioCSR, err := loadIstioCSRFromTemplate(istioNS.Name, IstioCSRConfig{}) + Expect(err).NotTo(HaveOccurred()) + + By("CM-786 steps 5/10: concurrent TrustManager and IstioCSR CR creation") + installOperandsConcurrently(ctx, qualificationTrustManagerCR().Build(), istioCSR) + + waitForOperandsReadyConcurrently(ctx, clientset, istioNS.Name) + + err = VerifyHealthyOperatorConditions(certmanageroperatorclient.OperatorV1alpha1()) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + if istioNS == nil { + return + } + expectDeleteClean(bundleClient.Delete(ctx, &trustapi.Bundle{ObjectMeta: metav1.ObjectMeta{Name: qualificationBundleName}}), + fmt.Sprintf("Bundle %s", qualificationBundleName)) + expectDeleteClean(k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Delete(ctx, qualificationBundleSourceCM, metav1.DeleteOptions{}), + fmt.Sprintf("ConfigMap %s/%s", trustManagerNamespace, qualificationBundleSourceCM)) + loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), istioNS.Name) + loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), istioNS.Name) + loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), istioNS.Name) + loader.DeleteTestingNS(istioNS.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + It("should have TrustManager and IstioCSR CRs ready after concurrent creation", func() { + tm, err := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + ready := meta.FindStatusCondition(tm.Status.Conditions, v1alpha1.Ready) + Expect(ready).NotTo(BeNil()) + Expect(ready.Status).To(Equal(metav1.ConditionTrue)) + Expect(tm.Status.TrustManagerImage).NotTo(BeEmpty()) + + istioCSRStatus, err := pollTillIstioCSRAvailable(ctx, loader, istioNS.Name, istioCSRResourceName) + Expect(err).NotTo(HaveOccurred()) + Expect(istioCSRStatus.IstioCSRGRPCEndpoint).NotTo(BeEmpty()) + Expect(istioCSRStatus.IstioCSRImage).NotTo(BeEmpty()) + }) + + It("CM-01: all operand deployments have desired replicas ready", func() { + verifyAllOperandDeploymentsReady(ctx, clientset, istioNS.Name) + }) + + It("CM-02: operand pods have no CrashLoopBackOff", func() { + Expect(allOperandPodsHealthy(ctx, clientset, istioNS.Name)).To(Succeed()) + }) + + It("CM-03: cert-manager webhook service has endpoints", func() { + verifyWebhookServiceHasEndpoints(ctx, clientset) + }) + + It("CM-10: issues a self-signed namespaced certificate", func() { + runCM10SelfSignedCertificateTest(ctx) + }) + + It("CM-12: certificate with bogus issuer ref stays not Ready and does not break operands", func() { + runCM12BogusIssuerTest(ctx, clientset, istioNS.Name) + }) + + It("CM-786 step 5: IstioCSR operand resources exist with operator-managed labels", func() { + verifyIstioCSROperandResources(ctx, clientset, istioNS.Name) + }) + + It("CM-786 step 10: TrustManager qualification configuration is applied", func() { + verifyTrustManagerQualificationConfig(ctx, clientset) + }) + + It("CM-786 step 10: Bundle propagates ConfigMap source to Secret in selected namespaces", func() { + runCM786BundleSecretTargetTest(ctx, qualificationBundlePEM) + }) + + It("CM-786 step 11: managed ClusterRoles are recreated after deletion", func() { + runCM786ManagedClusterRoleRecreationTest(ctx, clientset, istioNS.Name) + }) + + It("CM-786 step 12: updating all three operator CRs keeps operands healthy", func() { + runCM786UpdateAllOperatorCRsTest(ctx, clientset, istioNS.Name) + }) + + It("CM-786 step 13: IstioCSR API updates reconcile successfully", func() { + runCM786IstioCSRSpecUpdateTest(ctx, istioNS.Name) + }) + + It("CM-786 step 14: IstioCSR gRPC CreateCertificate returns a cert chain", func() { + runCM786IstioCSRGRPCCertificateTest(ctx, clientset, istioNS) + }) + + It("CM-786 steps 6-7: Service Mesh uses istio-csr when Service Mesh is installed", func() { + runCM786ServiceMeshIstioCSRIntegrationTest(ctx, clientset, istioNS.Name) + }) + }) +}) + +func qualificationTrustManagerCR() *trustManagerCRBuilder { + return newTrustManagerCR(). + WithLabels(map[string]string{"env": "trustmanager-test"}). + WithAnnotations(map[string]string{"trustmanager.operator.openshift.io/cluster": "trustmanager-test"}). + WithDefaultCAPackage(v1alpha1.DefaultCAPackagePolicyEnabled). + WithFilterExpiredCertificates(v1alpha1.FilterExpiredCertificatesPolicyEnabled). + WithSecretTargets(v1alpha1.SecretTargetsPolicyCustom, []string{"ca-bundle-secret", qualificationBundleName}). + WithTrustNamespace(trustManagerNamespace) +} + +func installOperandsConcurrently(ctx context.Context, trustManager *v1alpha1.TrustManager, istioCSR *v1alpha1.IstioCSR) { + start := make(chan struct{}) + var wg sync.WaitGroup + var tmErr, istioErr error + + wg.Add(2) + go func() { + defer wg.Done() + <-start + _, tmErr = trustManagerClient().Create(ctx, trustManager, metav1.CreateOptions{}) + }() + go func() { + defer wg.Done() + <-start + _, istioErr = certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioCSR.Namespace).Create(ctx, istioCSR, metav1.CreateOptions{}) + }() + + close(start) + wg.Wait() + + Expect(tmErr).NotTo(HaveOccurred()) + Expect(istioErr).NotTo(HaveOccurred()) +} + +func waitForOperandsReadyConcurrently(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { + var eg errgroup.Group + eg.Go(func() error { + if err := pollTillDeploymentAvailableWithTimeout(ctx, clientset, trustManagerNamespace, trustManagerDeploymentName, multiOperandReadyTimeout); err != nil { + return fmt.Errorf("trust-manager deployment: %w", err) + } + _, err := pollTillTrustManagerAvailableStrictWithTimeout(ctx, trustManagerClient(), "cluster", multiOperandReadyTimeout) + if err != nil { + return fmt.Errorf("trust-manager CR: %w", err) + } + return nil + }) + eg.Go(func() error { + if err := pollTillDeploymentAvailableWithTimeout(ctx, clientset, istioNamespace, istioCSRDeploymentName, multiOperandReadyTimeout); err != nil { + return fmt.Errorf("istio-csr deployment: %w", err) + } + _, err := pollTillIstioCSRAvailableWithTimeout(ctx, loader, istioNamespace, istioCSRResourceName, multiOperandReadyTimeout) + if err != nil { + return fmt.Errorf("istio-csr CR: %w", err) + } + return nil + }) + err := eg.Wait() + Expect(err).To(Succeed(), formatMultiOperandReadinessStatus(ctx, clientset, istioNamespace)) +} + +func formatMultiOperandReadinessStatus(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) string { + var b strings.Builder + + tm, tmErr := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) + if tmErr != nil { + fmt.Fprintf(&b, "TrustManager CR: get error: %v; ", tmErr) + } else { + fmt.Fprintf(&b, "TrustManager conditions=%v image=%q; ", tm.Status.Conditions, tm.Status.TrustManagerImage) + } + + if dep, depErr := clientset.AppsV1().Deployments(trustManagerNamespace).Get(ctx, trustManagerDeploymentName, metav1.GetOptions{}); depErr != nil { + fmt.Fprintf(&b, "trust-manager deployment: get error: %v; ", depErr) + } else { + fmt.Fprintf(&b, "trust-manager deployment ready=%d/%d available=%d; ", + dep.Status.ReadyReplicas, ptr.Deref(dep.Spec.Replicas, 1), dep.Status.AvailableReplicas) + } + + istioCSR, istioErr := certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Get(ctx, istioCSRResourceName, metav1.GetOptions{}) + if istioErr != nil { + fmt.Fprintf(&b, "IstioCSR CR: get error: %v; ", istioErr) + } else { + fmt.Fprintf(&b, "IstioCSR conditions=%v grpc=%q; ", istioCSR.Status.Conditions, istioCSR.Status.IstioCSRGRPCEndpoint) + } + + if dep, depErr := clientset.AppsV1().Deployments(istioNamespace).Get(ctx, istioCSRDeploymentName, metav1.GetOptions{}); depErr != nil { + fmt.Fprintf(&b, "istio-csr deployment: get error: %v", depErr) + } else { + fmt.Fprintf(&b, "istio-csr deployment ready=%d/%d available=%d", + dep.Status.ReadyReplicas, ptr.Deref(dep.Spec.Replicas, 1), dep.Status.AvailableReplicas) + } + + return strings.TrimSpace(b.String()) +} + +func verifyAllOperandDeploymentsReady(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { + deployments := append([]operandDeployment{}, coreOperandDeployments...) + deployments = append(deployments, operandDeployment{istioNamespace, istioCSRDeploymentName}) + + for _, dep := range deployments { + dep := dep + Eventually(func(g Gomega) { + g.Expect(assertDeploymentReplicasReady(ctx, clientset, dep.namespace, dep.name)).To(Succeed()) + }, lowTimeout, fastPollInterval).Should(Succeed(), "deployment %s/%s", dep.namespace, dep.name) + } +} + +func assertDeploymentReplicasReady(ctx context.Context, clientset *kubernetes.Clientset, namespace, name string) error { + deployment, err := clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + + desired := int32(1) + if deployment.Spec.Replicas != nil { + desired = *deployment.Spec.Replicas + } + + for _, cond := range deployment.Status.Conditions { + if cond.Type == appsv1.DeploymentAvailable && cond.Status == corev1.ConditionTrue { + if deployment.Status.ReadyReplicas >= desired && deployment.Status.AvailableReplicas >= desired { + return nil + } + } + } + + return fmt.Errorf("deployment %s/%s not ready: desired=%d ready=%d available=%d", + namespace, name, desired, deployment.Status.ReadyReplicas, deployment.Status.AvailableReplicas) +} + +func verifyWebhookServiceHasEndpoints(ctx context.Context, clientset *kubernetes.Clientset) { + Eventually(func(g Gomega) { + ep, err := clientset.CoreV1().Endpoints(operandNamespace).Get(ctx, certmanagerWebhookDeployment, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ep.Subsets).NotTo(BeEmpty()) + + var endpointCount int + for _, subset := range ep.Subsets { + endpointCount += len(subset.Addresses) + } + g.Expect(endpointCount).To(BeNumerically(">", 0)) + + pods, err := clientset.CoreV1().Pods(operandNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=webhook,app.kubernetes.io/component=webhook", + }) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(pods.Items).NotTo(BeEmpty()) + }, lowTimeout, fastPollInterval).Should(Succeed()) +} + +func runCM10SelfSignedCertificateTest(ctx context.Context) { + ns, err := loader.CreateTestingNS("multi-operand-cm10", false) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + const ( + clusterIssuerName = "cm10-multi-operand-selfsigned" + certName = "cm10-multi-operand-cert" + secretName = "cm10-multi-operand-tls" + ) + + _, err = certmanagerClient.CertmanagerV1().ClusterIssuers().Create(ctx, &certmanagerv1.ClusterIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: clusterIssuerName}, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{SelfSigned: &certmanagerv1.SelfSignedIssuer{}}, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = certmanagerClient.CertmanagerV1().ClusterIssuers().Delete(ctx, clusterIssuerName, metav1.DeleteOptions{}) + }) + + _, err = certmanagerClient.CertmanagerV1().Certificates(ns.Name).Create(ctx, &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: ns.Name}, + Spec: certmanagerv1.CertificateSpec{ + SecretName: secretName, + Duration: &metav1.Duration{Duration: time.Hour}, + DNSNames: []string{cm10TestDNSName}, + IssuerRef: cmmetav1.ObjectReference{ + Name: clusterIssuerName, Kind: "ClusterIssuer", Group: "cert-manager.io", + }, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + err = waitForCertificateReadiness(ctx, certName, ns.Name) + Expect(err).NotTo(HaveOccurred()) + + secret, err := k8sClientSet.CoreV1().Secrets(ns.Name).Get(ctx, secretName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(secret.Data).To(HaveKey("tls.crt")) + Expect(secret.Data).To(HaveKey("tls.key")) +} + +func runCM12BogusIssuerTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { + ns, err := loader.CreateTestingNS("multi-operand-cm12", false) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + const ( + bogusIssuerName = "cm12-nonexistent-clusterissuer" + certName = "cm12-bogus-issuer-cert" + secretName = "cm12-tls-should-not-exist" + ) + + _, err = certmanagerClient.CertmanagerV1().Certificates(ns.Name).Create(ctx, &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: ns.Name}, + Spec: certmanagerv1.CertificateSpec{ + SecretName: secretName, + DNSNames: []string{cm12TestDNSName}, + IssuerRef: cmmetav1.ObjectReference{ + Name: bogusIssuerName, Kind: "ClusterIssuer", Group: "cert-manager.io", + }, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + cert, getErr := certmanagerClient.CertmanagerV1().Certificates(ns.Name).Get(ctx, certName, metav1.GetOptions{}) + g.Expect(getErr).NotTo(HaveOccurred()) + var readyCondition *certmanagerv1.CertificateCondition + for i := range cert.Status.Conditions { + if cert.Status.Conditions[i].Type == certmanagerv1.CertificateConditionReady { + readyCondition = &cert.Status.Conditions[i] + break + } + } + g.Expect(readyCondition).NotTo(BeNil()) + g.Expect(readyCondition.Status).To(Equal(cmmetav1.ConditionFalse)) + g.Expect(readyCondition.Message).NotTo(BeEmpty()) + }, lowTimeout, fastPollInterval).Should(Succeed()) + + _, err = k8sClientSet.CoreV1().Secrets(ns.Name).Get(ctx, secretName, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + + Expect(allOperandPodsHealthy(ctx, clientset, istioNamespace)).To(Succeed()) + Expect(VerifyHealthyOperatorConditions(certmanageroperatorclient.OperatorV1alpha1())).To(Succeed()) + verifyAllOperandDeploymentsReady(ctx, clientset, istioNamespace) +} + +func verifyIstioCSROperandResources(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { + Eventually(func(g Gomega) { + sa, err := clientset.CoreV1().ServiceAccounts(istioNamespace).Get(ctx, istioCSRServiceAccountName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + verifyOperatorManagedLabels(sa.Labels, "cert-manager-istio-csr") + + svc, err := clientset.CoreV1().Services(istioNamespace).Get(ctx, istioCSRServiceName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + verifyOperatorManagedLabels(svc.Labels, "cert-manager-istio-csr") + + dep, err := clientset.AppsV1().Deployments(istioNamespace).Get(ctx, istioCSRDeploymentName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + verifyOperatorManagedLabels(dep.Labels, "cert-manager-istio-csr") + }, lowTimeout, fastPollInterval).Should(Succeed()) + + crs, err := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=cert-manager-istio-csr", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(crs.Items).NotTo(BeEmpty()) + verifyOperatorManagedLabels(crs.Items[0].Labels, "cert-manager-istio-csr") +} + +func verifyTrustManagerQualificationConfig(ctx context.Context, clientset *kubernetes.Clientset) { + Eventually(func(g Gomega) { + _, err := k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Get(ctx, defaultCAPackageConfigMapName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + + dep, err := clientset.AppsV1().Deployments(trustManagerNamespace).Get(ctx, trustManagerDeploymentName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(dep.Spec.Template.Spec.Containers).NotTo(BeEmpty()) + args := dep.Spec.Template.Spec.Containers[0].Args + g.Expect(args).To(ContainElement("--secret-targets-enabled=true")) + g.Expect(args).To(ContainElement("--filter-expired-certificates=true")) + hasDefaultPackageArg := false + for _, arg := range args { + if strings.Contains(arg, "--default-package-location=") { + hasDefaultPackageArg = true + break + } + } + g.Expect(hasDefaultPackageArg).To(BeTrue()) + }, lowTimeout, fastPollInterval).Should(Succeed()) + + tm, err := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(tm.Spec.TrustManagerConfig.DefaultCAPackage.Policy).To(Equal(v1alpha1.DefaultCAPackagePolicyEnabled)) + Expect(tm.Spec.TrustManagerConfig.SecretTargets.Policy).To(Equal(v1alpha1.SecretTargetsPolicyCustom)) + Expect(tm.Spec.TrustManagerConfig.FilterExpiredCertificates).To(Equal(v1alpha1.FilterExpiredCertificatesPolicyEnabled)) +} + +func runCM786BundleSecretTargetTest(ctx context.Context, sourcePEM string) { + targetNS, err := k8sClientSet.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cm786-bundle-target-", + Labels: map[string]string{"istio-injection": "enabled"}, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = k8sClientSet.CoreV1().Namespaces().Delete(ctx, targetNS.Name, metav1.DeleteOptions{}) + }) + + _, err = k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: qualificationBundleSourceCM}, + Data: map[string]string{qualificationBundleSourceKey: sourcePEM}, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + bundle := newBundle(qualificationBundleName). + WithConfigMapSource(qualificationBundleSourceCM, qualificationBundleSourceKey). + WithSecretTarget(qualificationBundleTargetKey). + WithNamespaceSelector(map[string]string{"istio-injection": "enabled"}). + Build() + + Expect(bundleClient.Create(ctx, bundle)).To(Succeed()) + + err = waitForBundleCondition(ctx, bundleClient, qualificationBundleName, trustapi.BundleConditionSynced, metav1.ConditionTrue, highTimeout) + Expect(err).NotTo(HaveOccurred()) + + err = waitForSecretTarget(ctx, bundleClient, qualificationBundleName, targetNS.Name, qualificationBundleTargetKey, sourcePEM, highTimeout) + Expect(err).NotTo(HaveOccurred()) +} + +func runCM786ManagedClusterRoleRecreationTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { + verifyResourceRecreation(ctx, func() error { + return clientset.RbacV1().ClusterRoles().Delete(ctx, trustManagerClusterRoleName, metav1.DeleteOptions{}) + }, func() error { + _, err := clientset.RbacV1().ClusterRoles().Get(ctx, trustManagerClusterRoleName, metav1.GetOptions{}) + return err + }) + + istioCRs, err := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=cert-manager-istio-csr", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(istioCRs.Items).NotTo(BeEmpty()) + deletedIstioCRName := istioCRs.Items[0].Name + + // IstioCSR ClusterRoles are created with GenerateName (cert-manager-istio-csr-). + // After deletion the controller recreates a new ClusterRole with a different suffix. + Expect(clientset.RbacV1().ClusterRoles().Delete(ctx, deletedIstioCRName, metav1.DeleteOptions{})).To(Succeed()) + Eventually(func(g Gomega) { + crs, listErr := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=cert-manager-istio-csr", + }) + g.Expect(listErr).NotTo(HaveOccurred()) + g.Expect(crs.Items).NotTo(BeEmpty(), "istio-csr ClusterRole was not recreated by controller") + }, lowTimeout, fastPollInterval).Should(Succeed()) + + if _, err := clientset.RbacV1().ClusterRoles().Get(ctx, "cert-manager-controller-challenges", metav1.GetOptions{}); err == nil { + verifyResourceRecreation(ctx, func() error { + return clientset.RbacV1().ClusterRoles().Delete(ctx, "cert-manager-controller-challenges", metav1.DeleteOptions{}) + }, func() error { + _, getErr := clientset.RbacV1().ClusterRoles().Get(ctx, "cert-manager-controller-challenges", metav1.GetOptions{}) + return getErr + }) + } + + Expect(VerifyHealthyOperatorConditions(certmanageroperatorclient.OperatorV1alpha1())).To(Succeed()) + + tm, err := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + ready := meta.FindStatusCondition(tm.Status.Conditions, v1alpha1.Ready) + Expect(ready).NotTo(BeNil()) + Expect(ready.Status).To(Equal(metav1.ConditionTrue)) + + _, err = pollTillIstioCSRAvailable(ctx, loader, istioNamespace, istioCSRResourceName) + Expect(err).NotTo(HaveOccurred()) +} + +func runCM786UpdateAllOperatorCRsTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cm, getErr := certmanageroperatorclient.OperatorV1alpha1().CertManagers().Get(ctx, "cluster", metav1.GetOptions{}) + if getErr != nil { + return getErr + } + cm.Labels = mergeLabels(cm.Labels, map[string]string{cm786QualificationLabelKey: cm786QualificationLabelValue}) + _, updateErr := certmanageroperatorclient.OperatorV1alpha1().CertManagers().Update(ctx, cm, metav1.UpdateOptions{}) + return updateErr + }) + Expect(err).NotTo(HaveOccurred()) + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + tm, getErr := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) + if getErr != nil { + return getErr + } + tm.Labels = mergeLabels(tm.Labels, map[string]string{cm786QualificationLabelKey: cm786QualificationLabelValue}) + _, updateErr := trustManagerClient().Update(ctx, tm, metav1.UpdateOptions{}) + return updateErr + }) + Expect(err).NotTo(HaveOccurred()) + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + istioCSR, getErr := certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Get(ctx, istioCSRResourceName, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + istioCSR.Labels = mergeLabels(istioCSR.Labels, map[string]string{cm786QualificationLabelKey: cm786QualificationLabelValue}) + _, updateErr := certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Update(ctx, istioCSR, metav1.UpdateOptions{}) + return updateErr + }) + Expect(err).NotTo(HaveOccurred()) + + Expect(VerifyHealthyOperatorConditions(certmanageroperatorclient.OperatorV1alpha1())).To(Succeed()) + verifyAllOperandDeploymentsReady(ctx, clientset, istioNamespace) + waitForOperandsReadyConcurrently(ctx, clientset, istioNamespace) +} + +func runCM786IstioCSRSpecUpdateTest(ctx context.Context, istioNamespace string) { + for _, level := range []int32{3, 2, 1} { + level := level + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + istioCSR, getErr := certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Get(ctx, istioCSRResourceName, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + istioCSR.Spec.IstioCSRConfig.LogLevel = level + istioCSR.Labels = mergeLabels(istioCSR.Labels, map[string]string{"test": fmt.Sprintf("cm786-log-%d", level)}) + _, updateErr := certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Update(ctx, istioCSR, metav1.UpdateOptions{}) + return updateErr + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = pollTillIstioCSRAvailable(ctx, loader, istioNamespace, istioCSRResourceName) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + dep, getErr := k8sClientSet.AppsV1().Deployments(istioNamespace).Get(ctx, istioCSRDeploymentName, metav1.GetOptions{}) + g.Expect(getErr).NotTo(HaveOccurred()) + g.Expect(dep.Spec.Template.Spec.Containers).NotTo(BeEmpty()) + g.Expect(dep.Spec.Template.Spec.Containers[0].Args).To(ContainElement(fmt.Sprintf("--log-level=%d", level))) + }, lowTimeout, fastPollInterval).Should(Succeed()) + } +} + +func runCM786IstioCSRGRPCCertificateTest(ctx context.Context, clientset *kubernetes.Clientset, ns *corev1.Namespace) { + const grpcAppName = "grpcurl-istio-csr" + + istioCSRStatus, err := pollTillIstioCSRAvailable(ctx, loader, ns.Name, istioCSRResourceName) + Expect(err).NotTo(HaveOccurred()) + + err = pollTillServiceAccountAvailable(ctx, clientset, ns.Name, istioCSRServiceAccountName) + Expect(err).NotTo(HaveOccurred()) + + protoBytes, err := testassets.ReadFile("testdata/ca.proto") + Expect(err).NotTo(HaveOccurred()) + _, err = clientset.CoreV1().ConfigMaps(ns.Name).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "proto-cm", Namespace: ns.Name}, + Data: map[string]string{"ca.proto": string(protoBytes)}, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = clientset.CoreV1().ConfigMaps(ns.Name).Delete(ctx, "proto-cm", metav1.DeleteOptions{}) + }) + + csr := generateCM786CSR(ns.Name) + loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( + IstioCSRGRPCurlJobConfig{ + CertificateSigningRequest: csr, + IstioCSRStatus: istioCSRStatus, + }, + ), filepath.Join("testdata", "istio", "grpcurl_job.yaml"), ns.Name) + DeferCleanup(func() { + policy := metav1.DeletePropagationBackground + _ = clientset.BatchV1().Jobs(ns.Name).Delete(ctx, grpcAppName, metav1.DeleteOptions{PropagationPolicy: &policy}) + }) + + err = pollTillJobCompleted(ctx, clientset, ns.Name, grpcAppName) + Expect(err).NotTo(HaveOccurred()) + + pods, err := clientset.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", grpcAppName)}) + Expect(err).NotTo(HaveOccurred()) + + var succeededPodName string + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodSucceeded { + succeededPodName = pod.Name + } + } + Expect(succeededPodName).NotTo(BeEmpty()) + + req := clientset.CoreV1().Pods(ns.Name).GetLogs(succeededPodName, &corev1.PodLogOptions{}) + logStream, err := req.Stream(ctx) + Expect(err).NotTo(HaveOccurred()) + defer logStream.Close() + + logData, err := io.ReadAll(logStream) + Expect(err).NotTo(HaveOccurred()) + + var entry LogEntry + Expect(json.Unmarshal(logData, &entry)).To(Succeed()) + Expect(entry.CertChain).NotTo(BeEmpty()) + + for _, certPEM := range entry.CertChain { + Expect(library.ValidateCertificate(certPEM, "my-selfsigned-ca")).To(Succeed()) + } +} + +func runCM786ServiceMeshIstioCSRIntegrationTest(ctx context.Context, clientset *kubernetes.Clientset, istioCSRNamespace string) { + istioGVR := schema.GroupVersionResource{Group: "sailoperator.io", Version: "v1", Resource: "istios"} + istioCR, err := loader.DynamicClient.Resource(istioGVR).Get(ctx, "default", metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + Skip("OpenShift Service Mesh (sailoperator.io/v1 Istio/default) is not installed; CM-786 steps 6-7 require manual mesh qualification") + } + Expect(err).NotTo(HaveOccurred()) + } + + caAddress, found, err := unstructuredNestedString(istioCR.Object, "spec", "values", "global", "caAddress") + if err != nil || !found { + Skip("Service Mesh Istio CR does not expose spec.values.global.caAddress; skipping CM-786 step 7 mesh CA check") + } + Expect(caAddress).To(ContainSubstring("cert-manager-istio-csr")) + + svc, err := clientset.CoreV1().Services(istioCSRNamespace).Get(ctx, istioCSRServiceName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(svc.Spec.Ports).NotTo(BeEmpty()) + + Eventually(func(g Gomega) { + ep, epErr := clientset.CoreV1().Endpoints(istioCSRNamespace).Get(ctx, istioCSRServiceName, metav1.GetOptions{}) + g.Expect(epErr).NotTo(HaveOccurred()) + g.Expect(ep.Subsets).NotTo(BeEmpty()) + }, lowTimeout, fastPollInterval).Should(Succeed()) +} + +func generateCM786CSR(istioNamespace string) string { + csrTemplate := &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"CM-786"}, + OrganizationalUnit: []string{"cert-manager-operator-e2e"}, + Country: []string{"US"}, + }, + URIs: []*url.URL{ + {Scheme: "spiffe", Host: "cluster.local", Path: fmt.Sprintf("/ns/%s/sa/cert-manager-istio-csr", istioNamespace)}, + }, + SignatureAlgorithm: x509.SHA256WithRSA, + } + csr, err := library.GenerateCSR(csrTemplate) + Expect(err).NotTo(HaveOccurred()) + return csr +} + +func verifyResourceRecreation(ctx context.Context, deleteFunc func() error, getFunc func() error) { + Expect(deleteFunc()).To(Succeed()) + Eventually(func() error { + return getFunc() + }, lowTimeout, fastPollInterval).Should(Succeed(), "resource was not recreated by controller") +} + +func verifyOperatorManagedLabels(labels map[string]string, name string) { + Expect(labels).To(HaveKeyWithValue("app.kubernetes.io/managed-by", "cert-manager-operator")) + Expect(labels).To(HaveKeyWithValue("app.kubernetes.io/name", name)) + Expect(labels).To(HaveKeyWithValue("app.kubernetes.io/part-of", "cert-manager-operator")) +} + +func allOperandPodsHealthy(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) error { + namespaces := []string{operandNamespace, istioNamespace} + for _, ns := range namespaces { + pods, err := clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + for _, pod := range pods.Items { + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil && cs.State.Waiting.Reason == "CrashLoopBackOff" { + return fmt.Errorf("pod %s/%s container %s in CrashLoopBackOff", ns, pod.Name, cs.Name) + } + } + } + } + return nil +} + +func mergeLabels(existing map[string]string, extra map[string]string) map[string]string { + out := make(map[string]string, len(existing)+len(extra)) + for k, v := range existing { + out[k] = v + } + for k, v := range extra { + out[k] = v + } + return out +} + +func unstructuredNestedString(obj map[string]interface{}, fields ...string) (string, bool, error) { + val, found, err := unstructuredNestedField(obj, fields...) + if !found || err != nil { + return "", found, err + } + s, ok := val.(string) + return s, ok, nil +} + +func unstructuredNestedField(obj map[string]interface{}, fields ...string) (interface{}, bool, error) { + var current interface{} = obj + for _, field := range fields { + m, ok := current.(map[string]interface{}) + if !ok { + return nil, false, nil + } + val, ok := m[field] + if !ok { + return nil, false, nil + } + current = val + } + return current, true, nil +} + +func loadIstioCSRFromTemplate(namespace string, cfg IstioCSRConfig) (*v1alpha1.IstioCSR, error) { + raw, err := testassets.ReadFile(filepath.Join("testdata", "istio", "istio_csr_template.yaml")) + if err != nil { + return nil, err + } + rendered, err := replaceWithTemplate(string(raw), cfg) + if err != nil { + return nil, err + } + var istioCSR v1alpha1.IstioCSR + if err := yaml.Unmarshal(rendered, &istioCSR); err != nil { + return nil, err + } + istioCSR.Namespace = namespace + // Align istio data-plane namespace with the test namespace. The template defaults to + // istio-system (matching istio_csr_test.go), but multi-operand tests use a dedicated NS. + istioCSR.Spec.IstioCSRConfig.Istio.Namespace = namespace + return &istioCSR, nil +} + +// expectDeleteClean fails the spec when delete returns an unexpected error (NotFound is OK). +func expectDeleteClean(err error, resource string) { + if err == nil || apierrors.IsNotFound(err) { + return + } + Expect(err).NotTo(HaveOccurred(), "cleanup delete "+resource) +} + +// isMicroShiftCluster reports whether the cluster is MicroShift (no OLM/Subscription). +func isMicroShiftCluster(ctx context.Context, clientset *kubernetes.Clientset) bool { + for _, ns := range []string{"kube-public", "kube-system"} { + _, err := clientset.CoreV1().ConfigMaps(ns).Get(ctx, "microshift-version", metav1.GetOptions{}) + if err == nil { + return true + } + if !apierrors.IsNotFound(err) { + continue + } + } + return false +} + +func ensureMultiOperandCRsAbsent(ctx context.Context, istioNamespace string) { + _, err := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + _, err = certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Get(ctx, istioCSRResourceName, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) +} diff --git a/test/e2e/trustmanager_helpers_test.go b/test/e2e/trustmanager_helpers_test.go index c33393686..9961c8169 100644 --- a/test/e2e/trustmanager_helpers_test.go +++ b/test/e2e/trustmanager_helpers_test.go @@ -132,6 +132,13 @@ func deleteTrustManager(ctx context.Context) { }, lowTimeout, fastPollInterval).Should(BeTrue()) } +// deleteTrustManagerDefaultCAPackageConfigMap removes the operand ConfigMap created when +// default CAPackage is Enabled. The operator does not delete it when the CR is removed +// or policy is Disabled, so e2e tests must clean it up to avoid cross-suite pollution. +func deleteTrustManagerDefaultCAPackageConfigMap(ctx context.Context) { + _ = k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Delete(ctx, defaultCAPackageConfigMapName, metav1.DeleteOptions{}) +} + // --------------------------------------------------------------------------- // Bundle builder // --------------------------------------------------------------------------- diff --git a/test/e2e/trustmanager_test.go b/test/e2e/trustmanager_test.go index 41369a242..ebfdf525c 100644 --- a/test/e2e/trustmanager_test.go +++ b/test/e2e/trustmanager_test.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "slices" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -1517,9 +1518,24 @@ var _ = Describe("TrustManager with operator feature gate disabled", Ordered, La // pollTillTrustManagerAvailable polls the TrustManager object and returns its status // once the TrustManager is available, otherwise returns a time-out error. func pollTillTrustManagerAvailable(ctx context.Context, client operatorclientv1alpha1.TrustManagerInterface, trustManagerName string) (v1alpha1.TrustManagerStatus, error) { + return pollTillTrustManagerAvailableWithTimeout(ctx, client, trustManagerName, highTimeout) +} + +func pollTillTrustManagerAvailableWithTimeout(ctx context.Context, client operatorclientv1alpha1.TrustManagerInterface, trustManagerName string, timeout time.Duration) (v1alpha1.TrustManagerStatus, error) { + return pollTillTrustManagerWithTimeout(ctx, client, trustManagerName, timeout, false) +} + +// pollTillTrustManagerAvailableStrictWithTimeout is like pollTillTrustManagerAvailableWithTimeout +// but returns immediately when Degraded=True. Use when a degraded TrustManager should not recover +// during the wait (e.g. concurrent multi-operand install). +func pollTillTrustManagerAvailableStrictWithTimeout(ctx context.Context, client operatorclientv1alpha1.TrustManagerInterface, trustManagerName string, timeout time.Duration) (v1alpha1.TrustManagerStatus, error) { + return pollTillTrustManagerWithTimeout(ctx, client, trustManagerName, timeout, true) +} + +func pollTillTrustManagerWithTimeout(ctx context.Context, client operatorclientv1alpha1.TrustManagerInterface, trustManagerName string, timeout time.Duration, failOnDegraded bool) (v1alpha1.TrustManagerStatus, error) { var trustManagerStatus v1alpha1.TrustManagerStatus - err := wait.PollUntilContextTimeout(ctx, slowPollInterval, highTimeout, true, func(context.Context) (bool, error) { + err := wait.PollUntilContextTimeout(ctx, slowPollInterval, timeout, true, func(context.Context) (bool, error) { trustManager, err := client.Get(ctx, trustManagerName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -1536,6 +1552,9 @@ func pollTillTrustManagerAvailable(ctx context.Context, client operatorclientv1a degradedCondition := meta.FindStatusCondition(trustManagerStatus.Conditions, v1alpha1.Degraded) if degradedCondition != nil && degradedCondition.Status == metav1.ConditionTrue { + if failOnDegraded { + return false, fmt.Errorf("TrustManager is degraded: %s", degradedCondition.Message) + } return false, nil } @@ -1734,5 +1753,7 @@ func trustManagerAfterEach(ctx context.Context) func() { _, err := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) return apierrors.IsNotFound(err) }, lowTimeout, fastPollInterval).Should(BeTrue()) + + deleteTrustManagerDefaultCAPackageConfigMap(ctx) } } diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index f800af99b..3aaabb5cf 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -82,6 +82,10 @@ var ( // Image pulls and PVC binding on CI often exceed the previous 3m default. vaultPodStartTimeout = 10 * time.Minute + // multiOperandReadyTimeout covers concurrent TrustManager and IstioCSR install on slow CI: + // operator rollout, image pulls, and CR reconciliation can exceed highTimeout individually. + multiOperandReadyTimeout = 20 * time.Minute + // fastPollInterval and lowTimeout are // used together in poll(s) with fast reaction and // smaller timeout window. @@ -1167,9 +1171,13 @@ func pollTillServiceAccountAvailable(ctx context.Context, clientset *kubernetes. // pollTillIstioCSRAvailable poll the istioCSR object and returns non-nil error and istioCSRStatus // once the istiocsr is available, otherwise should return a time-out error func pollTillIstioCSRAvailable(ctx context.Context, loader library.DynamicResourceLoader, namespace, istioCsrName string) (v1alpha1.IstioCSRStatus, error) { + return pollTillIstioCSRAvailableWithTimeout(ctx, loader, namespace, istioCsrName, highTimeout) +} + +func pollTillIstioCSRAvailableWithTimeout(ctx context.Context, loader library.DynamicResourceLoader, namespace, istioCsrName string, timeout time.Duration) (v1alpha1.IstioCSRStatus, error) { var istioCSRStatus v1alpha1.IstioCSRStatus istiocsrClient := loader.DynamicClient.Resource(istiocsrSchema).Namespace(namespace) - err := wait.PollUntilContextTimeout(ctx, slowPollInterval, highTimeout, true, func(context.Context) (bool, error) { + err := wait.PollUntilContextTimeout(ctx, slowPollInterval, timeout, true, func(context.Context) (bool, error) { customResource, err := istiocsrClient.Get(ctx, istioCsrName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -1343,7 +1351,11 @@ func cleanupAllIstioCSROperands(ctx context.Context, loader library.DynamicResou // pollTillDeploymentAvailable poll the deployment object and returns non-nil error // once the deployment is available, otherwise should return a time-out error func pollTillDeploymentAvailable(ctx context.Context, clientSet *kubernetes.Clientset, namespace, deploymentName string) error { - err := wait.PollUntilContextTimeout(ctx, slowPollInterval, highTimeout, true, func(context.Context) (bool, error) { + return pollTillDeploymentAvailableWithTimeout(ctx, clientSet, namespace, deploymentName, highTimeout) +} + +func pollTillDeploymentAvailableWithTimeout(ctx context.Context, clientSet *kubernetes.Clientset, namespace, deploymentName string, timeout time.Duration) error { + err := wait.PollUntilContextTimeout(ctx, slowPollInterval, timeout, true, func(context.Context) (bool, error) { deployment, err := clientSet.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -2085,15 +2097,15 @@ func formatVaultPodsStatus(pods []corev1.Pod) string { case cs.State.Running != nil: b.WriteString(" state=running") case cs.State.Waiting != nil: - fmt.Fprintf(&b, " state=waiting reason=%s message=%q", cs.State.Waiting.Reason, cs.State.Waiting.Message) + fmt.Fprintf(&b, " state=waiting reason=%s", cs.State.Waiting.Reason) case cs.State.Terminated != nil: - fmt.Fprintf(&b, " state=terminated reason=%s exitCode=%d message=%q", - cs.State.Terminated.Reason, cs.State.Terminated.ExitCode, cs.State.Terminated.Message) + fmt.Fprintf(&b, " state=terminated reason=%s exitCode=%d", + cs.State.Terminated.Reason, cs.State.Terminated.ExitCode) } } for _, cond := range pod.Status.Conditions { if cond.Status != corev1.ConditionTrue { - fmt.Fprintf(&b, " condition=%s:%s:%s", cond.Type, cond.Status, cond.Message) + fmt.Fprintf(&b, " condition=%s:%s", cond.Type, cond.Status) } } b.WriteString("; ") diff --git a/test/go.mod b/test/go.mod index 77c538f77..f1dd0ef1b 100644 --- a/test/go.mod +++ b/test/go.mod @@ -25,6 +25,7 @@ require ( github.com/stretchr/testify v1.11.1 github.com/tidwall/gjson v1.18.0 github.com/vmware-archive/yaml-patch v0.0.11 + golang.org/x/sync v0.20.0 google.golang.org/api v0.269.0 k8s.io/api v0.35.2 k8s.io/apiextensions-apiserver v0.35.2 @@ -128,7 +129,6 @@ require ( golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect From 3a053732488e58a338c95675d4d80342c12577eb Mon Sep 17 00:00:00 2001 From: Arun Maurya Date: Thu, 16 Jul 2026 01:24:20 +0530 Subject: [PATCH 2/2] test(e2e): address multi-operand PR review feedback Drop qualification-doc identifiers in favor of behavior-based names, rename TrustManager config assertion, poll operand deployments in parallel, move TrustManager operand cleanup to teardown, restore Vault pod status messages for CI debugging, and split ACME HTTP01 certificate validation into explicit readiness and verification steps. Co-authored-by: Cursor --- test/e2e/issuer_acme_http01_test.go | 12 +- test/e2e/multiple_operands_test.go | 195 ++++++++++++++------------ test/e2e/trustmanager_helpers_test.go | 15 ++ test/e2e/trustmanager_test.go | 21 +-- test/e2e/utils_test.go | 8 +- 5 files changed, 134 insertions(+), 117 deletions(-) diff --git a/test/e2e/issuer_acme_http01_test.go b/test/e2e/issuer_acme_http01_test.go index 7fb6ac698..655ec0bc8 100644 --- a/test/e2e/issuer_acme_http01_test.go +++ b/test/e2e/issuer_acme_http01_test.go @@ -224,8 +224,16 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered It("should obtain a valid certificate", func() { + By("waiting for certificate to become ready") + err := waitForCertificateReadiness(ctx, secretName, ns.Name) + Expect(err).NotTo(HaveOccurred(), "timeout waiting for certificate to become ready") + By("checking TLS certificate contents") - err := wait.PollUntilContextTimeout(ctx, slowPollInterval, highTimeout, true, func(context.Context) (bool, error) { + err = verifyCertificate(ctx, secretName, ns.Name, ingressHost) + Expect(err).NotTo(HaveOccurred(), "certificate verification failed") + + By("verifying live Ingress TLS endpoint") + err = wait.PollUntilContextTimeout(ctx, slowPollInterval, highTimeout, true, func(context.Context) (bool, error) { secret, err := loader.KubeClient.CoreV1().Secrets(ns.Name).Get(ctx, secretName, metav1.GetOptions{}) if err != nil { return false, nil // keep polling until the Secret exists @@ -246,7 +254,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered return isHostCorrect && isNotExpired, nil }) - Expect(err).NotTo(HaveOccurred(), "timeout waiting for valid TLS certificate") + Expect(err).NotTo(HaveOccurred(), "timeout waiting for valid live Ingress TLS certificate") }) It("should create solver pods with custom resource limits and requests", func() { diff --git a/test/e2e/multiple_operands_test.go b/test/e2e/multiple_operands_test.go index bc39e976e..b86dc8e9a 100644 --- a/test/e2e/multiple_operands_test.go +++ b/test/e2e/multiple_operands_test.go @@ -1,9 +1,8 @@ //go:build e2e // +build e2e -// CM-786 IstioCSR Controller qualification (doc section "IstioCSR Controller specific tests" -// through "Record about cert-manager Operator"): multi-operand install plus sanity and -// qualification steps 5–14 that are automatable in e2e. +// Multi-operand e2e coverage: concurrent IstioCSR and TrustManager install, operand +// health checks, cert-manager behavior, and integration scenarios automatable in e2e. package e2e import ( @@ -37,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" @@ -49,16 +49,16 @@ const ( bothOperandsFeatureGates = "IstioCSR=true,TrustManager=true" - cm786QualificationLabelKey = "cm-786-qualification" - cm786QualificationLabelValue = "true" + multiOperandTestLabelKey = "cert-manager.operator.openshift.io/e2e-test" + multiOperandTestLabelValue = "multi-operands" - cm10TestDNSName = "multi-operand.cm10.test.example" - cm12TestDNSName = "multi-operand.cm12.invalid.example" + selfSignedCertDNSName = "multi-operand.selfsigned.test.example" + bogusIssuerCertDNSName = "multi-operand.bogus-issuer.invalid.example" - qualificationBundleName = "cm786-ca-bundle-secret" - qualificationBundleSourceCM = "cm786-bundle-source" - qualificationBundleSourceKey = "ca-bundle.crt" - qualificationBundleTargetKey = "ca-bundle.crt" + multiOperandBundleName = "multi-operand-ca-bundle-secret" + multiOperandBundleSourceCM = "multi-operand-bundle-source" + multiOperandBundleSourceKey = "ca-bundle.crt" + multiOperandBundleTargetKey = "ca-bundle.crt" ) type operandDeployment struct { @@ -73,7 +73,7 @@ var coreOperandDeployments = []operandDeployment{ {operandNamespace, trustManagerDeploymentName}, } -var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platform:Generic", "Feature:MultipleOperands", "TechPreview", "Skipped:MicroShift"), func() { +var _ = Describe("Multiple operands", Ordered, Label("Platform:Generic", "Feature:MultipleOperands", "TechPreview", "Skipped:MicroShift"), func() { var ( ctx = context.Background() @@ -81,7 +81,7 @@ var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platf originalUnsupportedAddonFeatures string originalOperatorLogLevel string istioNS *corev1.Namespace - qualificationBundlePEM string + multiOperandBundlePEM string ) BeforeAll(func() { @@ -93,15 +93,13 @@ var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platf Skip("MicroShift: OLM not available") } - trustManagerRemoveStaleOperandServiceAccount(ctx, clientset) - originalUnsupportedAddonFeatures, err = getSubscriptionEnvVar(ctx, loader, "UNSUPPORTED_ADDON_FEATURES") Expect(err).NotTo(HaveOccurred()) originalOperatorLogLevel, err = getSubscriptionEnvVar(ctx, loader, "OPERATOR_LOG_LEVEL") Expect(err).NotTo(HaveOccurred()) - By("CM-786 step 8: enable IstioCSR and TrustManager feature gates") + By("enabling IstioCSR and TrustManager feature gates") err = patchSubscriptionWithEnvVars(ctx, loader, map[string]string{ "UNSUPPORTED_ADDON_FEATURES": bothOperandsFeatureGates, "OPERATOR_LOG_LEVEL": "4", @@ -116,12 +114,12 @@ var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platf cert.IsCA = true cert.KeyUsage |= x509.KeyUsageCertSign } - qualificationBundlePEM = testutils.GenerateCertificate("cm786-bundle-ca", []string{"cert-manager-operator-e2e"}, caTweak) + multiOperandBundlePEM = testutils.GenerateCertificate("multi-operand-bundle-ca", []string{"cert-manager-operator-e2e"}, caTweak) }) AfterAll(func() { expectDeleteClean(trustManagerClient().Delete(ctx, "cluster", metav1.DeleteOptions{}), "TrustManager CR cluster") - deleteTrustManagerDefaultCAPackageConfigMap(ctx) + cleanupTrustManagerOperandLeavings(ctx) if istioNS != nil { expectDeleteClean( certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNS.Name).Delete(ctx, istioCSRResourceName, metav1.DeleteOptions{}), @@ -162,8 +160,8 @@ var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platf istioCSR, err := loadIstioCSRFromTemplate(istioNS.Name, IstioCSRConfig{}) Expect(err).NotTo(HaveOccurred()) - By("CM-786 steps 5/10: concurrent TrustManager and IstioCSR CR creation") - installOperandsConcurrently(ctx, qualificationTrustManagerCR().Build(), istioCSR) + By("creating TrustManager and IstioCSR CRs concurrently") + installOperandsConcurrently(ctx, multiOperandTrustManagerCR().Build(), istioCSR) waitForOperandsReadyConcurrently(ctx, clientset, istioNS.Name) @@ -175,10 +173,10 @@ var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platf if istioNS == nil { return } - expectDeleteClean(bundleClient.Delete(ctx, &trustapi.Bundle{ObjectMeta: metav1.ObjectMeta{Name: qualificationBundleName}}), - fmt.Sprintf("Bundle %s", qualificationBundleName)) - expectDeleteClean(k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Delete(ctx, qualificationBundleSourceCM, metav1.DeleteOptions{}), - fmt.Sprintf("ConfigMap %s/%s", trustManagerNamespace, qualificationBundleSourceCM)) + expectDeleteClean(bundleClient.Delete(ctx, &trustapi.Bundle{ObjectMeta: metav1.ObjectMeta{Name: multiOperandBundleName}}), + fmt.Sprintf("Bundle %s", multiOperandBundleName)) + expectDeleteClean(k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Delete(ctx, multiOperandBundleSourceCM, metav1.DeleteOptions{}), + fmt.Sprintf("ConfigMap %s/%s", trustManagerNamespace, multiOperandBundleSourceCM)) loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), istioNS.Name) loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), istioNS.Name) loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), istioNS.Name) @@ -199,71 +197,75 @@ var _ = Describe("Multiple operands CM-786 qualification", Ordered, Label("Platf Expect(istioCSRStatus.IstioCSRImage).NotTo(BeEmpty()) }) - It("CM-01: all operand deployments have desired replicas ready", func() { + It("should have all operand deployments with desired replicas ready", func() { verifyAllOperandDeploymentsReady(ctx, clientset, istioNS.Name) }) - It("CM-02: operand pods have no CrashLoopBackOff", func() { + It("should keep operand pods out of CrashLoopBackOff", func() { Expect(allOperandPodsHealthy(ctx, clientset, istioNS.Name)).To(Succeed()) }) - It("CM-03: cert-manager webhook service has endpoints", func() { + It("should expose cert-manager webhook service endpoints", func() { verifyWebhookServiceHasEndpoints(ctx, clientset) }) - It("CM-10: issues a self-signed namespaced certificate", func() { - runCM10SelfSignedCertificateTest(ctx) + It("should issue a self-signed namespaced certificate", func() { + runMultiOperandSelfSignedCertificateTest(ctx) }) - It("CM-12: certificate with bogus issuer ref stays not Ready and does not break operands", func() { - runCM12BogusIssuerTest(ctx, clientset, istioNS.Name) + It("should keep operands healthy when a certificate references a bogus issuer", func() { + runMultiOperandBogusIssuerCertificateTest(ctx, clientset, istioNS.Name) }) - It("CM-786 step 5: IstioCSR operand resources exist with operator-managed labels", func() { + It("should create IstioCSR operand resources with operator-managed labels", func() { verifyIstioCSROperandResources(ctx, clientset, istioNS.Name) }) - It("CM-786 step 10: TrustManager qualification configuration is applied", func() { - verifyTrustManagerQualificationConfig(ctx, clientset) + It("should apply TrustManager operand configuration", func() { + assertTrustManagerCRConfigPropagation(ctx, clientset) }) - It("CM-786 step 10: Bundle propagates ConfigMap source to Secret in selected namespaces", func() { - runCM786BundleSecretTargetTest(ctx, qualificationBundlePEM) + It("should propagate Bundle ConfigMap source to Secret in selected namespaces", func() { + runMultiOperandBundleSecretTargetTest(ctx, multiOperandBundlePEM) }) - It("CM-786 step 11: managed ClusterRoles are recreated after deletion", func() { - runCM786ManagedClusterRoleRecreationTest(ctx, clientset, istioNS.Name) + It("should recreate managed ClusterRoles after deletion", func() { + runMultiOperandManagedClusterRoleRecreationTest(ctx, clientset, istioNS.Name) }) - It("CM-786 step 12: updating all three operator CRs keeps operands healthy", func() { - runCM786UpdateAllOperatorCRsTest(ctx, clientset, istioNS.Name) + It("should keep operands healthy after updating all three operator CRs", func() { + runMultiOperandUpdateAllOperatorCRsTest(ctx, clientset, istioNS.Name) }) - It("CM-786 step 13: IstioCSR API updates reconcile successfully", func() { - runCM786IstioCSRSpecUpdateTest(ctx, istioNS.Name) + It("should reconcile IstioCSR API updates successfully", func() { + runMultiOperandIstioCSRSpecUpdateTest(ctx, istioNS.Name) }) - It("CM-786 step 14: IstioCSR gRPC CreateCertificate returns a cert chain", func() { - runCM786IstioCSRGRPCCertificateTest(ctx, clientset, istioNS) + It("should return a cert chain from IstioCSR gRPC CreateCertificate", func() { + runMultiOperandIstioCSRGRPCCertificateTest(ctx, clientset, istioNS) }) - It("CM-786 steps 6-7: Service Mesh uses istio-csr when Service Mesh is installed", func() { - runCM786ServiceMeshIstioCSRIntegrationTest(ctx, clientset, istioNS.Name) + It("should use istio-csr for Service Mesh when Service Mesh is installed", func() { + runMultiOperandServiceMeshIstioCSRIntegrationTest(ctx, clientset, istioNS.Name) }) }) }) -func qualificationTrustManagerCR() *trustManagerCRBuilder { +func multiOperandTrustManagerCR() *trustManagerCRBuilder { return newTrustManagerCR(). WithLabels(map[string]string{"env": "trustmanager-test"}). WithAnnotations(map[string]string{"trustmanager.operator.openshift.io/cluster": "trustmanager-test"}). WithDefaultCAPackage(v1alpha1.DefaultCAPackagePolicyEnabled). WithFilterExpiredCertificates(v1alpha1.FilterExpiredCertificatesPolicyEnabled). - WithSecretTargets(v1alpha1.SecretTargetsPolicyCustom, []string{"ca-bundle-secret", qualificationBundleName}). + WithSecretTargets(v1alpha1.SecretTargetsPolicyCustom, []string{"ca-bundle-secret", multiOperandBundleName}). WithTrustNamespace(trustManagerNamespace) } +// installOperandsConcurrently creates TrustManager and IstioCSR CRs at the same time so both +// controllers reconcile in parallel. Each Create is blocking, but running them in separate +// goroutines overlaps the requests; sequential Create would not exercise concurrent install. func installOperandsConcurrently(ctx context.Context, trustManager *v1alpha1.TrustManager, istioCSR *v1alpha1.IstioCSR) { + // Gate both goroutines so close(start) releases them together. start := make(chan struct{}) var wg sync.WaitGroup var tmErr, istioErr error @@ -271,16 +273,16 @@ func installOperandsConcurrently(ctx context.Context, trustManager *v1alpha1.Tru wg.Add(2) go func() { defer wg.Done() - <-start + <-start // block until both workers are ready _, tmErr = trustManagerClient().Create(ctx, trustManager, metav1.CreateOptions{}) }() go func() { defer wg.Done() - <-start + <-start // block until both workers are ready _, istioErr = certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioCSR.Namespace).Create(ctx, istioCSR, metav1.CreateOptions{}) }() - close(start) + close(start) // release both Create calls at once wg.Wait() Expect(tmErr).NotTo(HaveOccurred()) @@ -351,12 +353,23 @@ func verifyAllOperandDeploymentsReady(ctx context.Context, clientset *kubernetes deployments := append([]operandDeployment{}, coreOperandDeployments...) deployments = append(deployments, operandDeployment{istioNamespace, istioCSRDeploymentName}) + var eg errgroup.Group for _, dep := range deployments { dep := dep - Eventually(func(g Gomega) { - g.Expect(assertDeploymentReplicasReady(ctx, clientset, dep.namespace, dep.name)).To(Succeed()) - }, lowTimeout, fastPollInterval).Should(Succeed(), "deployment %s/%s", dep.namespace, dep.name) + eg.Go(func() error { + err := wait.PollUntilContextTimeout(ctx, fastPollInterval, lowTimeout, true, func(context.Context) (bool, error) { + if err := assertDeploymentReplicasReady(ctx, clientset, dep.namespace, dep.name); err != nil { + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("deployment %s/%s: %w", dep.namespace, dep.name, err) + } + return nil + }) } + Expect(eg.Wait()).To(Succeed()) } func assertDeploymentReplicasReady(ctx context.Context, clientset *kubernetes.Clientset, namespace, name string) error { @@ -402,17 +415,17 @@ func verifyWebhookServiceHasEndpoints(ctx context.Context, clientset *kubernetes }, lowTimeout, fastPollInterval).Should(Succeed()) } -func runCM10SelfSignedCertificateTest(ctx context.Context) { - ns, err := loader.CreateTestingNS("multi-operand-cm10", false) +func runMultiOperandSelfSignedCertificateTest(ctx context.Context) { + ns, err := loader.CreateTestingNS("multi-operand-selfsigned", false) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) }) const ( - clusterIssuerName = "cm10-multi-operand-selfsigned" - certName = "cm10-multi-operand-cert" - secretName = "cm10-multi-operand-tls" + clusterIssuerName = "multi-operand-selfsigned-issuer" + certName = "multi-operand-selfsigned-cert" + secretName = "multi-operand-selfsigned-tls" ) _, err = certmanagerClient.CertmanagerV1().ClusterIssuers().Create(ctx, &certmanagerv1.ClusterIssuer{ @@ -431,7 +444,7 @@ func runCM10SelfSignedCertificateTest(ctx context.Context) { Spec: certmanagerv1.CertificateSpec{ SecretName: secretName, Duration: &metav1.Duration{Duration: time.Hour}, - DNSNames: []string{cm10TestDNSName}, + DNSNames: []string{selfSignedCertDNSName}, IssuerRef: cmmetav1.ObjectReference{ Name: clusterIssuerName, Kind: "ClusterIssuer", Group: "cert-manager.io", }, @@ -448,24 +461,24 @@ func runCM10SelfSignedCertificateTest(ctx context.Context) { Expect(secret.Data).To(HaveKey("tls.key")) } -func runCM12BogusIssuerTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { - ns, err := loader.CreateTestingNS("multi-operand-cm12", false) +func runMultiOperandBogusIssuerCertificateTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { + ns, err := loader.CreateTestingNS("multi-operand-bogus-issuer", false) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) }) const ( - bogusIssuerName = "cm12-nonexistent-clusterissuer" - certName = "cm12-bogus-issuer-cert" - secretName = "cm12-tls-should-not-exist" + bogusIssuerName = "multi-operand-nonexistent-clusterissuer" + certName = "multi-operand-bogus-issuer-cert" + secretName = "multi-operand-bogus-issuer-tls" ) _, err = certmanagerClient.CertmanagerV1().Certificates(ns.Name).Create(ctx, &certmanagerv1.Certificate{ ObjectMeta: metav1.ObjectMeta{Name: certName, Namespace: ns.Name}, Spec: certmanagerv1.CertificateSpec{ SecretName: secretName, - DNSNames: []string{cm12TestDNSName}, + DNSNames: []string{bogusIssuerCertDNSName}, IssuerRef: cmmetav1.ObjectReference{ Name: bogusIssuerName, Kind: "ClusterIssuer", Group: "cert-manager.io", }, @@ -519,7 +532,7 @@ func verifyIstioCSROperandResources(ctx context.Context, clientset *kubernetes.C verifyOperatorManagedLabels(crs.Items[0].Labels, "cert-manager-istio-csr") } -func verifyTrustManagerQualificationConfig(ctx context.Context, clientset *kubernetes.Clientset) { +func assertTrustManagerCRConfigPropagation(ctx context.Context, clientset *kubernetes.Clientset) { Eventually(func(g Gomega) { _, err := k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Get(ctx, defaultCAPackageConfigMapName, metav1.GetOptions{}) g.Expect(err).NotTo(HaveOccurred()) @@ -547,10 +560,10 @@ func verifyTrustManagerQualificationConfig(ctx context.Context, clientset *kuber Expect(tm.Spec.TrustManagerConfig.FilterExpiredCertificates).To(Equal(v1alpha1.FilterExpiredCertificatesPolicyEnabled)) } -func runCM786BundleSecretTargetTest(ctx context.Context, sourcePEM string) { +func runMultiOperandBundleSecretTargetTest(ctx context.Context, sourcePEM string) { targetNS, err := k8sClientSet.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "cm786-bundle-target-", + GenerateName: "multi-operand-bundle-target-", Labels: map[string]string{"istio-injection": "enabled"}, }, }, metav1.CreateOptions{}) @@ -560,27 +573,27 @@ func runCM786BundleSecretTargetTest(ctx context.Context, sourcePEM string) { }) _, err = k8sClientSet.CoreV1().ConfigMaps(trustManagerNamespace).Create(ctx, &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: qualificationBundleSourceCM}, - Data: map[string]string{qualificationBundleSourceKey: sourcePEM}, + ObjectMeta: metav1.ObjectMeta{Name: multiOperandBundleSourceCM}, + Data: map[string]string{multiOperandBundleSourceKey: sourcePEM}, }, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - bundle := newBundle(qualificationBundleName). - WithConfigMapSource(qualificationBundleSourceCM, qualificationBundleSourceKey). - WithSecretTarget(qualificationBundleTargetKey). + bundle := newBundle(multiOperandBundleName). + WithConfigMapSource(multiOperandBundleSourceCM, multiOperandBundleSourceKey). + WithSecretTarget(multiOperandBundleTargetKey). WithNamespaceSelector(map[string]string{"istio-injection": "enabled"}). Build() Expect(bundleClient.Create(ctx, bundle)).To(Succeed()) - err = waitForBundleCondition(ctx, bundleClient, qualificationBundleName, trustapi.BundleConditionSynced, metav1.ConditionTrue, highTimeout) + err = waitForBundleCondition(ctx, bundleClient, multiOperandBundleName, trustapi.BundleConditionSynced, metav1.ConditionTrue, highTimeout) Expect(err).NotTo(HaveOccurred()) - err = waitForSecretTarget(ctx, bundleClient, qualificationBundleName, targetNS.Name, qualificationBundleTargetKey, sourcePEM, highTimeout) + err = waitForSecretTarget(ctx, bundleClient, multiOperandBundleName, targetNS.Name, multiOperandBundleTargetKey, sourcePEM, highTimeout) Expect(err).NotTo(HaveOccurred()) } -func runCM786ManagedClusterRoleRecreationTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { +func runMultiOperandManagedClusterRoleRecreationTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { verifyResourceRecreation(ctx, func() error { return clientset.RbacV1().ClusterRoles().Delete(ctx, trustManagerClusterRoleName, metav1.DeleteOptions{}) }, func() error { @@ -627,13 +640,13 @@ func runCM786ManagedClusterRoleRecreationTest(ctx context.Context, clientset *ku Expect(err).NotTo(HaveOccurred()) } -func runCM786UpdateAllOperatorCRsTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { +func runMultiOperandUpdateAllOperatorCRsTest(ctx context.Context, clientset *kubernetes.Clientset, istioNamespace string) { err := retry.RetryOnConflict(retry.DefaultRetry, func() error { cm, getErr := certmanageroperatorclient.OperatorV1alpha1().CertManagers().Get(ctx, "cluster", metav1.GetOptions{}) if getErr != nil { return getErr } - cm.Labels = mergeLabels(cm.Labels, map[string]string{cm786QualificationLabelKey: cm786QualificationLabelValue}) + cm.Labels = mergeLabels(cm.Labels, map[string]string{multiOperandTestLabelKey: multiOperandTestLabelValue}) _, updateErr := certmanageroperatorclient.OperatorV1alpha1().CertManagers().Update(ctx, cm, metav1.UpdateOptions{}) return updateErr }) @@ -644,7 +657,7 @@ func runCM786UpdateAllOperatorCRsTest(ctx context.Context, clientset *kubernetes if getErr != nil { return getErr } - tm.Labels = mergeLabels(tm.Labels, map[string]string{cm786QualificationLabelKey: cm786QualificationLabelValue}) + tm.Labels = mergeLabels(tm.Labels, map[string]string{multiOperandTestLabelKey: multiOperandTestLabelValue}) _, updateErr := trustManagerClient().Update(ctx, tm, metav1.UpdateOptions{}) return updateErr }) @@ -655,7 +668,7 @@ func runCM786UpdateAllOperatorCRsTest(ctx context.Context, clientset *kubernetes if getErr != nil { return getErr } - istioCSR.Labels = mergeLabels(istioCSR.Labels, map[string]string{cm786QualificationLabelKey: cm786QualificationLabelValue}) + istioCSR.Labels = mergeLabels(istioCSR.Labels, map[string]string{multiOperandTestLabelKey: multiOperandTestLabelValue}) _, updateErr := certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Update(ctx, istioCSR, metav1.UpdateOptions{}) return updateErr }) @@ -666,7 +679,7 @@ func runCM786UpdateAllOperatorCRsTest(ctx context.Context, clientset *kubernetes waitForOperandsReadyConcurrently(ctx, clientset, istioNamespace) } -func runCM786IstioCSRSpecUpdateTest(ctx context.Context, istioNamespace string) { +func runMultiOperandIstioCSRSpecUpdateTest(ctx context.Context, istioNamespace string) { for _, level := range []int32{3, 2, 1} { level := level err := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -675,7 +688,7 @@ func runCM786IstioCSRSpecUpdateTest(ctx context.Context, istioNamespace string) return getErr } istioCSR.Spec.IstioCSRConfig.LogLevel = level - istioCSR.Labels = mergeLabels(istioCSR.Labels, map[string]string{"test": fmt.Sprintf("cm786-log-%d", level)}) + istioCSR.Labels = mergeLabels(istioCSR.Labels, map[string]string{"test": fmt.Sprintf("multi-operand-log-%d", level)}) _, updateErr := certmanageroperatorclient.OperatorV1alpha1().IstioCSRs(istioNamespace).Update(ctx, istioCSR, metav1.UpdateOptions{}) return updateErr }) @@ -693,7 +706,7 @@ func runCM786IstioCSRSpecUpdateTest(ctx context.Context, istioNamespace string) } } -func runCM786IstioCSRGRPCCertificateTest(ctx context.Context, clientset *kubernetes.Clientset, ns *corev1.Namespace) { +func runMultiOperandIstioCSRGRPCCertificateTest(ctx context.Context, clientset *kubernetes.Clientset, ns *corev1.Namespace) { const grpcAppName = "grpcurl-istio-csr" istioCSRStatus, err := pollTillIstioCSRAvailable(ctx, loader, ns.Name, istioCSRResourceName) @@ -713,7 +726,7 @@ func runCM786IstioCSRGRPCCertificateTest(ctx context.Context, clientset *kuberne _ = clientset.CoreV1().ConfigMaps(ns.Name).Delete(ctx, "proto-cm", metav1.DeleteOptions{}) }) - csr := generateCM786CSR(ns.Name) + csr := generateMultiOperandIstioCSRCSR(ns.Name) loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( IstioCSRGRPCurlJobConfig{ CertificateSigningRequest: csr, @@ -756,19 +769,19 @@ func runCM786IstioCSRGRPCCertificateTest(ctx context.Context, clientset *kuberne } } -func runCM786ServiceMeshIstioCSRIntegrationTest(ctx context.Context, clientset *kubernetes.Clientset, istioCSRNamespace string) { +func runMultiOperandServiceMeshIstioCSRIntegrationTest(ctx context.Context, clientset *kubernetes.Clientset, istioCSRNamespace string) { istioGVR := schema.GroupVersionResource{Group: "sailoperator.io", Version: "v1", Resource: "istios"} istioCR, err := loader.DynamicClient.Resource(istioGVR).Get(ctx, "default", metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { - Skip("OpenShift Service Mesh (sailoperator.io/v1 Istio/default) is not installed; CM-786 steps 6-7 require manual mesh qualification") + Skip("OpenShift Service Mesh (sailoperator.io/v1 Istio/default) is not installed; skipping Service Mesh istio-csr integration check") } Expect(err).NotTo(HaveOccurred()) } caAddress, found, err := unstructuredNestedString(istioCR.Object, "spec", "values", "global", "caAddress") if err != nil || !found { - Skip("Service Mesh Istio CR does not expose spec.values.global.caAddress; skipping CM-786 step 7 mesh CA check") + Skip("Service Mesh Istio CR does not expose spec.values.global.caAddress; skipping mesh CA address check") } Expect(caAddress).To(ContainSubstring("cert-manager-istio-csr")) @@ -783,11 +796,11 @@ func runCM786ServiceMeshIstioCSRIntegrationTest(ctx context.Context, clientset * }, lowTimeout, fastPollInterval).Should(Succeed()) } -func generateCM786CSR(istioNamespace string) string { +func generateMultiOperandIstioCSRCSR(istioNamespace string) string { csrTemplate := &x509.CertificateRequest{ Subject: pkix.Name{ - Organization: []string{"CM-786"}, - OrganizationalUnit: []string{"cert-manager-operator-e2e"}, + Organization: []string{"cert-manager-operator-e2e"}, + OrganizationalUnit: []string{"multi-operand"}, Country: []string{"US"}, }, URIs: []*url.URL{ diff --git a/test/e2e/trustmanager_helpers_test.go b/test/e2e/trustmanager_helpers_test.go index 9961c8169..6a542594c 100644 --- a/test/e2e/trustmanager_helpers_test.go +++ b/test/e2e/trustmanager_helpers_test.go @@ -130,6 +130,21 @@ func deleteTrustManager(ctx context.Context) { _, err := trustManagerClient().Get(ctx, "cluster", metav1.GetOptions{}) return apierrors.IsNotFound(err) }, lowTimeout, fastPollInterval).Should(BeTrue()) + + cleanupTrustManagerOperandLeavings(ctx) +} + +// cleanupTrustManagerOperandLeavings removes operand resources left behind when the TrustManager CR +// is deleted. The operator does not tear these down today (see controller cleanUp TODO). +func cleanupTrustManagerOperandLeavings(ctx context.Context) { + By("cleaning up trust-manager operand ServiceAccount if present") + _ = k8sClientSet.CoreV1().ServiceAccounts(trustManagerNamespace).Delete(ctx, trustManagerServiceAccountName, metav1.DeleteOptions{}) + Eventually(func() bool { + _, err := k8sClientSet.CoreV1().ServiceAccounts(trustManagerNamespace).Get(ctx, trustManagerServiceAccountName, metav1.GetOptions{}) + return apierrors.IsNotFound(err) + }, lowTimeout, fastPollInterval).Should(BeTrue()) + + deleteTrustManagerDefaultCAPackageConfigMap(ctx) } // deleteTrustManagerDefaultCAPackageConfigMap removes the operand ConfigMap created when diff --git a/test/e2e/trustmanager_test.go b/test/e2e/trustmanager_test.go index ebfdf525c..0b7026e7b 100644 --- a/test/e2e/trustmanager_test.go +++ b/test/e2e/trustmanager_test.go @@ -1638,29 +1638,12 @@ func findSecretRule(rules []rbacv1.PolicyRule, verb string) *rbacv1.PolicyRule { return nil } -// trustManagerRemoveStaleOperandServiceAccount deletes the trust-manager operand ServiceAccount if it -// exists and waits until it is gone. TrustManager CR deletion does not remove this SA today, so a -// prior e2e run can leave it behind and affect later tests. -// -// TODO: Remove this when TrustManager is GA and operand teardown removes the ServiceAccount (or -// equivalent) when the TrustManager CR is deleted. -func trustManagerRemoveStaleOperandServiceAccount(ctx context.Context, cs *kubernetes.Clientset) { - By("removing stale trust-manager ServiceAccount if present") - _ = cs.CoreV1().ServiceAccounts(trustManagerNamespace).Delete(ctx, trustManagerServiceAccountName, metav1.DeleteOptions{}) - Eventually(func() bool { - _, err := cs.CoreV1().ServiceAccounts(trustManagerNamespace).Get(ctx, trustManagerServiceAccountName, metav1.GetOptions{}) - return apierrors.IsNotFound(err) - }, lowTimeout, fastPollInterval).Should(BeTrue()) -} - func trustManagerFeatureGateDisabledBeforeAll(ctx context.Context, clientset **kubernetes.Clientset, unsupportedAddonFeatures, operatorLogLevel *string) func() { return func() { cs, err := kubernetes.NewForConfig(cfg) Expect(err).Should(BeNil()) *clientset = cs - trustManagerRemoveStaleOperandServiceAccount(ctx, cs) - By("capturing original UNSUPPORTED_ADDON_FEATURES from subscription before patching") *unsupportedAddonFeatures, err = getSubscriptionEnvVar(ctx, loader, "UNSUPPORTED_ADDON_FEATURES") Expect(err).NotTo(HaveOccurred()) @@ -1690,8 +1673,6 @@ func trustManagerBeforeAll(ctx context.Context, clientset **kubernetes.Clientset Expect(err).Should(BeNil()) *clientset = cs - trustManagerRemoveStaleOperandServiceAccount(ctx, cs) - By("capturing original UNSUPPORTED_ADDON_FEATURES from subscription before patching") *unsupportedAddonFeatures, err = getSubscriptionEnvVar(ctx, loader, "UNSUPPORTED_ADDON_FEATURES") Expect(err).NotTo(HaveOccurred()) @@ -1754,6 +1735,6 @@ func trustManagerAfterEach(ctx context.Context) func() { return apierrors.IsNotFound(err) }, lowTimeout, fastPollInterval).Should(BeTrue()) - deleteTrustManagerDefaultCAPackageConfigMap(ctx) + cleanupTrustManagerOperandLeavings(ctx) } } diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 3aaabb5cf..6115bec6b 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -2097,15 +2097,15 @@ func formatVaultPodsStatus(pods []corev1.Pod) string { case cs.State.Running != nil: b.WriteString(" state=running") case cs.State.Waiting != nil: - fmt.Fprintf(&b, " state=waiting reason=%s", cs.State.Waiting.Reason) + fmt.Fprintf(&b, " state=waiting reason=%s message=%q", cs.State.Waiting.Reason, cs.State.Waiting.Message) case cs.State.Terminated != nil: - fmt.Fprintf(&b, " state=terminated reason=%s exitCode=%d", - cs.State.Terminated.Reason, cs.State.Terminated.ExitCode) + fmt.Fprintf(&b, " state=terminated reason=%s exitCode=%d message=%q", + cs.State.Terminated.Reason, cs.State.Terminated.ExitCode, cs.State.Terminated.Message) } } for _, cond := range pod.Status.Conditions { if cond.Status != corev1.ConditionTrue { - fmt.Fprintf(&b, " condition=%s:%s", cond.Type, cond.Status) + fmt.Fprintf(&b, " condition=%s:%s:%s", cond.Type, cond.Status, cond.Message) } } b.WriteString("; ")