diff --git a/Makefile b/Makefile index 3950e126d..d9894f831 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,6 @@ GO_TEST_PACKAGES :=./pkg/... ./cmd/... $(call build-image,ocp-cluster-openshift-controller-manager-operator,$(IMAGE_REGISTRY)/ocp/4.3:cluster-openshift-controller-manager-operator,./Dockerfile,.) test-e2e: GO_TEST_PACKAGES :=./test/e2e/... -test-e2e: GO_TEST_FLAGS += -v -count=1 +test-e2e: GO_TEST_FLAGS += -v -count=1 -timeout=20m test-e2e: test-unit .PHONY: test-e2e diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index dcf3043b4..a6e276f3a 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -1,4 +1,4 @@ -package e2e_test +package e2e import ( "os" diff --git a/test/e2e/network_policy_enforcement_test.go b/test/e2e/network_policy_enforcement_test.go new file mode 100644 index 000000000..5b7f1477e --- /dev/null +++ b/test/e2e/network_policy_enforcement_test.go @@ -0,0 +1,464 @@ +package e2e + +import ( + "context" + "fmt" + "net" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + "github.com/openshift/cluster-openshift-controller-manager-operator/pkg/util" +) + +const ( + agnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.45" +) + +// Import constants from network_policy_test.go - these are defined there: +// - defaultDenyAllPolicyName +// - controllerManagerPolicyName +// - routeControllerManagerPolicyName +// - operatorPolicyName + +func TestGenericNetworkPolicyEnforcement(t *testing.T) { + kubeConfig, err := getKubeConfig() + if err != nil { + t.Fatalf("failed to get kubeconfig: %v", err) + } + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Fatalf("failed to create kubernetes client: %v", err) + } + + t.Log("Creating a temporary namespace for policy enforcement checks") + nsName := "np-enforcement-" + rand.String(5) + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}} + _, err = kubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create test namespace: %v", err) + } + defer func() { + t.Logf("deleting test namespace %s", nsName) + _ = kubeClient.CoreV1().Namespaces().Delete(context.TODO(), nsName, metav1.DeleteOptions{}) + }() + + serverName := "np-server" + clientLabels := map[string]string{"app": "np-client"} + serverLabels := map[string]string{"app": "np-server"} + + t.Logf("creating netexec server pod %s/%s", nsName, serverName) + serverPod := netexecPod(serverName, nsName, serverLabels, 8080) + _, err = kubeClient.CoreV1().Pods(nsName).Create(context.TODO(), serverPod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create server pod: %v", err) + } + if err := waitForPodReadyT(t, kubeClient, nsName, serverName); err != nil { + t.Fatalf("server pod not ready: %v", err) + } + + server, err := kubeClient.CoreV1().Pods(nsName).Get(context.TODO(), serverName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get server pod: %v", err) + } + if len(server.Status.PodIPs) == 0 { + t.Fatalf("server pod has no IPs") + } + serverIPs := podIPs(server) + t.Logf("server pod %s/%s ips=%v", nsName, serverName, serverIPs) + + t.Log("Verifying allow-all when no policies select the pod") + expectConnectivity(t, kubeClient, nsName, clientLabels, serverIPs, 8080, true) + + t.Log("Applying default deny and verifying traffic is blocked") + t.Logf("creating default-deny policy in %s", nsName) + _, err = kubeClient.NetworkingV1().NetworkPolicies(nsName).Create(context.TODO(), defaultDenyPolicy("default-deny", nsName), metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create default-deny policy: %v", err) + } + + t.Log("Adding ingress allow only and verifying traffic is still blocked") + t.Logf("creating allow-ingress policy in %s", nsName) + _, err = kubeClient.NetworkingV1().NetworkPolicies(nsName).Create(context.TODO(), allowIngressPolicy("allow-ingress", nsName, serverLabels, clientLabels, 8080), metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create allow-ingress policy: %v", err) + } + expectConnectivity(t, kubeClient, nsName, clientLabels, serverIPs, 8080, false) + + t.Log("Adding egress allow and verifying traffic is permitted") + t.Logf("creating allow-egress policy in %s", nsName) + _, err = kubeClient.NetworkingV1().NetworkPolicies(nsName).Create(context.TODO(), allowEgressPolicy("allow-egress", nsName, clientLabels, serverLabels, 8080), metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create allow-egress policy: %v", err) + } + expectConnectivity(t, kubeClient, nsName, clientLabels, serverIPs, 8080, true) +} + +func TestControllerManagerNetworkPolicyEnforcement(t *testing.T) { + kubeConfig, err := getKubeConfig() + if err != nil { + t.Fatalf("failed to get kubeconfig: %v", err) + } + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Fatalf("failed to create kubernetes client: %v", err) + } + + // Labels must match the NetworkPolicy pod selectors for egress to work + controllerManagerLabels := map[string]string{ + "app": "openshift-controller-manager-a", + "controller-manager": "true", + } + routeControllerManagerLabels := map[string]string{ + "app": "route-controller-manager", + "route-controller-manager": "true", + } + operatorLabels := map[string]string{"app": "openshift-controller-manager-operator"} + + t.Log("Verifying controller manager NetworkPolicies exist") + _, err = kubeClient.NetworkingV1().NetworkPolicies(util.TargetNamespace).Get(context.TODO(), controllerManagerPolicyName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get controller manager NetworkPolicy: %v", err) + } + _, err = kubeClient.NetworkingV1().NetworkPolicies(util.RouteControllerTargetNamespace).Get(context.TODO(), routeControllerManagerPolicyName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get route controller manager NetworkPolicy: %v", err) + } + _, err = kubeClient.NetworkingV1().NetworkPolicies(util.OperatorNamespace).Get(context.TODO(), operatorPolicyName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get operator NetworkPolicy: %v", err) + } + + t.Log("Creating test pods in openshift-controller-manager-operator for allow/deny checks") + t.Logf("creating operator server pods in %s", util.OperatorNamespace) + allowedServerIPs, cleanupAllowed := createServerPodT(t, kubeClient, util.OperatorNamespace, "np-operator-allowed", operatorLabels, 8443) + defer cleanupAllowed() + deniedServerIPs, cleanupDenied := createServerPodT(t, kubeClient, util.OperatorNamespace, "np-operator-denied", operatorLabels, 12345) + defer cleanupDenied() + + t.Log("Verifying allowed port 8443 ingress to operator") + expectConnectivity(t, kubeClient, util.OperatorNamespace, operatorLabels, allowedServerIPs, 8443, true) + + t.Log("Verifying denied port 12345 (not in NetworkPolicy)") + expectConnectivity(t, kubeClient, util.OperatorNamespace, operatorLabels, deniedServerIPs, 12345, false) + + t.Log("Verifying denied ports even from same namespace") + for _, port := range []int32{12346, 12347, 12348, 12349} { + ips, cleanup := createServerPodT(t, kubeClient, util.OperatorNamespace, fmt.Sprintf("np-operator-denied-%d", port), operatorLabels, port) + defer cleanup() + expectConnectivity(t, kubeClient, util.OperatorNamespace, operatorLabels, ips, port, false) + } + + t.Log("Verifying operator egress to DNS") + dnsSvc, err := kubeClient.CoreV1().Services("openshift-dns").Get(context.TODO(), "dns-default", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get DNS service: %v", err) + } + dnsIPs := serviceClusterIPs(dnsSvc) + t.Logf("expecting allow from %s to DNS %v:53", util.OperatorNamespace, dnsIPs) + expectConnectivity(t, kubeClient, util.OperatorNamespace, operatorLabels, dnsIPs, 53, true) + + t.Log("Verifying controller manager pods egress to DNS") + expectConnectivity(t, kubeClient, util.TargetNamespace, controllerManagerLabels, dnsIPs, 53, true) + + t.Log("Verifying route controller manager pods egress to DNS") + expectConnectivity(t, kubeClient, util.RouteControllerTargetNamespace, routeControllerManagerLabels, dnsIPs, 53, true) +} + +func netexecPod(name, namespace string, labels map[string]string, port int32) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{ + { + Name: "netexec", + Image: agnhostImage, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolptr(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + }, + Command: []string{"/agnhost"}, + Args: []string{"netexec", fmt.Sprintf("--http-port=%d", port)}, + Ports: []corev1.ContainerPort{ + {ContainerPort: port}, + }, + }, + }, + }, + } +} + +func createServerPodT(t *testing.T, kubeClient kubernetes.Interface, namespace, name string, labels map[string]string, port int32) ([]string, func()) { + t.Helper() + + t.Logf("creating server pod %s/%s port=%d labels=%v", namespace, name, port, labels) + pod := netexecPod(name, namespace, labels, port) + _, err := kubeClient.CoreV1().Pods(namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create server pod: %v", err) + } + if err := waitForPodReadyT(t, kubeClient, namespace, name); err != nil { + t.Fatalf("server pod not ready: %v", err) + } + + created, err := kubeClient.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get created server pod: %v", err) + } + if len(created.Status.PodIPs) == 0 { + t.Fatalf("server pod has no IPs") + } + + ips := podIPs(created) + t.Logf("server pod %s/%s ips=%v", namespace, name, ips) + + return ips, func() { + t.Logf("deleting server pod %s/%s", namespace, name) + _ = kubeClient.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + } +} + +// podIPs returns all IP addresses assigned to a pod (dual-stack aware). +func podIPs(pod *corev1.Pod) []string { + var ips []string + for _, podIP := range pod.Status.PodIPs { + if podIP.IP != "" { + ips = append(ips, podIP.IP) + } + } + if len(ips) == 0 && pod.Status.PodIP != "" { + ips = append(ips, pod.Status.PodIP) + } + return ips +} + +// isIPv6 returns true if the given IP string is an IPv6 address. +func isIPv6(ip string) bool { + return net.ParseIP(ip) != nil && strings.Contains(ip, ":") +} + +// formatIPPort formats an IP:port pair, using brackets for IPv6 addresses. +func formatIPPort(ip string, port int32) string { + if isIPv6(ip) { + return fmt.Sprintf("[%s]:%d", ip, port) + } + return fmt.Sprintf("%s:%d", ip, port) +} + +// serviceClusterIPs returns all ClusterIPs for a service (dual-stack aware). +func serviceClusterIPs(svc *corev1.Service) []string { + if len(svc.Spec.ClusterIPs) > 0 { + return svc.Spec.ClusterIPs + } + if svc.Spec.ClusterIP != "" { + return []string{svc.Spec.ClusterIP} + } + return nil +} + +func defaultDenyPolicy(name, namespace string) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, + }, + } +} + +func allowIngressPolicy(name, namespace string, podLabels, fromLabels map[string]string, port int32) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: podLabels}, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &metav1.LabelSelector{MatchLabels: fromLabels}}, + }, + Ports: []networkingv1.NetworkPolicyPort{ + {Port: &intstr.IntOrString{Type: intstr.Int, IntVal: port}, Protocol: protocolPtr(corev1.ProtocolTCP)}, + }, + }, + }, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + }, + } +} + +func allowEgressPolicy(name, namespace string, podLabels, toLabels map[string]string, port int32) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: podLabels}, + Egress: []networkingv1.NetworkPolicyEgressRule{ + { + To: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &metav1.LabelSelector{MatchLabels: toLabels}}, + }, + Ports: []networkingv1.NetworkPolicyPort{ + {Port: &intstr.IntOrString{Type: intstr.Int, IntVal: port}, Protocol: protocolPtr(corev1.ProtocolTCP)}, + }, + }, + }, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + }, + } +} + +// expectConnectivityForIP checks connectivity to a single IP address. +func expectConnectivityForIP(t *testing.T, kubeClient kubernetes.Interface, namespace string, clientLabels map[string]string, serverIP string, port int32, shouldSucceed bool) { + t.Helper() + + err := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + succeeded, err := runConnectivityCheck(t, kubeClient, namespace, clientLabels, serverIP, port) + if err != nil { + return false, err + } + return succeeded == shouldSucceed, nil + }) + if err != nil { + t.Fatalf("connectivity check failed for %s/%s expected=%t: %v", namespace, formatIPPort(serverIP, port), shouldSucceed, err) + } + t.Logf("connectivity %s/%s expected=%t", namespace, formatIPPort(serverIP, port), shouldSucceed) +} + +// expectConnectivity checks connectivity to all provided IPs (dual-stack aware). +func expectConnectivity(t *testing.T, kubeClient kubernetes.Interface, namespace string, clientLabels map[string]string, serverIPs []string, port int32, shouldSucceed bool) { + t.Helper() + + for _, ip := range serverIPs { + family := "IPv4" + if isIPv6(ip) { + family = "IPv6" + } + t.Logf("checking %s connectivity %s -> %s expected=%t", family, namespace, formatIPPort(ip, port), shouldSucceed) + expectConnectivityForIP(t, kubeClient, namespace, clientLabels, ip, port, shouldSucceed) + } +} + +func runConnectivityCheck(t *testing.T, kubeClient kubernetes.Interface, namespace string, labels map[string]string, serverIP string, port int32) (bool, error) { + t.Helper() + + name := fmt.Sprintf("np-client-%s", rand.String(5)) + t.Logf("creating client pod %s/%s to connect %s:%d", namespace, name, serverIP, port) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{ + { + Name: "connect", + Image: agnhostImage, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolptr(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + }, + Command: []string{"/agnhost"}, + Args: []string{ + "connect", + "--protocol=tcp", + "--timeout=5s", + formatIPPort(serverIP, port), + }, + }, + }, + }, + } + + _, err := kubeClient.CoreV1().Pods(namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { + return false, err + } + defer func() { + _ = kubeClient.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + }() + + if err := waitForPodCompletion(kubeClient, namespace, name); err != nil { + return false, err + } + completed, err := kubeClient.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if len(completed.Status.ContainerStatuses) == 0 { + return false, fmt.Errorf("no container status recorded for pod %s", name) + } + state := completed.Status.ContainerStatuses[0].State + if state.Terminated == nil { + return false, fmt.Errorf("pod %s completed without a terminated container state: phase=%s reason=%s", name, completed.Status.Phase, completed.Status.Reason) + } + exitCode := state.Terminated.ExitCode + t.Logf("client pod %s/%s exitCode=%d", namespace, name, exitCode) + return exitCode == 0, nil +} + +func waitForPodReadyT(t *testing.T, kubeClient kubernetes.Interface, namespace, name string) error { + return wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := kubeClient.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if pod.Status.Phase != corev1.PodRunning { + return false, nil + } + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) +} + +func waitForPodCompletion(kubeClient kubernetes.Interface, namespace, name string) error { + return wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := kubeClient.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed, nil + }) +} + +func protocolPtr(protocol corev1.Protocol) *corev1.Protocol { + return &protocol +} + +func boolptr(value bool) *bool { + return &value +} + +func int64ptr(value int64) *int64 { + return &value +} diff --git a/test/e2e/network_policy_test.go b/test/e2e/network_policy_test.go new file mode 100644 index 000000000..176b51cc3 --- /dev/null +++ b/test/e2e/network_policy_test.go @@ -0,0 +1,434 @@ +package e2e + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/openshift/cluster-openshift-controller-manager-operator/pkg/util" +) + +const ( + defaultDenyAllPolicyName = "default-deny" + controllerManagerPolicyName = "allow-controller-manager" + routeControllerManagerPolicyName = "allow-route-controller-manager" + operatorPolicyName = "allow-operator" +) + +func TestControllerManagerNetworkPolicies(t *testing.T) { + ctx := context.Background() + t.Log("Creating Kubernetes clients") + kubeConfig, err := getKubeConfig() + if err != nil { + t.Fatalf("failed to get kubeconfig: %v", err) + } + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Fatalf("failed to create kubernetes client: %v", err) + } + + t.Log("Validating NetworkPolicies in openshift-controller-manager") + controllerManagerDefaultDeny := getNetworkPolicyT(t, ctx, kubeClient, util.TargetNamespace, defaultDenyAllPolicyName) + logNetworkPolicySummary(t, "controller-manager/default-deny-all", controllerManagerDefaultDeny) + logNetworkPolicyDetails(t, "controller-manager/default-deny-all", controllerManagerDefaultDeny) + requireDefaultDenyAll(t, controllerManagerDefaultDeny) + + controllerManagerPolicy := getNetworkPolicyT(t, ctx, kubeClient, util.TargetNamespace, controllerManagerPolicyName) + logNetworkPolicySummary(t, "controller-manager/allow-controller-manager", controllerManagerPolicy) + logNetworkPolicyDetails(t, "controller-manager/allow-controller-manager", controllerManagerPolicy) + requirePodSelectorHasLabel(t, controllerManagerPolicy, "controller-manager") + requireIngressPort(t, controllerManagerPolicy, corev1.ProtocolTCP, 8443) + logEgressAllowAll(t, controllerManagerPolicy) + + t.Log("Validating NetworkPolicies in openshift-route-controller-manager") + routeControllerManagerDefaultDeny := getNetworkPolicyT(t, ctx, kubeClient, util.RouteControllerTargetNamespace, defaultDenyAllPolicyName) + logNetworkPolicySummary(t, "route-controller-manager/default-deny-all", routeControllerManagerDefaultDeny) + logNetworkPolicyDetails(t, "route-controller-manager/default-deny-all", routeControllerManagerDefaultDeny) + requireDefaultDenyAll(t, routeControllerManagerDefaultDeny) + + routeControllerManagerPolicy := getNetworkPolicyT(t, ctx, kubeClient, util.RouteControllerTargetNamespace, routeControllerManagerPolicyName) + logNetworkPolicySummary(t, "route-controller-manager/allow-route-controller-manager", routeControllerManagerPolicy) + logNetworkPolicyDetails(t, "route-controller-manager/allow-route-controller-manager", routeControllerManagerPolicy) + requirePodSelectorHasLabel(t, routeControllerManagerPolicy, "route-controller-manager") + requireIngressPort(t, routeControllerManagerPolicy, corev1.ProtocolTCP, 8443) + logEgressAllowAll(t, routeControllerManagerPolicy) + + t.Log("Validating NetworkPolicies in openshift-controller-manager-operator") + operatorDefaultDeny := getNetworkPolicyT(t, ctx, kubeClient, util.OperatorNamespace, defaultDenyAllPolicyName) + logNetworkPolicySummary(t, "operator/default-deny-all", operatorDefaultDeny) + logNetworkPolicyDetails(t, "operator/default-deny-all", operatorDefaultDeny) + requireDefaultDenyAll(t, operatorDefaultDeny) + + operatorPolicy := getNetworkPolicyT(t, ctx, kubeClient, util.OperatorNamespace, operatorPolicyName) + logNetworkPolicySummary(t, "operator/allow-operator", operatorPolicy) + logNetworkPolicyDetails(t, "operator/allow-operator", operatorPolicy) + requirePodSelectorLabel(t, operatorPolicy, "app", "openshift-controller-manager-operator") + requireIngressPort(t, operatorPolicy, corev1.ProtocolTCP, 8443) + logEgressAllowAll(t, operatorPolicy) + + t.Log("Verifying pods are ready in controller manager namespaces") + waitForPodsReadyByLabel(t, ctx, kubeClient, util.TargetNamespace, "controller-manager=true") + waitForPodsReadyByLabel(t, ctx, kubeClient, util.RouteControllerTargetNamespace, "route-controller-manager=true") + waitForPodsReadyByLabel(t, ctx, kubeClient, util.OperatorNamespace, "app=openshift-controller-manager-operator") +} + +func TestControllerManagerNetworkPolicyReconcile(t *testing.T) { + ctx := context.Background() + t.Log("Creating Kubernetes clients") + kubeConfig, err := getKubeConfig() + if err != nil { + t.Fatalf("failed to get kubeconfig: %v", err) + } + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Fatalf("failed to create kubernetes client: %v", err) + } + + t.Log("Capturing expected NetworkPolicy specs") + expectedControllerManagerDefaultDeny := getNetworkPolicyT(t, ctx, kubeClient, util.TargetNamespace, defaultDenyAllPolicyName) + expectedControllerManagerPolicy := getNetworkPolicyT(t, ctx, kubeClient, util.TargetNamespace, controllerManagerPolicyName) + expectedRouteControllerManagerDefaultDeny := getNetworkPolicyT(t, ctx, kubeClient, util.RouteControllerTargetNamespace, defaultDenyAllPolicyName) + expectedRouteControllerManagerPolicy := getNetworkPolicyT(t, ctx, kubeClient, util.RouteControllerTargetNamespace, routeControllerManagerPolicyName) + expectedOperatorDefaultDeny := getNetworkPolicyT(t, ctx, kubeClient, util.OperatorNamespace, defaultDenyAllPolicyName) + expectedOperatorPolicy := getNetworkPolicyT(t, ctx, kubeClient, util.OperatorNamespace, operatorPolicyName) + + t.Log("Deleting main policies and waiting for restoration") + t.Logf("deleting NetworkPolicy %s/%s", util.TargetNamespace, controllerManagerPolicyName) + restoreNetworkPolicy(t, ctx, kubeClient, expectedControllerManagerPolicy) + t.Logf("deleting NetworkPolicy %s/%s", util.RouteControllerTargetNamespace, routeControllerManagerPolicyName) + restoreNetworkPolicy(t, ctx, kubeClient, expectedRouteControllerManagerPolicy) + t.Logf("deleting NetworkPolicy %s/%s (operator namespace may need longer to reconcile)", util.OperatorNamespace, operatorPolicyName) + restoreNetworkPolicyWithTimeout(t, ctx, kubeClient, expectedOperatorPolicy, 15*time.Minute) + + t.Log("Deleting default-deny-all policies and waiting for restoration") + t.Logf("deleting NetworkPolicy %s/%s", util.TargetNamespace, defaultDenyAllPolicyName) + restoreNetworkPolicy(t, ctx, kubeClient, expectedControllerManagerDefaultDeny) + t.Logf("deleting NetworkPolicy %s/%s", util.RouteControllerTargetNamespace, defaultDenyAllPolicyName) + restoreNetworkPolicy(t, ctx, kubeClient, expectedRouteControllerManagerDefaultDeny) + t.Logf("deleting NetworkPolicy %s/%s (operator namespace may need longer to reconcile)", util.OperatorNamespace, defaultDenyAllPolicyName) + restoreNetworkPolicyWithTimeout(t, ctx, kubeClient, expectedOperatorDefaultDeny, 15*time.Minute) + + t.Log("Mutating main policies and waiting for reconciliation") + t.Logf("mutating NetworkPolicy %s/%s", util.TargetNamespace, controllerManagerPolicyName) + mutateAndRestoreNetworkPolicy(t, ctx, kubeClient, util.TargetNamespace, controllerManagerPolicyName) + t.Logf("mutating NetworkPolicy %s/%s", util.RouteControllerTargetNamespace, routeControllerManagerPolicyName) + mutateAndRestoreNetworkPolicy(t, ctx, kubeClient, util.RouteControllerTargetNamespace, routeControllerManagerPolicyName) + t.Logf("mutating NetworkPolicy %s/%s (operator namespace may need longer to reconcile)", util.OperatorNamespace, operatorPolicyName) + mutateAndRestoreNetworkPolicyWithTimeout(t, ctx, kubeClient, util.OperatorNamespace, operatorPolicyName, 15*time.Minute) + + t.Log("Mutating default-deny-all policies and waiting for reconciliation") + t.Logf("mutating NetworkPolicy %s/%s", util.TargetNamespace, defaultDenyAllPolicyName) + mutateAndRestoreNetworkPolicy(t, ctx, kubeClient, util.TargetNamespace, defaultDenyAllPolicyName) + t.Logf("mutating NetworkPolicy %s/%s", util.RouteControllerTargetNamespace, defaultDenyAllPolicyName) + mutateAndRestoreNetworkPolicy(t, ctx, kubeClient, util.RouteControllerTargetNamespace, defaultDenyAllPolicyName) + t.Logf("mutating NetworkPolicy %s/%s (operator namespace may need longer to reconcile)", util.OperatorNamespace, defaultDenyAllPolicyName) + mutateAndRestoreNetworkPolicyWithTimeout(t, ctx, kubeClient, util.OperatorNamespace, defaultDenyAllPolicyName, 15*time.Minute) + + t.Log("Checking NetworkPolicy-related events (best-effort)") + logNetworkPolicyEvents(t, ctx, kubeClient, []string{util.OperatorNamespace, util.TargetNamespace, util.RouteControllerTargetNamespace}, controllerManagerPolicyName) +} + +func getKubeConfig() (*restclient.Config, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + return kubeConfig.ClientConfig() +} + +func getNetworkPolicyT(t *testing.T, ctx context.Context, client kubernetes.Interface, namespace, name string) *networkingv1.NetworkPolicy { + t.Helper() + policy, err := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get NetworkPolicy %s/%s: %v", namespace, name, err) + } + return policy +} + +func requireDefaultDenyAll(t *testing.T, policy *networkingv1.NetworkPolicy) { + t.Helper() + if len(policy.Spec.PodSelector.MatchLabels) != 0 || len(policy.Spec.PodSelector.MatchExpressions) != 0 { + t.Errorf("%s/%s: expected empty podSelector", policy.Namespace, policy.Name) + } + if len(policy.Spec.Ingress) != 0 || len(policy.Spec.Egress) != 0 { + t.Errorf("%s/%s: expected no ingress/egress rules, got ingress=%d egress=%d", policy.Namespace, policy.Name, len(policy.Spec.Ingress), len(policy.Spec.Egress)) + } + + policyTypes := sets.NewString() + for _, policyType := range policy.Spec.PolicyTypes { + policyTypes.Insert(string(policyType)) + } + if !policyTypes.Has(string(networkingv1.PolicyTypeIngress)) || !policyTypes.Has(string(networkingv1.PolicyTypeEgress)) { + t.Errorf("%s/%s: expected both Ingress and Egress policyTypes, got %v", policy.Namespace, policy.Name, policy.Spec.PolicyTypes) + } +} + +func requirePodSelectorLabel(t *testing.T, policy *networkingv1.NetworkPolicy, key, value string) { + t.Helper() + actual, ok := policy.Spec.PodSelector.MatchLabels[key] + if !ok || actual != value { + t.Errorf("%s/%s: expected podSelector %s=%s, got %v", policy.Namespace, policy.Name, key, value, policy.Spec.PodSelector.MatchLabels) + } +} + +func requirePodSelectorHasLabel(t *testing.T, policy *networkingv1.NetworkPolicy, key string) { + t.Helper() + if _, ok := policy.Spec.PodSelector.MatchLabels[key]; !ok { + t.Errorf("%s/%s: expected podSelector to have label %s, got %v", policy.Namespace, policy.Name, key, policy.Spec.PodSelector.MatchLabels) + } +} + +func requireIngressPort(t *testing.T, policy *networkingv1.NetworkPolicy, protocol corev1.Protocol, port int32) { + t.Helper() + if !hasPortInIngress(policy.Spec.Ingress, protocol, port) { + t.Errorf("%s/%s: expected ingress port %s/%d", policy.Namespace, policy.Name, protocol, port) + } +} + +func hasPortInIngress(rules []networkingv1.NetworkPolicyIngressRule, protocol corev1.Protocol, port int32) bool { + for _, rule := range rules { + if hasPort(rule.Ports, protocol, port) { + return true + } + } + return false +} + +func hasPort(ports []networkingv1.NetworkPolicyPort, protocol corev1.Protocol, port int32) bool { + for _, p := range ports { + if p.Port == nil || p.Port.IntValue() != int(port) { + continue + } + if p.Protocol == nil || *p.Protocol == protocol { + return true + } + } + return false +} + +func logEgressAllowAll(t *testing.T, policy *networkingv1.NetworkPolicy) { + t.Helper() + if hasEgressAllowAll(policy.Spec.Egress) { + t.Logf("networkpolicy %s/%s: egress allow-all rule present", policy.Namespace, policy.Name) + return + } + t.Logf("networkpolicy %s/%s: no egress allow-all rule", policy.Namespace, policy.Name) +} + +func hasEgressAllowAll(rules []networkingv1.NetworkPolicyEgressRule) bool { + for _, rule := range rules { + if len(rule.To) == 0 && len(rule.Ports) == 0 { + return true + } + } + return false +} + +func restoreNetworkPolicy(t *testing.T, ctx context.Context, client kubernetes.Interface, expected *networkingv1.NetworkPolicy) { + restoreNetworkPolicyWithTimeout(t, ctx, client, expected, 10*time.Minute) +} + +func restoreNetworkPolicyWithTimeout(t *testing.T, ctx context.Context, client kubernetes.Interface, expected *networkingv1.NetworkPolicy, timeout time.Duration) { + t.Helper() + namespace := expected.Namespace + name := expected.Name + t.Logf("deleting NetworkPolicy %s/%s", namespace, name) + if err := client.NetworkingV1().NetworkPolicies(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + t.Fatalf("failed to delete NetworkPolicy %s/%s: %v", namespace, name, err) + } + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + current, err := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + return equality.Semantic.DeepEqual(expected.Spec, current.Spec), nil + }) + if err != nil { + t.Fatalf("timed out waiting for NetworkPolicy %s/%s spec to be restored after %v: %v", namespace, name, timeout, err) + } + t.Logf("NetworkPolicy %s/%s spec restored after delete", namespace, name) +} + +func mutateAndRestoreNetworkPolicy(t *testing.T, ctx context.Context, client kubernetes.Interface, namespace, name string) { + mutateAndRestoreNetworkPolicyWithTimeout(t, ctx, client, namespace, name, 10*time.Minute) +} + +func mutateAndRestoreNetworkPolicyWithTimeout(t *testing.T, ctx context.Context, client kubernetes.Interface, namespace, name string, timeout time.Duration) { + t.Helper() + original := getNetworkPolicyT(t, ctx, client, namespace, name) + t.Logf("mutating NetworkPolicy %s/%s (podSelector override)", namespace, name) + patch := []byte(`{"spec":{"podSelector":{"matchLabels":{"np-reconcile":"mutated"}}}}`) + _, err := client.NetworkingV1().NetworkPolicies(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + t.Fatalf("failed to patch NetworkPolicy %s/%s: %v", namespace, name, err) + } + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + current := getNetworkPolicyT(t, ctx, client, namespace, name) + return equality.Semantic.DeepEqual(original.Spec, current.Spec), nil + }) + if err != nil { + t.Fatalf("timed out waiting for NetworkPolicy %s/%s spec to be restored after %v: %v", namespace, name, timeout, err) + } + t.Logf("NetworkPolicy %s/%s spec restored", namespace, name) +} + +func waitForPodsReadyByLabel(t *testing.T, ctx context.Context, client kubernetes.Interface, namespace, labelSelector string) { + t.Helper() + t.Logf("waiting for pods ready in %s with selector %s", namespace, labelSelector) + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + return false, err + } + if len(pods.Items) == 0 { + return false, nil + } + for _, pod := range pods.Items { + if !isPodReady(&pod) { + return false, nil + } + } + return true, nil + }) + if err != nil { + t.Fatalf("timed out waiting for pods in %s with selector %s to be ready: %v", namespace, labelSelector, err) + } +} + +func isPodReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func logNetworkPolicyEvents(t *testing.T, ctx context.Context, client kubernetes.Interface, namespaces []string, policyName string) { + t.Helper() + found := false + _ = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + for _, namespace := range namespaces { + events, err := client.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Logf("unable to list events in %s: %v", namespace, err) + continue + } + for _, event := range events.Items { + // Check if event is directly on a NetworkPolicy object with matching name + if event.InvolvedObject.Kind == "NetworkPolicy" && event.InvolvedObject.Name == policyName { + t.Logf("event in %s: %s %s %s", namespace, event.Type, event.Reason, event.Message) + found = true + } + // Also check if the event message mentions the policy name + // (operator emits events on Deployment with policy name in message) + if event.Message != "" && strings.Contains(event.Message, policyName) { + t.Logf("event in %s: %s %s %s", namespace, event.Type, event.Reason, event.Message) + found = true + } + } + } + if found { + return true, nil + } + t.Logf("no NetworkPolicy events yet for %s (namespaces: %v)", policyName, namespaces) + return false, nil + }) + if !found { + t.Logf("no NetworkPolicy events observed for %s (best-effort)", policyName) + } +} + +func logNetworkPolicySummary(t *testing.T, label string, policy *networkingv1.NetworkPolicy) { + t.Logf("networkpolicy %s namespace=%s name=%s podSelector=%v policyTypes=%v ingress=%d egress=%d", + label, + policy.Namespace, + policy.Name, + policy.Spec.PodSelector.MatchLabels, + policy.Spec.PolicyTypes, + len(policy.Spec.Ingress), + len(policy.Spec.Egress), + ) +} + +func logNetworkPolicyDetails(t *testing.T, label string, policy *networkingv1.NetworkPolicy) { + t.Helper() + t.Logf("networkpolicy %s details:", label) + t.Logf(" podSelector=%v policyTypes=%v", policy.Spec.PodSelector.MatchLabels, policy.Spec.PolicyTypes) + for i, rule := range policy.Spec.Ingress { + t.Logf(" ingress[%d]: ports=%s from=%s", i, formatPorts(rule.Ports), formatPeers(rule.From)) + } + for i, rule := range policy.Spec.Egress { + t.Logf(" egress[%d]: ports=%s to=%s", i, formatPorts(rule.Ports), formatPeers(rule.To)) + } +} + +func formatPorts(ports []networkingv1.NetworkPolicyPort) string { + if len(ports) == 0 { + return "[]" + } + out := make([]string, 0, len(ports)) + for _, p := range ports { + proto := "TCP" + if p.Protocol != nil { + proto = string(*p.Protocol) + } + if p.Port == nil { + out = append(out, fmt.Sprintf("%s:any", proto)) + continue + } + out = append(out, fmt.Sprintf("%s:%s", proto, p.Port.String())) + } + return fmt.Sprintf("[%s]", joinStrings(out)) +} + +func formatPeers(peers []networkingv1.NetworkPolicyPeer) string { + if len(peers) == 0 { + return "[]" + } + out := make([]string, 0, len(peers)) + for _, peer := range peers { + ns := formatSelector(peer.NamespaceSelector) + pod := formatSelector(peer.PodSelector) + if ns == "" && pod == "" { + out = append(out, "{}") + continue + } + out = append(out, fmt.Sprintf("ns=%s pod=%s", ns, pod)) + } + return fmt.Sprintf("[%s]", joinStrings(out)) +} + +func formatSelector(sel *metav1.LabelSelector) string { + if sel == nil { + return "" + } + if len(sel.MatchLabels) == 0 && len(sel.MatchExpressions) == 0 { + return "{}" + } + return fmt.Sprintf("labels=%v exprs=%v", sel.MatchLabels, sel.MatchExpressions) +} + +func joinStrings(items []string) string { + if len(items) == 0 { + return "" + } + out := items[0] + for i := 1; i < len(items); i++ { + out += ", " + items[i] + } + return out +}