From 6532a387ad356c0b48ab91a44ecff8199efa1532 Mon Sep 17 00:00:00 2001 From: rkoster Date: Mon, 20 Apr 2026 14:09:19 +0000 Subject: [PATCH] Add acceptance tests for Identity-Aware Routing (domain-scoped mTLS) Add comprehensive test coverage for the Identity-Aware Routing feature that enables mTLS-based authentication and authorization for app-to-app communication on dedicated domains (e.g., *.apps.identity). Test coverage: - Default-deny behavior and access rule creation - Authorization enforcement (denies unauthorized apps) - XFCC header forwarding with caller identity in Envoy format Infrastructure changes: - Extended proxy app with /headers and /mtls_proxy endpoints - Added identity_aware_routing test suite with 3 test cases - Added IncludeIdentityAwareRouting and IdentityAwareDomain config - Added IdentityAwareRoutingDescribe wrapper for test gating Tests use cf add-access-rule/remove-access-rule CLI commands and validate mTLS certificate-based authentication via Diego instance identity certificates. Requires custom CF CLI with access rule management commands. --- assets/proxy/main.go | 89 +++++++++ cats_suite_helpers/cats_suite_helpers.go | 11 ++ cats_suite_test.go | 1 + helpers/config/config.go | 2 + helpers/config/config_struct.go | 15 ++ helpers/skip_messages/skip_messages.go | 2 + .../identity_aware_routing.go | 183 ++++++++++++++++++ 7 files changed, 303 insertions(+) create mode 100644 identity_aware_routing/identity_aware_routing.go diff --git a/assets/proxy/main.go b/assets/proxy/main.go index 3c59eb1d8..87e1352e4 100644 --- a/assets/proxy/main.go +++ b/assets/proxy/main.go @@ -24,6 +24,8 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("/proxy/", proxyHandler) mux.HandleFunc("/https_proxy/", httpsProxyHandler) + mux.HandleFunc("/mtls_proxy/", mtlsProxyHandler) + mux.HandleFunc("/headers", headersHandler) mux.HandleFunc("/", infoHandler(systemPort)) server := &http.Server{ @@ -91,6 +93,93 @@ func handleRequest(destination string, resp http.ResponseWriter, req *http.Reque _, _ = resp.Write(readBytes) } +func headersHandler(resp http.ResponseWriter, req *http.Request) { + headers := make(map[string]string) + for name, values := range req.Header { + headers[name] = strings.Join(values, ", ") + } + resp.Header().Set("Content-Type", "application/json") + json.NewEncoder(resp).Encode(headers) +} + +func mtlsProxyHandler(resp http.ResponseWriter, req *http.Request) { + destination := strings.TrimPrefix(req.URL.Path, "/mtls_proxy/") + destination = fmt.Sprintf("https://%s", destination) + + certFile := os.Getenv("CF_INSTANCE_CERT") + keyFile := os.Getenv("CF_INSTANCE_KEY") + if certFile == "" || keyFile == "" { + resp.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(resp).Encode(map[string]interface{}{ + "status": "error", + "status_code": 500, + "error": "CF_INSTANCE_CERT or CF_INSTANCE_KEY not set", + }) + return + } + + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + resp.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(resp).Encode(map[string]interface{}{ + "status": "error", + "status_code": 500, + "error": fmt.Sprintf("failed to load client cert: %s", err), + }) + return + } + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 0, + }).Dial, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{cert}, + }, + }, + } + + getResp, err := client.Get(destination) + if err != nil { + resp.Header().Set("Content-Type", "application/json") + json.NewEncoder(resp).Encode(map[string]interface{}{ + "status": "error", + "status_code": 0, + "error": fmt.Sprintf("request failed: %s", err), + }) + return + } + defer getResp.Body.Close() + + body, err := io.ReadAll(getResp.Body) + if err != nil { + resp.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(resp).Encode(map[string]interface{}{ + "status": "error", + "status_code": 0, + "error": fmt.Sprintf("read body failed: %s", err), + }) + return + } + + respHeaders := make(map[string]string) + for name, values := range getResp.Header { + respHeaders[name] = strings.Join(values, ", ") + } + + resp.Header().Set("Content-Type", "application/json") + json.NewEncoder(resp).Encode(map[string]interface{}{ + "status": "success", + "status_code": getResp.StatusCode, + "body": string(body), + "headers": respHeaders, + }) +} + var httpClient = &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, diff --git a/cats_suite_helpers/cats_suite_helpers.go b/cats_suite_helpers/cats_suite_helpers.go index dee5bd993..27e4745ab 100644 --- a/cats_suite_helpers/cats_suite_helpers.go +++ b/cats_suite_helpers/cats_suite_helpers.go @@ -250,6 +250,17 @@ func CommaDelimitedSecurityGroupsDescribe(description string, callback func()) b }) } +func IdentityAwareRoutingDescribe(description string, callback func()) bool { + return Describe("[identity aware routing]", func() { + BeforeEach(func() { + if !Config.GetIncludeIdentityAwareRouting() { + Skip(skip_messages.SkipIdentityAwareRoutingMessage) + } + }) + Describe(description, callback) + }) +} + func ServiceDiscoveryDescribe(description string, callback func()) bool { return Describe("[service discovery]", func() { BeforeEach(func() { diff --git a/cats_suite_test.go b/cats_suite_test.go index 5034bd9a9..c182c35ce 100644 --- a/cats_suite_test.go +++ b/cats_suite_test.go @@ -20,6 +20,7 @@ import ( _ "github.com/cloudfoundry/cf-acceptance-tests/docker" _ "github.com/cloudfoundry/cf-acceptance-tests/file_based_service_bindings" _ "github.com/cloudfoundry/cf-acceptance-tests/http2_routing" + _ "github.com/cloudfoundry/cf-acceptance-tests/identity_aware_routing" _ "github.com/cloudfoundry/cf-acceptance-tests/internet_dependent" _ "github.com/cloudfoundry/cf-acceptance-tests/ipv6" _ "github.com/cloudfoundry/cf-acceptance-tests/isolation_segments" diff --git a/helpers/config/config.go b/helpers/config/config.go index 24434e2a0..96a99834a 100644 --- a/helpers/config/config.go +++ b/helpers/config/config.go @@ -24,6 +24,8 @@ type CatsConfig interface { GetIncludeSecurityGroups() bool GetIncludeServices() bool GetIncludeUserProvidedServices() bool + GetIncludeIdentityAwareRouting() bool + GetIdentityAwareDomain() string GetIncludeServiceDiscovery() bool GetIncludeSsh() bool GetIncludeTasks() bool diff --git a/helpers/config/config_struct.go b/helpers/config/config_struct.go index 46f4c6814..fdfd5800d 100644 --- a/helpers/config/config_struct.go +++ b/helpers/config/config_struct.go @@ -90,6 +90,8 @@ type config struct { IncludeRoutingIsolationSegments *bool `json:"include_routing_isolation_segments"` IncludeSSO *bool `json:"include_sso"` IncludeSecurityGroups *bool `json:"include_security_groups"` + IncludeIdentityAwareRouting *bool `json:"include_identity_aware_routing"` + IdentityAwareDomain *string `json:"identity_aware_domain"` IncludeServiceDiscovery *bool `json:"include_service_discovery"` IncludeServiceInstanceSharing *bool `json:"include_service_instance_sharing"` IncludeServices *bool `json:"include_services"` @@ -199,6 +201,8 @@ func getDefaults() config { defaults.IncludeRouteServices = ptrToBool(false) defaults.IncludeSSO = ptrToBool(false) defaults.IncludeSecurityGroups = ptrToBool(false) + defaults.IncludeIdentityAwareRouting = ptrToBool(false) + defaults.IdentityAwareDomain = ptrToString("apps.identity") defaults.IncludeServiceDiscovery = ptrToBool(false) defaults.IncludeServices = ptrToBool(false) defaults.IncludeUserProvidedServices = ptrToBool(false) @@ -479,6 +483,9 @@ func validateConfig(config *config) error { if config.IncludeSecurityGroups == nil { errs = errors.Join(errs, fmt.Errorf("* 'include_security_groups' must not be null")) } + if config.IncludeIdentityAwareRouting == nil { + errs = errors.Join(errs, fmt.Errorf("* 'include_identity_aware_routing' must not be null")) + } if config.IncludeServiceDiscovery == nil { errs = errors.Join(errs, fmt.Errorf("* 'include_service_discovery' must not be null")) } @@ -1115,6 +1122,14 @@ func (c *config) GetIncludeWindows() bool { return *c.IncludeWindows } +func (c *config) GetIncludeIdentityAwareRouting() bool { + return *c.IncludeIdentityAwareRouting +} + +func (c *config) GetIdentityAwareDomain() string { + return *c.IdentityAwareDomain +} + func (c *config) GetIncludeServiceDiscovery() bool { return *c.IncludeServiceDiscovery } diff --git a/helpers/skip_messages/skip_messages.go b/helpers/skip_messages/skip_messages.go index c64bc8cf9..d15ce41ff 100644 --- a/helpers/skip_messages/skip_messages.go +++ b/helpers/skip_messages/skip_messages.go @@ -55,6 +55,8 @@ const SkipIsolationSegmentsMessage = `Skipping this test because config.IncludeI const SkipRoutingIsolationSegmentsMessage = `Skipping this test because Config.IncludeRoutingIsolationSegments is set to 'false'.` const SkipZipkinMessage = `Skipping this test because config.IncludeZipkin is set to 'false'` const SkipServiceDiscoveryMessage = `Skipping this test because config.IncludeServiceDiscovery is set to 'false'.` +const SkipIdentityAwareRoutingMessage = `Skipping this test because config.IncludeIdentityAwareRouting is set to 'false'. +NOTE: Ensure that identity-aware routing is enabled and the identity-aware domain is configured before running this test.` const SkipServiceInstanceSharingMessage = `Skipping this test because config.IncludeServiceInstanceSharing is set to 'false'.` const SkipCapiExperimentalMessage = `Skipping this test because config.IncludeCapiExperimental is set to 'false'.` const SkipWindowsTasksMessage = `Skipping Windows tasks tests (requires diego-release v1.20.0 and above)` diff --git a/identity_aware_routing/identity_aware_routing.go b/identity_aware_routing/identity_aware_routing.go new file mode 100644 index 000000000..f261dd1fa --- /dev/null +++ b/identity_aware_routing/identity_aware_routing.go @@ -0,0 +1,183 @@ +package identity_aware_routing + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + . "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/assets" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gexec" + + "github.com/cloudfoundry/cf-test-helpers/v2/cf" + "github.com/cloudfoundry/cf-test-helpers/v2/helpers" +) + +type mtlsProxyResponse struct { + Status string `json:"status"` + StatusCode int `json:"status_code"` + Body string `json:"body"` + Headers map[string]string `json:"headers"` + Error string `json:"error"` +} + +var _ = IdentityAwareRoutingDescribe("Identity-Aware Routing", func() { + var appNameFrontend string + var appNameBackend string + var appNameUnauthorized string + var backendHostName string + var identityAwareDomain string + + BeforeEach(func() { + identityAwareDomain = Config.GetIdentityAwareDomain() + + backendHostName = random_name.CATSRandomName("HOST") + appNameFrontend = random_name.CATSRandomName("APP-FRONT") + appNameBackend = random_name.CATSRandomName("APP-BACK") + appNameUnauthorized = random_name.CATSRandomName("APP-UNAUTH") + + // push backend app (proxy app so it has /headers endpoint) + Expect(cf.Cf( + "push", appNameBackend, + "-b", Config.GetGoBuildpackName(), + "-m", DEFAULT_MEMORY_LIMIT, + "-p", assets.NewAssets().Proxy, + "-f", assets.NewAssets().Proxy+"/manifest.yml", + ).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + // map identity-aware route to backend app + Expect(cf.Cf("map-route", appNameBackend, identityAwareDomain, "--hostname", backendHostName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + // push frontend app (proxy app with /mtls_proxy endpoint) + Expect(cf.Cf( + "push", appNameFrontend, + "-b", Config.GetGoBuildpackName(), + "-m", DEFAULT_MEMORY_LIMIT, + "-p", assets.NewAssets().Proxy, + "-f", assets.NewAssets().Proxy+"/manifest.yml", + ).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + // push unauthorized app (same proxy app, different identity) + Expect(cf.Cf( + "push", appNameUnauthorized, + "-b", Config.GetGoBuildpackName(), + "-m", DEFAULT_MEMORY_LIMIT, + "-p", assets.NewAssets().Proxy, + "-f", assets.NewAssets().Proxy+"/manifest.yml", + ).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + }) + + AfterEach(func() { + app_helpers.AppReport(appNameFrontend) + app_helpers.AppReport(appNameBackend) + app_helpers.AppReport(appNameUnauthorized) + + Expect(cf.Cf("delete", appNameFrontend, "-f", "-r").Wait()).To(Exit(0)) + Expect(cf.Cf("delete", appNameBackend, "-f", "-r").Wait()).To(Exit(0)) + Expect(cf.Cf("delete", appNameUnauthorized, "-f", "-r").Wait()).To(Exit(0)) + }) + + mtlsProxyURL := func(appName, backendHost, domain, path string) string { + return fmt.Sprintf("%s%s.%s/mtls_proxy/%s.%s/%s", + Config.Protocol(), appName, Config.GetAppsDomain(), + backendHost, domain, path) + } + + curlMtlsProxy := func(appName, backendHost, domain, path string) mtlsProxyResponse { + curlArgs := mtlsProxyURL(appName, backendHost, domain, path) + curl := helpers.Curl(Config, curlArgs).Wait() + var resp mtlsProxyResponse + err := json.Unmarshal(curl.Out.Contents(), &resp) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to parse mtls_proxy response: %s", string(curl.Out.Contents())) + return resp + } + + Describe("mTLS authorization with access rules", func() { + It("denies access by default and allows after adding an access rule", func() { + By("verifying the frontend is denied without access rules (default deny)") + Eventually(func() int { + resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers") + return resp.StatusCode + }, 2*time.Minute).Should(Equal(403)) + + By("creating an access rule for the frontend app") + Expect(cf.Cf( + "add-access-rule", identityAwareDomain, + "--source-app", appNameFrontend, + "--hostname", backendHostName, + ).Wait(Config.DefaultTimeoutDuration())).To(Exit(0)) + + By("verifying the access rule is listed") + accessRulesOutput := cf.Cf("access-rules", "--domain", identityAwareDomain).Wait(Config.DefaultTimeoutDuration()) + Expect(accessRulesOutput).To(Exit(0)) + Expect(string(accessRulesOutput.Out.Contents())).To(ContainSubstring(appNameFrontend)) + + By("verifying the frontend can now reach the backend") + Eventually(func() int { + resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers") + return resp.StatusCode + }, 2*time.Minute).Should(Equal(200)) + }) + + It("denies access from an unauthorized app even with a valid certificate", func() { + By("creating an access rule only for the frontend app") + Expect(cf.Cf( + "add-access-rule", identityAwareDomain, + "--source-app", appNameFrontend, + "--hostname", backendHostName, + ).Wait(Config.DefaultTimeoutDuration())).To(Exit(0)) + + By("verifying the authorized frontend can reach the backend") + Eventually(func() int { + resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers") + return resp.StatusCode + }, 2*time.Minute).Should(Equal(200)) + + By("verifying the unauthorized app is denied") + Consistently(func() int { + resp := curlMtlsProxy(appNameUnauthorized, backendHostName, identityAwareDomain, "headers") + return resp.StatusCode + }, 30*time.Second).Should(Equal(403)) + }) + + It("forwards X-Forwarded-Client-Cert header with caller identity in Envoy format", func() { + frontendGuid := GuidForAppName(appNameFrontend) + + By("creating an access rule for the frontend app") + Expect(cf.Cf( + "add-access-rule", identityAwareDomain, + "--source-app", appNameFrontend, + "--hostname", backendHostName, + ).Wait(Config.DefaultTimeoutDuration())).To(Exit(0)) + + By("calling the backend and examining the XFCC header") + var xfcc string + Eventually(func() string { + resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers") + if resp.StatusCode != 200 { + return "" + } + // The backend returns its request headers as JSON via /headers + var headers map[string]string + err := json.Unmarshal([]byte(resp.Body), &headers) + if err != nil { + return "" + } + xfcc = headers["X-Forwarded-Client-Cert"] + return xfcc + }, 2*time.Minute).ShouldNot(BeEmpty()) + + By("verifying the XFCC header is in Envoy format") + Expect(xfcc).To(ContainSubstring("Hash=")) + Expect(xfcc).To(ContainSubstring("Subject=")) + + By("verifying the XFCC header contains the frontend app GUID") + Expect(strings.ToLower(xfcc)).To(ContainSubstring("ou=app:" + strings.ToLower(frontendGuid))) + }) + }) +})