diff --git a/pkg/operator/encryption/kms/preflight/always_succeed_deployer.go b/pkg/operator/encryption/kms/preflight/always_succeed_deployer.go index c90eec1cc9..eaeffffd6c 100644 --- a/pkg/operator/encryption/kms/preflight/always_succeed_deployer.go +++ b/pkg/operator/encryption/kms/preflight/always_succeed_deployer.go @@ -2,6 +2,9 @@ package preflight import ( "context" + "crypto/rand" + "encoding/hex" + "fmt" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -20,16 +23,31 @@ func NewAlwaysSucceedKMSPreflightDeployer() *AlwaysSucceedKMSPreflightDeployer { // AlwaysSucceedKMSPreflightDeployer is a KMSPreflightDeployer that immediately // reports a successful preflight result without deploying any workload. type AlwaysSucceedKMSPreflightDeployer struct { - configHash string - deployed bool + configHash string + deployed bool + remoteKeyID string } func (d *AlwaysSucceedKMSPreflightDeployer) Deploy(_ context.Context, configHash string, _ *corev1.Secret) error { + remoteKeyID, err := mintRemoteKeyID() + if err != nil { + return err + } d.configHash = configHash d.deployed = true + d.remoteKeyID = remoteKeyID return nil } +// mintRemoteKeyID returns a random remote key id so every deployment reports a distinct one. +func mintRemoteKeyID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("failed to generate remote key id: %w", err) + } + return hex.EncodeToString(b), nil +} + func (d *AlwaysSucceedKMSPreflightDeployer) Status(_ context.Context) (corev1.PodStatus, error) { if !d.deployed { return corev1.PodStatus{}, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, "kms-preflight") @@ -49,7 +67,7 @@ func (d *AlwaysSucceedKMSPreflightDeployer) Status(_ context.Context) (corev1.Po { Type: controllers.KMSPreflightRemoteKeyIDPodCondition, Status: corev1.ConditionTrue, - Message: "always-succeed", + Message: d.remoteKeyID, }, }, }, nil @@ -58,5 +76,6 @@ func (d *AlwaysSucceedKMSPreflightDeployer) Status(_ context.Context) (corev1.Po func (d *AlwaysSucceedKMSPreflightDeployer) Cleanup(_ context.Context) error { d.configHash = "" d.deployed = false + d.remoteKeyID = "" return nil } diff --git a/test/e2e-encryption/encryption_test.go b/test/e2e-encryption/encryption_test.go index 79fe2a23d1..ebd9824b96 100644 --- a/test/e2e-encryption/encryption_test.go +++ b/test/e2e-encryption/encryption_test.go @@ -1141,7 +1141,9 @@ func (d *configurableKMSPreflightDeployer) Status(_ context.Context) (corev1.Pod Conditions: []corev1.PodCondition{ {Type: controllers.KMSPreflightConfigHashPodCondition, Status: corev1.ConditionTrue, Message: d.configHash}, {Type: controllers.KMSPreflightResultPodCondition, Status: resultStatus, Message: resultMessage}, - {Type: controllers.KMSPreflightRemoteKeyIDPodCondition, Status: corev1.ConditionTrue, Message: "configurable"}, + // Vary the remote key id per distinct config (stable on redeploy of the + // same config), mirroring a real KMS backend's per-key id. + {Type: controllers.KMSPreflightRemoteKeyIDPodCondition, Status: corev1.ConditionTrue, Message: "configurable-" + d.configHash}, }, }, nil } diff --git a/test/library/encryption/assertion.go b/test/library/encryption/assertion.go index bb310b5b1d..28be587813 100644 --- a/test/library/encryption/assertion.go +++ b/test/library/encryption/assertion.go @@ -20,12 +20,16 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" configv1 "github.com/openshift/api/config/v1" oauthapiv1 "github.com/openshift/api/oauth/v1" + operatorv1 "github.com/openshift/api/operator/v1" routev1 "github.com/openshift/api/route/v1" + "github.com/openshift/library-go/pkg/operator/v1helpers" ) var protoEncodingPrefix = []byte{0x6b, 0x38, 0x73, 0x00} @@ -390,3 +394,72 @@ func assertWellKnownRoutes(t testing.TB, etcdClient EtcdClient, expectedMode str t.Logf("Verified %d Routes", totalRoutes) require.NoError(t, err) } + +// preflightDegradedConditionType is set by kmsPreflightController, which exports no constant for it. +const preflightDegradedConditionType = "EncryptionKMSPreflightControllerDegraded" + +// kmsOperatorStatus is the subset of an operator CR status the KMS preflight assertions read. +type kmsOperatorStatus struct { + Conditions []operatorv1.OperatorCondition `json:"conditions"` + EncryptionStatus operatorv1.KMSEncryptionStatus `json:"encryptionStatus"` +} + +func decodeKMSOperatorStatus(obj map[string]interface{}) (kmsOperatorStatus, error) { + var cr struct { + Status kmsOperatorStatus `json:"status"` + } + err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj, &cr) + return cr.Status, err +} + +// operatorGVRForNamespace maps an operator namespace to its operator CR. +func operatorGVRForNamespace(t testing.TB, operatorNamespace string) schema.GroupVersionResource { + t.Helper() + byNamespace := map[string]schema.GroupVersionResource{ + "openshift-kube-apiserver-operator": {Group: "operator.openshift.io", Version: "v1", Resource: "kubeapiservers"}, + "openshift-authentication-operator": {Group: "operator.openshift.io", Version: "v1", Resource: "authentications"}, + "openshift-apiserver-operator": {Group: "operator.openshift.io", Version: "v1", Resource: "openshiftapiservers"}, + } + gvr, ok := byNamespace[operatorNamespace] + require.Truef(t, ok, "no known operator CR for namespace %q; cannot read/assert KMS preflight", operatorNamespace) + return gvr +} + +// AssertKMSPreflightSucceededForOperator asserts KMS preflight passed for the operator owning +// operatorNamespace. previous is the pre-apply snapshot (see ReadKMSPreflightForOperator). +func AssertKMSPreflightSucceededForOperator(ctx context.Context, t testing.TB, clientSet ClientSet, operatorNamespace string, previous operatorv1.KMSPreflightCheck) { + t.Helper() + gvr := operatorGVRForNamespace(t, operatorNamespace) + assertKMSPreflightSucceeded(ctx, t, clientSet.DynamicClient, gvr, "cluster", previous) +} + +// assertKMSPreflightSucceeded asserts preflight passed for the CR's current config: degraded is +// False, preflight reports Succeeded for the observed config hash, remoteKeyID is set (proving a +// live KMS check ran), and remoteKeyID advanced when the config changed since previous. +func assertKMSPreflightSucceeded(ctx context.Context, t testing.TB, dynamicClient dynamic.Interface, gvr schema.GroupVersionResource, name string, previous operatorv1.KMSPreflightCheck) { + t.Helper() + + var preflight operatorv1.KMSPreflightCheck + var degradedFalse bool + err := wait.PollUntilContextTimeout(ctx, 2*time.Second, time.Minute, true, func(ctx context.Context) (bool, error) { + obj, err := dynamicClient.Resource(gvr).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + status, err := decodeKMSOperatorStatus(obj.Object) + if err != nil { + return false, err + } + preflight = status.EncryptionStatus.Preflight + result := preflight.Result + degradedFalse = v1helpers.IsOperatorConditionFalse(status.Conditions, preflightDegradedConditionType) + passed := result.Status == operatorv1.KMSPreflightResultSucceeded && result.ConfigHash != "" && result.ConfigHash == preflight.ObservedConfigHash + fresh := preflight.ObservedConfigHash == previous.ObservedConfigHash || result.RemoteKeyID != previous.Result.RemoteKeyID + ran := result.RemoteKeyID != "" + return degradedFalse && passed && ran && fresh, nil + }) + require.NoErrorf(t, err, + "KMS preflight not confirmed for %s/%s: degradedFalse=%t result.status=%q result.configHash=%q observedConfigHash=%q remoteKeyID=%q (previous observedConfigHash=%q remoteKeyID=%q)", + gvr.Resource, name, degradedFalse, preflight.Result.Status, preflight.Result.ConfigHash, preflight.ObservedConfigHash, preflight.Result.RemoteKeyID, + previous.ObservedConfigHash, previous.Result.RemoteKeyID) +} diff --git a/test/library/encryption/helpers.go b/test/library/encryption/helpers.go index 62b059235f..10befcc003 100644 --- a/test/library/encryption/helpers.go +++ b/test/library/encryption/helpers.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/util/retry" configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" oauthapiv1 "github.com/openshift/api/oauth/v1" @@ -168,6 +169,23 @@ func GetClients(t testing.TB) ClientSet { return ClientSet{Etcd: etcdClient, ApiServerConfig: apiServerConfigClient, Kube: kubeClient, DynamicClient: dynamicClient} } +// ReadKMSPreflightForOperator returns the operator's current preflight snapshot (zero when unset). +// Snapshot it before applying a new KMS config and pass it to AssertKMSPreflightSucceededForOperator +// to confirm a fresh preflight ran for that config. +func ReadKMSPreflightForOperator(ctx context.Context, t testing.TB, clientSet ClientSet, operatorNamespace string) (operatorv1.KMSPreflightCheck, error) { + t.Helper() + gvr := operatorGVRForNamespace(t, operatorNamespace) + obj, err := clientSet.DynamicClient.Resource(gvr).Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return operatorv1.KMSPreflightCheck{}, err + } + status, err := decodeKMSOperatorStatus(obj.Object) + if err != nil { + return operatorv1.KMSPreflightCheck{}, err + } + return status.EncryptionStatus.Preflight, nil +} + func WaitForEncryptionKeyBasedOn(t testing.TB, kubeClient kubernetes.Interface, prevKeyMeta EncryptionKeyMeta, encryptionType configv1.EncryptionType, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) { encryptionMode := string(encryptionType) if encryptionMode == "" { diff --git a/test/library/encryption/scenarios.go b/test/library/encryption/scenarios.go index b93c391740..14753e1c7c 100644 --- a/test/library/encryption/scenarios.go +++ b/test/library/encryption/scenarios.go @@ -75,9 +75,14 @@ func TestEncryptionTypeAESGCM(ctx context.Context, t testing.TB, scenario BasicS func TestEncryptionTypeKMS(ctx context.Context, t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { provider := resolveProvider(t, configv1.EncryptionTypeKMS, providers) e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) + // Snapshot preflight before applying the new config so the assertion can confirm a fresh + // preflight ran for it (the remote key id advances when the config genuinely changes). + previousPreflight, err := ReadKMSPreflightForOperator(ctx, e, GetClients(e), scenario.OperatorNamespace) + require.NoError(e, err) clientSet := SetAndWaitForEncryptionType(ctx, e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, provider.Type, scenario.Namespace, scenario.LabelSelector) AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) + AssertKMSPreflightSucceededForOperator(ctx, e, clientSet, scenario.OperatorNamespace, previousPreflight) } func TestEncryptionType(ctx context.Context, t testing.TB, scenario BasicScenario, provider EncryptionProvider) {