diff --git a/velero-plugins/common/types.go b/velero-plugins/common/types.go index b34ccf5d..c9329d7a 100644 --- a/velero-plugins/common/types.go +++ b/velero-plugins/common/types.go @@ -104,6 +104,17 @@ const ( DCPodLabels string = "oadp.openshift.io/pod-labels" // labels from DC pod ) +// Namespace SCC UID/GID range bookkeeping annotations. +// Namespace objects never pass through RestoreItemAction plugins (Velero core +// special-cases and skips them), so these values are stashed onto each backed-up +// ServiceAccount at backup time and read back at restore time to detect a +// UID/GID-range mismatch between the backed-up and restored namespace. +const ( + BackupNsSccUIDRange string = "oadp.openshift.io/backup-ns-scc-uid-range" + BackupNsSccSupplementalGroups string = "oadp.openshift.io/backup-ns-scc-supplemental-groups" + BackupNsSccMcs string = "oadp.openshift.io/backup-ns-scc-mcs" +) + // Configmap Name const RegistryConfigMap string = "oadp-registry-config" @@ -127,8 +138,8 @@ const ( // CSI-Addons controller (shipped with ODF) uses to create a ReclaimSpaceCronJob // that runs rbd sparsify automatically. No-op when CSI-Addons is not installed. const ( - ReclaimSpaceScheduleAnnotation string = "oadp.openshift.io/reclaim-space-schedule" // set on Restore CR - CSIAddonsReclaimSpaceScheduleAnnotation string = "reclaimspace.csiaddons.openshift.io/schedule" // set on restored PVCs + ReclaimSpaceScheduleAnnotation string = "oadp.openshift.io/reclaim-space-schedule" // set on Restore CR + CSIAddonsReclaimSpaceScheduleAnnotation string = "reclaimspace.csiaddons.openshift.io/schedule" // set on restored PVCs ) // OVN-Kubernetes and Multus CNI-injected annotations. diff --git a/velero-plugins/main.go b/velero-plugins/main.go index 8df60b3b..33d54a37 100644 --- a/velero-plugins/main.go +++ b/velero-plugins/main.go @@ -15,6 +15,7 @@ import ( "github.com/konveyor/openshift-velero-plugin/velero-plugins/imagestreamtag" "github.com/konveyor/openshift-velero-plugin/velero-plugins/imagetag" "github.com/konveyor/openshift-velero-plugin/velero-plugins/job" + "github.com/konveyor/openshift-velero-plugin/velero-plugins/namespacescc" "github.com/konveyor/openshift-velero-plugin/velero-plugins/nonadmin" "github.com/konveyor/openshift-velero-plugin/velero-plugins/persistentvolume" "github.com/konveyor/openshift-velero-plugin/velero-plugins/pod" @@ -40,6 +41,8 @@ func main() { RegisterBackupItemAction("openshift.io/02-serviceaccount-backup-plugin", newServiceAccountBackupPlugin). RegisterRestoreItemAction("openshift.io/02-serviceaccount-restore-plugin", newServiceAccountRestorePlugin). RegisterItemBlockAction("openshift.io/02-serviceaccount-iba-plugin", newServiceAccountIBAPlugin). + RegisterBackupItemAction("openshift.io/02-namespacescc-backup-plugin", newNamespaceSccBackupPlugin). + RegisterRestoreItemAction("openshift.io/02-namespacescc-restore-plugin", newNamespaceSccRestorePlugin). RegisterBackupItemAction("openshift.io/03-pv-backup-plugin", newPVBackupPlugin). RegisterRestoreItemAction("openshift.io/03-pv-restore-plugin", newPVRestorePlugin). RegisterRestoreItemAction("openshift.io/04-pvc-restore-plugin", newPVCRestorePlugin). @@ -240,3 +243,11 @@ func newNonAdminRestorePlugin(logger logrus.FieldLogger) (interface{}, error) { func newRBACRoleBindingRestorePlugin(logger logrus.FieldLogger) (interface{}, error) { return &rolebindings.K8sRestorePlugin{Log: logger}, nil } + +func newNamespaceSccBackupPlugin(logger logrus.FieldLogger) (interface{}, error) { + return &namespacescc.BackupPlugin{Log: logger}, nil +} + +func newNamespaceSccRestorePlugin(logger logrus.FieldLogger) (interface{}, error) { + return &namespacescc.RestorePlugin{Log: logger}, nil +} diff --git a/velero-plugins/namespacescc/backup.go b/velero-plugins/namespacescc/backup.go new file mode 100644 index 00000000..83fd9e0c --- /dev/null +++ b/velero-plugins/namespacescc/backup.go @@ -0,0 +1,127 @@ +package namespacescc + +import ( + "context" + "encoding/json" + "sync" + + "github.com/konveyor/openshift-velero-plugin/velero-plugins/clients" + "github.com/konveyor/openshift-velero-plugin/velero-plugins/common" + apisecurity "github.com/openshift/api/security/v1" + "github.com/sirupsen/logrus" + v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +// sccAnnotationCarrier maps the namespace's SCC UID/GID-range annotations to the +// bookkeeping annotation keys they're stashed under on the ServiceAccount. +var sccAnnotationCarrier = map[string]string{ + apisecurity.UIDRangeAnnotation: common.BackupNsSccUIDRange, + apisecurity.SupplementalGroupsAnnotation: common.BackupNsSccSupplementalGroups, + apisecurity.MCSAnnotation: common.BackupNsSccMcs, +} + +// BackupPlugin stashes the parent namespace's SCC UID/GID-range annotations onto +// each ServiceAccount at backup time. Namespace objects never pass through +// RestoreItemAction plugins on restore (Velero core special-cases and skips +// them), so ServiceAccounts - always present in every namespace - are used as +// the carrier to detect a range mismatch at restore time (see restore.go). +type BackupPlugin struct { + Log logrus.FieldLogger + + // namespaceAnnotationCache avoids one Namespaces().Get() per ServiceAccount + // when a namespace has multiple service accounts. Cleared whenever + // cachedForBackup no longer matches the current backup, so entries don't + // leak across backups handled by the same long-lived plugin process. + // Guarded by mu since the shared plugin process may serve concurrent + // operations. + mu sync.Mutex + namespaceAnnotationCache map[string]map[string]string + cachedForBackup string +} + +// AppliesTo returns a velero.ResourceSelector that applies to service accounts. +func (p *BackupPlugin) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"serviceaccounts"}, + }, nil +} + +// Execute stashes the namespace's SCC UID/GID-range annotations onto the service account being backed up. +func (p *BackupPlugin) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) { + p.Log.Info("[namespacescc-backup] Entering namespace SCC range backup plugin") + + serviceAccount := corev1.ServiceAccount{} + itemMarshal, _ := json.Marshal(item) + json.Unmarshal(itemMarshal, &serviceAccount) + + namespaceAnnotations, err := p.getNamespaceAnnotations(backup.Name, serviceAccount.Namespace) + if err != nil { + return nil, nil, err + } + + annotations, stashed := stashNamespaceSCCAnnotations(serviceAccount.Annotations, namespaceAnnotations) + if !stashed { + return item, nil, nil + } + serviceAccount.Annotations = annotations + + var out map[string]interface{} + objrec, _ := json.Marshal(serviceAccount) + json.Unmarshal(objrec, &out) + + return &unstructured.Unstructured{Object: out}, nil, nil +} + +// getNamespaceAnnotations returns the annotations of the named namespace, +// caching the result for the lifetime of the current backup so a namespace +// with multiple service accounts only needs one Get() call. +func (p *BackupPlugin) getNamespaceAnnotations(backupName, namespace string) (map[string]string, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.cachedForBackup != backupName { + p.namespaceAnnotationCache = map[string]map[string]string{} + p.cachedForBackup = backupName + } + if annotations, ok := p.namespaceAnnotationCache[namespace]; ok { + return annotations, nil + } + + client, err := clients.CoreClient() + if err != nil { + return nil, err + } + ns, err := client.Namespaces().Get(context.Background(), namespace, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + p.namespaceAnnotationCache[namespace] = ns.Annotations + return ns.Annotations, nil +} + +// stashNamespaceSCCAnnotations returns a copy of saAnnotations with the +// namespace's SCC UID/GID-range annotations (if present on namespaceAnnotations) +// stashed under bookkeeping keys. The second return value reports whether +// anything was stashed. +func stashNamespaceSCCAnnotations(saAnnotations, namespaceAnnotations map[string]string) (map[string]string, bool) { + var stashed bool + out := saAnnotations + for src, dst := range sccAnnotationCarrier { + v, ok := namespaceAnnotations[src] + if !ok || v == "" { + continue + } + if out == nil { + out = map[string]string{} + } + out[dst] = v + stashed = true + } + return out, stashed +} diff --git a/velero-plugins/namespacescc/backup_test.go b/velero-plugins/namespacescc/backup_test.go new file mode 100644 index 00000000..5de05f1a --- /dev/null +++ b/velero-plugins/namespacescc/backup_test.go @@ -0,0 +1,77 @@ +package namespacescc + +import ( + "reflect" + "testing" + + "github.com/konveyor/openshift-velero-plugin/velero-plugins/util/test" + apisecurity "github.com/openshift/api/security/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +func TestBackupPluginAppliesTo(t *testing.T) { + backupPlugin := &BackupPlugin{Log: test.NewLogger()} + actual, err := backupPlugin.AppliesTo() + require.NoError(t, err) + assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"serviceaccounts"}}, actual) +} + +func Test_stashNamespaceSCCAnnotations(t *testing.T) { + tests := []struct { + name string + saAnnotations map[string]string + namespaceAnnotations map[string]string + wantAnnotations map[string]string + wantStashed bool + }{ + { + name: "namespace has no SCC annotations, nothing stashed", + saAnnotations: nil, + namespaceAnnotations: map[string]string{}, + wantAnnotations: nil, + wantStashed: false, + }, + { + name: "namespace has all 3 SCC annotations, all stashed onto nil SA annotations", + saAnnotations: nil, + namespaceAnnotations: map[string]string{ + apisecurity.UIDRangeAnnotation: "1000700000/10000", + apisecurity.SupplementalGroupsAnnotation: "1000700000/10000", + apisecurity.MCSAnnotation: "s0:c26,c5", + }, + wantAnnotations: map[string]string{ + "oadp.openshift.io/backup-ns-scc-uid-range": "1000700000/10000", + "oadp.openshift.io/backup-ns-scc-supplemental-groups": "1000700000/10000", + "oadp.openshift.io/backup-ns-scc-mcs": "s0:c26,c5", + }, + wantStashed: true, + }, + { + name: "existing SA annotations are preserved alongside stashed ones", + saAnnotations: map[string]string{ + "kubernetes.io/service-account.name": "default", + }, + namespaceAnnotations: map[string]string{ + apisecurity.UIDRangeAnnotation: "1000700000/10000", + }, + wantAnnotations: map[string]string{ + "kubernetes.io/service-account.name": "default", + "oadp.openshift.io/backup-ns-scc-uid-range": "1000700000/10000", + }, + wantStashed: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotAnnotations, gotStashed := stashNamespaceSCCAnnotations(tt.saAnnotations, tt.namespaceAnnotations) + if gotStashed != tt.wantStashed { + t.Errorf("stashNamespaceSCCAnnotations() stashed = %v, want %v", gotStashed, tt.wantStashed) + } + if !reflect.DeepEqual(gotAnnotations, tt.wantAnnotations) { + t.Errorf("stashNamespaceSCCAnnotations() annotations = %v, want %v", gotAnnotations, tt.wantAnnotations) + } + }) + } +} diff --git a/velero-plugins/namespacescc/restore.go b/velero-plugins/namespacescc/restore.go new file mode 100644 index 00000000..2786bfd7 --- /dev/null +++ b/velero-plugins/namespacescc/restore.go @@ -0,0 +1,156 @@ +package namespacescc + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "sync" + + "github.com/konveyor/openshift-velero-plugin/velero-plugins/clients" + "github.com/sirupsen/logrus" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// RestorePlugin compares the SCC UID/GID-range annotations stashed at backup +// time (see backup.go) against the actual annotations on the restored +// namespace, and logs a warning on mismatch. This is the closest available +// signal: Namespace objects never pass through RestoreItemAction plugins, so +// there's no way to add to the restore's Warnings the way other item actions +// can (RestoreItemActionExecuteOutput has no Warning field in this Velero API +// version). +type RestorePlugin struct { + Log logrus.FieldLogger + + // namespaceAnnotationCache avoids one Namespaces().Get() per ServiceAccount + // when a namespace has multiple service accounts. Cleared whenever + // cachedForRestore no longer matches the current restore, so entries don't + // leak across restores handled by the same long-lived plugin process. + // Guarded by mu since the shared plugin process may serve concurrent + // operations. + mu sync.Mutex + namespaceAnnotationCache map[string]map[string]string + cachedForRestore string +} + +// AppliesTo returns a velero.ResourceSelector that applies to service accounts. +func (p *RestorePlugin) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"serviceaccounts"}, + }, nil +} + +// Execute compares backed-up vs. actual namespace SCC UID/GID-range annotations and strips the bookkeeping annotations. +func (p *RestorePlugin) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + p.Log.Info("[namespacescc-restore] Entering namespace SCC range restore plugin") + + backedUp := corev1.ServiceAccount{} + backedUpMarshal, _ := json.Marshal(input.ItemFromBackup) + json.Unmarshal(backedUpMarshal, &backedUp) + + expected := expectedSCCAnnotations(backedUp.Annotations) + + serviceAccount := corev1.ServiceAccount{} + itemMarshal, _ := json.Marshal(input.Item) + json.Unmarshal(itemMarshal, &serviceAccount) + + if len(expected) > 0 { + targetNamespace := serviceAccount.Namespace + if mapped := input.Restore.Spec.NamespaceMapping[targetNamespace]; mapped != "" { + targetNamespace = mapped + } + + namespaceAnnotations, err := p.getNamespaceAnnotations(input.Restore.Name, targetNamespace) + if err != nil { + p.Log.Warnf("[namespacescc-restore] unable to verify namespace %s SCC range: %v", targetNamespace, err) + } else { + for _, m := range sccAnnotationMismatches(targetNamespace, expected, namespaceAnnotations) { + p.Log.Warn(m) + } + } + } + + serviceAccount.Annotations = stripSCCBookkeepingAnnotations(serviceAccount.Annotations) + + var out map[string]interface{} + objrec, _ := json.Marshal(serviceAccount) + json.Unmarshal(objrec, &out) + + return velero.NewRestoreItemActionExecuteOutput(&unstructured.Unstructured{Object: out}), nil +} + +// getNamespaceAnnotations returns the annotations of the named namespace, +// caching the result for the lifetime of the current restore so a namespace +// with multiple service accounts only needs one Get() call. +func (p *RestorePlugin) getNamespaceAnnotations(restoreName, namespace string) (map[string]string, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.cachedForRestore != restoreName { + p.namespaceAnnotationCache = map[string]map[string]string{} + p.cachedForRestore = restoreName + } + if annotations, ok := p.namespaceAnnotationCache[namespace]; ok { + return annotations, nil + } + + client, err := clients.CoreClient() + if err != nil { + return nil, err + } + ns, err := client.Namespaces().Get(context.Background(), namespace, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + p.namespaceAnnotationCache[namespace] = ns.Annotations + return ns.Annotations, nil +} + +// expectedSCCAnnotations extracts the backed-up namespace SCC annotations +// stashed on the ServiceAccount's bookkeeping annotations, keyed by the real +// SCC annotation name. +func expectedSCCAnnotations(backedUpAnnotations map[string]string) map[string]string { + expected := map[string]string{} + for src, dst := range sccAnnotationCarrier { + if v, ok := backedUpAnnotations[dst]; ok && v != "" { + expected[src] = v + } + } + return expected +} + +// sccAnnotationMismatches compares expected (backed-up) vs. actual namespace SCC +// annotations and returns a human-readable message per mismatch, sorted by +// annotation name for deterministic output. +func sccAnnotationMismatches(namespaceName string, expected, actual map[string]string) []string { + var annotations []string + for annotation := range expected { + annotations = append(annotations, annotation) + } + sort.Strings(annotations) + + var mismatches []string + for _, annotation := range annotations { + expectedValue := expected[annotation] + actualValue := actual[annotation] + if actualValue != expectedValue { + mismatches = append(mismatches, fmt.Sprintf( + "[namespacescc-restore] namespace %q annotation %q mismatch: backed up as %q, restored as %q. "+ + "Workloads relying on the backed-up UID/GID range may have incorrect file ownership.", + namespaceName, annotation, expectedValue, actualValue)) + } + } + return mismatches +} + +// stripSCCBookkeepingAnnotations removes the bookkeeping annotations added at backup time. +func stripSCCBookkeepingAnnotations(annotations map[string]string) map[string]string { + for _, dst := range sccAnnotationCarrier { + delete(annotations, dst) + } + return annotations +} diff --git a/velero-plugins/namespacescc/restore_test.go b/velero-plugins/namespacescc/restore_test.go new file mode 100644 index 00000000..55a0c4bc --- /dev/null +++ b/velero-plugins/namespacescc/restore_test.go @@ -0,0 +1,133 @@ +package namespacescc + +import ( + "reflect" + "testing" + + "github.com/konveyor/openshift-velero-plugin/velero-plugins/util/test" + apisecurity "github.com/openshift/api/security/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +func TestRestorePluginAppliesTo(t *testing.T) { + restorePlugin := &RestorePlugin{Log: test.NewLogger()} + actual, err := restorePlugin.AppliesTo() + require.NoError(t, err) + assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"serviceaccounts"}}, actual) +} + +func Test_expectedSCCAnnotations(t *testing.T) { + tests := []struct { + name string + backedUpAnnotations map[string]string + want map[string]string + }{ + { + name: "no bookkeeping annotations", + backedUpAnnotations: map[string]string{"kubernetes.io/service-account.name": "default"}, + want: map[string]string{}, + }, + { + name: "bookkeeping annotations decoded back to real SCC annotation names", + backedUpAnnotations: map[string]string{ + "oadp.openshift.io/backup-ns-scc-uid-range": "1000700000/10000", + "oadp.openshift.io/backup-ns-scc-mcs": "s0:c26,c5", + }, + want: map[string]string{ + apisecurity.UIDRangeAnnotation: "1000700000/10000", + apisecurity.MCSAnnotation: "s0:c26,c5", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := expectedSCCAnnotations(tt.backedUpAnnotations); !reflect.DeepEqual(got, tt.want) { + t.Errorf("expectedSCCAnnotations() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_sccAnnotationMismatches(t *testing.T) { + tests := []struct { + name string + expected map[string]string + actual map[string]string + want []string + }{ + { + name: "matching ranges produce no mismatches", + expected: map[string]string{ + apisecurity.UIDRangeAnnotation: "1000700000/10000", + }, + actual: map[string]string{ + apisecurity.UIDRangeAnnotation: "1000700000/10000", + }, + want: nil, + }, + { + name: "differing uid-range produces one mismatch", + expected: map[string]string{ + apisecurity.UIDRangeAnnotation: "1000700000/10000", + }, + actual: map[string]string{ + apisecurity.UIDRangeAnnotation: "1000710000/10000", + }, + want: []string{ + `[namespacescc-restore] namespace "ns-1" annotation "openshift.io/sa.scc.uid-range" mismatch: backed up as "1000700000/10000", restored as "1000710000/10000". Workloads relying on the backed-up UID/GID range may have incorrect file ownership.`, + }, + }, + { + name: "annotation missing on actual namespace produces a mismatch", + expected: map[string]string{ + apisecurity.MCSAnnotation: "s0:c26,c5", + }, + actual: map[string]string{}, + want: []string{ + `[namespacescc-restore] namespace "ns-1" annotation "openshift.io/sa.scc.mcs" mismatch: backed up as "s0:c26,c5", restored as "". Workloads relying on the backed-up UID/GID range may have incorrect file ownership.`, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := sccAnnotationMismatches("ns-1", tt.expected, tt.actual) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("sccAnnotationMismatches() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_stripSCCBookkeepingAnnotations(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + want map[string]string + }{ + { + name: "nil annotations stay nil", + annotations: nil, + want: nil, + }, + { + name: "bookkeeping annotations removed, others preserved", + annotations: map[string]string{ + "kubernetes.io/service-account.name": "default", + "oadp.openshift.io/backup-ns-scc-uid-range": "1000700000/10000", + }, + want: map[string]string{ + "kubernetes.io/service-account.name": "default", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripSCCBookkeepingAnnotations(tt.annotations) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("stripSCCBookkeepingAnnotations() = %v, want %v", got, tt.want) + } + }) + } +}