LIB-876: Propagate custom users to shard replsets in sharded clusters

config/crd/patches/versionlabel_in_psmdb.yaml

@@ -4,6 +4,6 @@ metadata:
   name: perconaservermongodbs.psmdb.percona.com
   labels:
     app.kubernetes.io/name: percona-server-mongodb
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
     app.kubernetes.io/component: crd
     app.kubernetes.io/part-of: percona-server-mongodb-operator

config/crd/patches/versionlabel_in_psmdbbackup.yaml

@@ -4,6 +4,6 @@ metadata:
   name: perconaservermongodbbackups.psmdb.percona.com
   labels:
     app.kubernetes.io/name: percona-server-mongodb
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
     app.kubernetes.io/component: crd
     app.kubernetes.io/part-of: percona-server-mongodb-operator

config/crd/patches/versionlabel_in_psmdbrestore.yaml

@@ -4,6 +4,6 @@ metadata:
   name: perconaservermongodbrestores.psmdb.percona.com
   labels:
     app.kubernetes.io/name: percona-server-mongodb
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
     app.kubernetes.io/component: crd
     app.kubernetes.io/part-of: percona-server-mongodb-operator

deploy/bundle.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbbackups.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -284,7 +284,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbrestores.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -571,7 +571,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbs.psmdb.percona.com
 spec:
   group: psmdb.percona.com

deploy/crd.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbbackups.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -284,7 +284,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbrestores.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -571,7 +571,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbs.psmdb.percona.com
 spec:
   group: psmdb.percona.com

deploy/cw-bundle.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbbackups.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -284,7 +284,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbrestores.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -571,7 +571,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0-3
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbs.psmdb.percona.com
 spec:
   group: psmdb.percona.com

e2e-tests/version-service/conf/crd.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbbackups.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -284,7 +284,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbrestores.psmdb.percona.com
 spec:
   group: psmdb.percona.com
@@ -571,7 +571,7 @@ metadata:
     app.kubernetes.io/component: crd
     app.kubernetes.io/name: percona-server-mongodb
     app.kubernetes.io/part-of: percona-server-mongodb-operator
-    app.kubernetes.io/version: v1.22.0
+    app.kubernetes.io/version: v1.22.0-4
   name: perconaservermongodbs.psmdb.percona.com
 spec:
   group: psmdb.percona.com

pkg/controller/perconaservermongodb/custom_users.go

@@ -56,9 +56,52 @@ func (r *ReconcilePerconaServerMongoDB) reconcileCustomUsers(ctx context.Context
 		return errors.Wrap(err, "handle users")
 	}
 
+	// For sharded clusters, custom users are created via mongos which stores them
+	// on the config server only. Individual shard replica sets maintain their own
+	// local admin databases and are unaware of users created at the cluster level.
+	// Only clusterSuperAdmin needs direct shard access for platform operations;
+	// other custom users should remain cluster-level only (accessed via mongos).
+	if cr.Spec.Sharding.Enabled {
+		shardUsers := filterShardUsers(cr.Spec.Users)
+		if len(shardUsers) > 0 {
+			shardCR := cr.DeepCopy()
+			shardCR.Spec.Users = shardUsers
+
+			for i := range cr.Spec.Replsets {
+				rs := cr.Spec.Replsets[i]
+				shardCli, err := r.mongoClientWithRole(ctx, cr, rs, api.RoleUserAdmin)
+				if err != nil {
+					log.Error(err, "failed to get mongo client for shard", "replset", rs.Name)
+					continue
+				}
+
+				if err := handleUsers(ctx, shardCR, shardCli, r.client); err != nil {
+					log.Error(err, "failed to handle users on shard", "replset", rs.Name)
+				}
+
+				if err := shardCli.Disconnect(ctx); err != nil {
+					log.Error(err, "failed to close shard mongo connection", "replset", rs.Name)
+				}
+			}
+		}
+	}
+
 	return nil
 }
 
+// filterShardUsers returns only the users that need to be created directly on
+// each shard replica set. Currently this is limited to clusterSuperAdmin which
+// requires direct shard access for platform operations (compaction, backups, etc).
+func filterShardUsers(users []api.User) []api.User {
+	var filtered []api.User
+	for i := range users {
+		if users[i].Name == "clusterSuperAdmin" {
+			filtered = append(filtered, users[i])
+		}
+	}
+	return filtered
+}
+
 func handleUsers(ctx context.Context, cr *api.PerconaServerMongoDB, mongoCli mongo.Client, client client.Client) error {
 	log := logf.FromContext(ctx)
 

pkg/controller/perconaservermongodb/custom_users_test.go

@@ -2,10 +2,13 @@ package perconaservermongodb
 
 import (
 	"context"
+	"sync"
 	"testing"
 
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/mongo/readpref"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -411,3 +414,188 @@ func TestGetCustomUserSecret(t *testing.T) {
 		})
 	}
 }
+
+// trackingMongoClientProvider records which connections (mongos vs shard replset)
+// were made during reconcileCustomUsers.
+type trackingMongoClientProvider struct {
+	mu          sync.Mutex
+	mongosCount int
+	shardCalls  []string // replset names connected to directly
+	client      client.Client
+}
+
+func (p *trackingMongoClientProvider) Mongo(ctx context.Context, cr *api.PerconaServerMongoDB, rs *api.ReplsetSpec, role api.SystemUserRole) (mongo.Client, error) {
+	p.mu.Lock()
+	p.shardCalls = append(p.shardCalls, rs.Name)
+	p.mu.Unlock()
+	return &noopMongoClient{client: p.client, cr: cr}, nil
+}
+
+func (p *trackingMongoClientProvider) Mongos(ctx context.Context, cr *api.PerconaServerMongoDB, role api.SystemUserRole) (mongo.Client, error) {
+	p.mu.Lock()
+	p.mongosCount++
+	p.mu.Unlock()
+	return &noopMongoClient{client: p.client, cr: cr}, nil
+}
+
+func (p *trackingMongoClientProvider) Standalone(ctx context.Context, cr *api.PerconaServerMongoDB, role api.SystemUserRole, host string, tlsEnabled bool) (mongo.Client, error) {
+	return &noopMongoClient{client: p.client, cr: cr}, nil
+}
+
+// noopMongoClient is a mongo.Client that returns empty results for all calls —
+// sufficient for testing that the right connections are made.
+type noopMongoClient struct {
+	client client.Client
+	cr     *api.PerconaServerMongoDB
+}
+
+func (c *noopMongoClient) Disconnect(ctx context.Context) error { return nil }
+func (c *noopMongoClient) Database(name string, opts ...*options.DatabaseOptions) mongo.ClientDatabase {
+	return nil
+}
+func (c *noopMongoClient) Ping(ctx context.Context, rp *readpref.ReadPref) error { return nil }
+func (c *noopMongoClient) GetUserInfo(ctx context.Context, username, db string) (*mongo.User, error) {
+	return nil, nil
+}
+func (c *noopMongoClient) CreateUser(ctx context.Context, db, username, pwd string, roles ...mongo.Role) error {
+	return nil
+}
+func (c *noopMongoClient) UpdateUserPass(ctx context.Context, db, name, pass string) error {
+	return nil
+}
+func (c *noopMongoClient) UpdateUserRoles(ctx context.Context, db, username string, roles []mongo.Role) error {
+	return nil
+}
+func (c *noopMongoClient) UpdateUser(ctx context.Context, currName, newName, pass string) error {
+	return nil
+}
+func (c *noopMongoClient) GetRole(ctx context.Context, db, role string) (*mongo.Role, error) {
+	return nil, nil
+}
+func (c *noopMongoClient) CreateRole(ctx context.Context, db string, role mongo.Role) error {
+	return nil
+}
+func (c *noopMongoClient) UpdateRole(ctx context.Context, db string, role mongo.Role) error {
+	return nil
+}
+func (c *noopMongoClient) RSBuildInfo(ctx context.Context) (mongo.BuildInfo, error) {
+	return mongo.BuildInfo{}, nil
+}
+func (c *noopMongoClient) RSStatus(ctx context.Context) (mongo.Status, error) {
+	return mongo.Status{}, nil
+}
+func (c *noopMongoClient) WriteConfig(ctx context.Context, cfg mongo.RSConfig, force bool) error {
+	return nil
+}
+func (c *noopMongoClient) ReadConfig(ctx context.Context) (mongo.RSConfig, error) {
+	return mongo.RSConfig{}, nil
+}
+func (c *noopMongoClient) StartBalancer(ctx context.Context) error { return nil }
+func (c *noopMongoClient) StopBalancer(ctx context.Context) error  { return nil }
+func (c *noopMongoClient) IsBalancerRunning(ctx context.Context) (bool, error) {
+	return false, nil
+}
+func (c *noopMongoClient) GetFCV(ctx context.Context) (string, error)       { return "", nil }
+func (c *noopMongoClient) SetFCV(ctx context.Context, version string) error { return nil }
+func (c *noopMongoClient) ListDBs(ctx context.Context) (mongo.DBList, error) {
+	return mongo.DBList{}, nil
+}
+func (c *noopMongoClient) ListShard(ctx context.Context) (mongo.ShardList, error) {
+	return mongo.ShardList{}, nil
+}
+func (c *noopMongoClient) RemoveShard(ctx context.Context, shard string) (mongo.ShardRemoveResp, error) {
+	return mongo.ShardRemoveResp{}, nil
+}
+func (c *noopMongoClient) StepDown(ctx context.Context, seconds int, force bool) error { return nil }
+func (c *noopMongoClient) IsMaster(ctx context.Context) (*mongo.IsMasterResp, error) {
+	return &mongo.IsMasterResp{}, nil
+}
+func (c *noopMongoClient) Freeze(ctx context.Context, seconds int) error { return nil }
+func (c *noopMongoClient) SetDefaultRWConcern(ctx context.Context, readConcern, writeConcern string) error {
+	return nil
+}
+func (c *noopMongoClient) AddShard(ctx context.Context, rsName, host string) error { return nil }
+
+// TestReconcileCustomUsers_ShardedPropagation verifies that for a sharded cluster,
+// reconcileCustomUsers connects to mongos once (cluster-level) AND to each shard
+// replset directly, ensuring custom users like clusterSuperAdmin are created on shards.
+func TestReconcileCustomUsers_ShardedPropagation(t *testing.T) {
+	ctx := context.Background()
+	scheme := runtime.NewScheme()
+	assert.NoError(t, corev1.AddToScheme(scheme))
+	assert.NoError(t, api.SchemeBuilder.AddToScheme(scheme))
+
+	ns := "test-ns"
+	clusterName := "test-cluster"
+
+	// Build internal secret so fetchSystemUserNames doesn't fail
+	internalSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "internal-" + clusterName + "-users",
+			Namespace: ns,
+		},
+		Data: map[string][]byte{},
+	}
+
+	// Password secret for clusterSuperAdmin
+	passSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "internal-" + clusterName + "-users",
+			Namespace: ns,
+		},
+		Data: map[string][]byte{
+			"MONGODB_CLUSTER_SUPER_ADMIN_PASSWORD": []byte("supersecret"),
+		},
+	}
+	_ = passSecret
+
+	fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(internalSecret).Build()
+
+	cr := &api.PerconaServerMongoDB{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: ns,
+		},
+		Spec: api.PerconaServerMongoDBSpec{
+			Sharding: api.Sharding{
+				Enabled: true,
+			},
+			Replsets: []*api.ReplsetSpec{
+				{Name: "rs0"},
+				{Name: "rs1"},
+			},
+			Users: []api.User{
+				{
+					Name: "clusterSuperAdmin",
+					DB:   "admin",
+					PasswordSecretRef: &api.SecretKeySelector{
+						Name: "internal-" + clusterName + "-users",
+						Key:  "MONGODB_CLUSTER_SUPER_ADMIN_PASSWORD",
+					},
+					Roles: []api.UserRole{
+						{Name: "clusterAdmin", DB: "admin"},
+					},
+				},
+			},
+		},
+		Status: api.PerconaServerMongoDBStatus{
+			State: api.AppStateReady,
+		},
+	}
+
+	tracker := &trackingMongoClientProvider{client: fakeClient}
+	r := &ReconcilePerconaServerMongoDB{
+		client:              fakeClient,
+		mongoClientProvider: tracker,
+	}
+
+	err := r.reconcileCustomUsers(ctx, cr)
+	assert.NoError(t, err)
+
+	// Must have connected to mongos exactly once
+	assert.Equal(t, 1, tracker.mongosCount, "expected exactly 1 mongos connection")
+
+	// Must have connected to each shard replset directly
+	assert.ElementsMatch(t, []string{"rs0", "rs1"}, tracker.shardCalls,
+		"expected direct connections to each shard replset")
+}

pkg/version/version.txt

@@ -1 +1 @@
-1.22.0-3
+1.22.0-4