CSI-addons volume replication: design, driver services, and operator wiring

.claude/settings.json

@@ -0,0 +1 @@
+{"worktree": {"bgIsolation": "none"}}

.cspell-crd-redesign.txt

@@ -0,0 +1,159 @@
+# Spell-check terms for operator/docs/designs/crd-redesign and its test plans.
+#
+# Produced by running cspell with the repository's .cspell.json over the ten design
+# documents, their assets, and the nine test plans. Every entry below is a word that
+# run reported and the dictionary does not carry.
+#
+# This file is deliberately untracked and is a staging list, not a dictionary the
+# build reads. Nothing consumes it yet: MegaLinter reads .cspell.json's `words`, so
+# the groups below are the candidates for that list once each has been decided.
+#
+# Four groups, and only the last one is a judgment call.
+#
+#   kinds    Kind names, short names, and generated file names. Add as they are.
+#   code     Go, kubebuilder, and package identifiers appearing in prose. Add as they are.
+#   product  Product, protocol, and platform names not this repository's to spell.
+#            Add as they are, and check the casing variants are the ones actually used.
+#   prose    Words the design prose coins. Each is either added here or the prose is
+#            reworded; that decision has not been taken.
+
+# --- kinds (35) -----------------------------------------------
+autoplacement
+backupops
+backuppolicy
+backuprestore
+clusterdeploymentconfig
+controlplane
+controlplaneops
+cpops
+nodeset
+operatorops
+persistentvolume
+persistentvolumeclaims
+persistentvolumeops
+persistentvolumes
+pvops
+rollingrestart
+sbops
+sdops
+simplyblockdriver
+Snode
+snode
+spops
+storagebackup
+storagebackupops
+storagebackuppolicy
+storagebackups
+storageclass
+storageclasses
+storagecluster
+storageclusters
+storagedevice
+storagedeviceops
+storagepool
+storagepoolops
+storagepools
+
+# --- code (26) ------------------------------------------------
+corev
+cpinformer
+customresources
+envtest
+foundationdb
+kubebuilder
+livenessprobe
+metav
+metricsapi
+mongodbcommunity
+omitempty
+pernodeconfig
+printcolumn
+resizer
+sbcli
+SBTLS
+simplybk
+statemachine
+subresource
+subtest
+#   the two below are British spellings, and they are shipped Go test function
+#   names the test plans quote verbatim, not prose this corpus can reword.
+#   operator/internal/controller: TestSanitiseDNSLabel_*, *_NodeRollingRestart_Initialises
+Initialises
+Sanitise
+WEBAPI
+webapi
+webapimock
+webappapi
+
+# --- product (42) ---------------------------------------------
+AXYZ
+dhchap
+DHCHAP
+elon
+finalizers
+HCLS
+hostnames
+hostpath
+hugepages
+Ifname
+kubelet
+Kubelet
+lvol
+Lvol
+Lvols
+mbps
+Mbytes
+mbytes
+mgmt
+Mgmt
+MZQL
+NDCS
+ndcs
+npcs
+NPCS
+nvme
+NVME
+Nvme
+nvmeof
+Nvmf
+nvmf
+schedulable
+smartctl
+spdk
+Spdk
+SPDK
+subsys
+SUBSYS
+tolerations
+Tolerations
+VCPU
+vcpu
+
+# --- prose (26) -----------------------------------------------
+alertable
+antipattern
+catchable
+cutover
+deadlineless
+graphable
+metacharacter
+parameterizes
+provisionable
+Recasing
+regroupings
+Repointing
+sourceable
+streamability
+streamable
+unaddressable
+unasserted
+unbuilt
+undrainable
+unexempted
+unpartitioned
+Unprovisioned
+unreworked
+upserted
+upserts
+waivable
+

atlas-lib/controlplane/consistencygroups.go

@@ -0,0 +1,152 @@
+// Consistency-group reads the csi-addons VolumeGroup service needs: resolving the
+// backend consistency group a set of member volumes already belongs to (design
+// design-csi-addons-replication.md §14.3). A group is formed at provisioning by
+// the storage.simplyblock.io/consistency-group label; this only reads it back.
+package controlplane
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/google/uuid"
+
+	"github.com/simplyblock/atlas/errs"
+	"github.com/simplyblock/atlas/internal/cpapi"
+	"github.com/simplyblock/atlas/lvol"
+)
+
+// ConsistencyGroupForLvols returns the id of the backend consistency group in
+// clusterID whose current (open-epoch) membership is exactly lvolIDs.
+//
+// The members provisioned under one storage.simplyblock.io/consistency-group
+// label share one backend group, so the VolumeGroup service resolves the group
+// by its members rather than by the name it is handed (which is the caller's own
+// generated name, not the group's). Returns an error when no group's membership
+// matches exactly, so a partial or mixed selection is never silently grouped.
+func (c *Client) ConsistencyGroupForLvols(ctx context.Context, clusterID string, lvolIDs []string) (string, error) {
+	cluster, err := parseUUID("cluster id", clusterID)
+	if err != nil {
+		return "", err
+	}
+	if len(lvolIDs) == 0 {
+		return "", fmt.Errorf("no volumes given to resolve a consistency group")
+	}
+	want := make(map[string]bool, len(lvolIDs))
+	for _, id := range lvolIDs {
+		want[id] = true
+	}
+
+	listResp, err := c.api.ClustersConsistencyGroupsListApiV2ClustersClusterIdConsistencyGroupsGetWithResponse(
+		ctx, cluster, &cpapi.ClustersConsistencyGroupsListApiV2ClustersClusterIdConsistencyGroupsGetParams{})
+	if err != nil {
+		return "", fmt.Errorf("list consistency groups in cluster %s: %w", clusterID, err)
+	}
+	groups, err := payload("consistency groups in cluster "+clusterID,
+		listResp.JSON200, listResp.StatusCode(), listResp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	for _, g := range *groups {
+		members, err := c.consistencyGroupMembers(ctx, cluster, g.Id)
+		if err != nil {
+			return "", err
+		}
+		if sameStringSet(members, want) {
+			return g.Id.String(), nil
+		}
+	}
+	return "", fmt.Errorf("no consistency group in cluster %s has exactly the %d requested member(s)",
+		clusterID, len(lvolIDs))
+}
+
+// consistencyGroupMembers returns a group's current open-epoch member lvol ids.
+func (c *Client) consistencyGroupMembers(ctx context.Context, cluster, group uuid.UUID) ([]string, error) {
+	resp, err := c.api.ClustersConsistencyGroupsMembersApiV2ClustersClusterIdConsistencyGroupsGroupIdMembersGetWithResponse(
+		ctx, cluster, group)
+	if err != nil {
+		return nil, fmt.Errorf("members of consistency group %s: %w", group, err)
+	}
+	dtos, err := payload("members of consistency group "+group.String(),
+		resp.JSON200, resp.StatusCode(), resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	open := make([]string, 0, len(*dtos))
+	for _, m := range *dtos {
+		if m.RemovedSeq == 0 {
+			open = append(open, m.LvolId)
+		}
+	}
+	return open, nil
+}
+
+// sameStringSet reports whether ids is exactly the set want (same length, same
+// elements), so a resolved group is neither a superset nor a subset of the
+// requested members.
+func sameStringSet(ids []string, want map[string]bool) bool {
+	if len(ids) != len(want) {
+		return false
+	}
+	for _, id := range ids {
+		if !want[id] {
+			return false
+		}
+	}
+	return true
+}
+
+// ConsistencyGroupMemberHandles returns the volume handles
+// ("<cluster>:<pool>:<lvol>") of a consistency group's current (open-epoch)
+// members, in the order the control plane lists them.
+//
+// The member listing carries bare lvol ids. Every member of one group lives in
+// one storage pool (the backend refuses a cross-pool join), so the pool is
+// resolved once, from the first member, by probing the cluster's pools. An
+// empty group is an error: a caller mapping members (csi-addons destination
+// info) must never return an empty map as if it were complete.
+func (c *Client) ConsistencyGroupMemberHandles(
+	ctx context.Context, gh lvol.GroupHandle,
+) ([]lvol.VolumeHandle, error) {
+	cluster, err := parseUUID("cluster id", gh.ClusterID)
+	if err != nil {
+		return nil, err
+	}
+	group, err := parseUUID("consistency group id", gh.GroupID)
+	if err != nil {
+		return nil, err
+	}
+	members, err := c.consistencyGroupMembers(ctx, cluster, group)
+	if err != nil {
+		return nil, err
+	}
+	if len(members) == 0 {
+		return nil, fmt.Errorf("consistency group %s has no current members", gh)
+	}
+	pools, err := c.ListStoragePools(ctx, gh.ClusterID)
+	if err != nil {
+		return nil, err
+	}
+	poolID := ""
+	for _, p := range pools {
+		h := lvol.Handle{ClusterID: gh.ClusterID, PoolRef: p.ID, VolumeID: members[0]}
+		_, err := c.Volume(ctx, h.Handle())
+		if err == nil {
+			poolID = p.ID
+			break
+		}
+		if !errors.Is(err, errs.ErrNotFound) {
+			return nil, err
+		}
+	}
+	if poolID == "" {
+		return nil, fmt.Errorf("member %s of consistency group %s is in none of the %d pools of cluster %s: %w",
+			members[0], gh, len(pools), gh.ClusterID, errs.ErrNotFound)
+	}
+	out := make([]lvol.VolumeHandle, 0, len(members))
+	for _, m := range members {
+		out = append(out, lvol.Handle{ClusterID: gh.ClusterID, PoolRef: poolID, VolumeID: m}.Handle())
+	}
+	return out, nil
+}

atlas-lib/controlplane/consistencygroups_members_test.go

@@ -0,0 +1,72 @@
+package controlplane
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/simplyblock/atlas/lvol"
+)
+
+// The member listing carries bare lvol ids; the handles need the pool, which
+// the group's first member is probed for across the cluster's pools. Here it
+// lives in the SECOND pool, and a removed member is left out.
+func TestConsistencyGroupMemberHandlesResolvesThePool(t *testing.T) {
+	const (
+		group     = "c9c9c9c9-c9c9-4c9c-8c9c-c9c9c9c9c9c9"
+		m1        = "a1111111-1111-4111-8111-111111111111"
+		m2        = "b2222222-2222-4222-8222-222222222222"
+		gone      = "d4444444-4444-4444-8444-444444444444"
+		otherPool = "55555555-5555-5555-5555-555555555555"
+	)
+	pool := func(id, name string) string {
+		return `{"id":"` + id + `","cluster_id":"` + testCluster + `","name":"` + name + `",` +
+			`"max_size":1000,"capacity":{},"max_r_mbytes":0,"max_rw_iops":0,"max_rw_mbytes":0,` +
+			`"max_w_mbytes":0,"volume_max_size":0,"status":"active"}`
+	}
+	member := func(id string, removed int) string {
+		return `{"lvol_id":"` + id + `","joined_seq":1,"removed_seq":` + string(rune('0'+removed)) +
+			`,"node_id":"n","lvs_name":"l","online":true}`
+	}
+	c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/consistency-groups/"+group+"/members"):
+			_, _ = w.Write([]byte("[" + member(m1, 0) + "," + member(gone, 3) + "," + member(m2, 0) + "]"))
+		case strings.HasSuffix(r.URL.Path, "/storage-pools/"):
+			_, _ = w.Write([]byte("[" + pool(otherPool, "other") + "," + pool(testPool, "pool1") + "]"))
+		case strings.Contains(r.URL.Path, "/storage-pools/"+testPool+"/volumes/"+m1):
+			_, _ = w.Write([]byte(`{"id":"` + m1 + `","name":"pvc-1","pool_name":"pool1",` +
+				`"size":20971520,"ns_id":1,"nqn":"nqn.2023-02.io.simplyblock:c:lvol:v"}`))
+		default:
+			w.WriteHeader(http.StatusNotFound)
+			_, _ = w.Write([]byte(`{"detail":"not found"}`))
+		}
+	})
+
+	got, err := c.ConsistencyGroupMemberHandles(context.Background(),
+		lvol.GroupHandle{ClusterID: testCluster, GroupID: group})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []lvol.VolumeHandle{
+		lvol.VolumeHandle(testCluster + ":" + testPool + ":" + m1),
+		lvol.VolumeHandle(testCluster + ":" + testPool + ":" + m2),
+	}
+	if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] {
+		t.Fatalf("handles = %v, want %v", got, want)
+	}
+}
+
+func TestConsistencyGroupMemberHandlesOfAnEmptyGroupIsAnError(t *testing.T) {
+	c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`[]`))
+	})
+	_, err := c.ConsistencyGroupMemberHandles(context.Background(),
+		lvol.GroupHandle{ClusterID: testCluster, GroupID: "c9c9c9c9-c9c9-4c9c-8c9c-c9c9c9c9c9c9"})
+	if err == nil {
+		t.Fatal("an empty group answered without an error")
+	}
+}