Add e2e coverage for kubevirt-datamover VM backup/restore: multi-PVC, incremental sequence, and CBT restore (+ max-concurrent-data-movers config)

Makefile

@@ -333,7 +333,7 @@ ifneq ($(shell $(OPERATOR_SDK) version | cut -d'"' -f2),$(OPERATOR_SDK_VERSION))
 	set -e; \
 	mkdir -p $(dir $(OPERATOR_SDK)) ;\
 	OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
-	curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$${OS}_$${ARCH} ;\
+	curl --retry 5 --retry-delay 5 -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$${OS}_$${ARCH} ;\
 	chmod +x $(OPERATOR_SDK);
 endif
 	@if [ -L "$(LOCALBIN)/operator-sdk" ]; then \
@@ -370,7 +370,7 @@ ifneq ($(shell $(OPM) version | cut -d'"' -f2),$(OPM_VERSION))
 	set -e ;\
 	mkdir -p $(dir $(OPM)) ;\
 	OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
-	curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/$(OPM_VERSION)/$${OS}-$${ARCH}-opm ;\
+	curl --retry 5 --retry-delay 5 -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/$(OPM_VERSION)/$${OS}-$${ARCH}-opm ;\
 	chmod +x $(OPM)
 endif
 	@if [ -L "$(LOCALBIN)/opm" ]; then \
@@ -499,7 +499,7 @@ else
     endif
 endif
 submit-coverage:
-	curl -Os https://uploader.codecov.io/latest/$(OS_String)/codecov
+	curl --retry 5 --retry-delay 5 -Os https://uploader.codecov.io/latest/$(OS_String)/codecov
 	chmod +x codecov
 	./codecov -C $(shell git rev-parse HEAD) -r openshift/oadp-operator --nonZero
 	rm -f codecov
@@ -965,6 +965,13 @@ ARTIFACT_DIR ?= /tmp
 HCO_UPSTREAM ?= false
 TEST_VIRT_GA ?= false
 TEST_VIRT ?= false
+# TEST_VIRT_KDM runs only the kubevirt-datamover-specific specs (ginkgo label
+# "kdm", a subset of "virt") -- for CI jobs that build/test against the
+# kubevirt-datamover-controller/-plugin repos specifically and don't need the
+# full TEST_VIRT suite's runtime. TEST_VIRT=true already covers these specs
+# too, since they carry both labels -- this only matters when TEST_VIRT_KDM
+# is set WITHOUT TEST_VIRT.
+TEST_VIRT_KDM ?= false
 HCO_INDEX_TAG ?= 1.18.0
 # hcp
 TEST_HCP ?= false
@@ -989,9 +996,21 @@ ifeq ($(TEST_VIRT),true)
 	TEST_FILTER += && (virt)
 else ifeq ($(TEST_VIRT_GA),true)
 	TEST_FILTER += && (virt)
+else ifeq ($(TEST_VIRT_KDM),true)
+	TEST_FILTER += && (kdm)
 else
 	TEST_FILTER += && (! virt)
 endif
+# kdm specs need the same community-HCO/KubeVirt setup as the rest of the virt
+# suite (TEST_VIRT's own -hco_community wiring below) -- without this,
+# TEST_VIRT_KDM=true alone (i.e. without TEST_VIRT=true) would leave
+# -hco_community=false and skip installing HCO entirely, breaking the kdm-only
+# run before any spec even gets a VM to test against.
+ifeq ($(TEST_VIRT_KDM),true)
+HCO_COMMUNITY := true
+else
+HCO_COMMUNITY := $(TEST_VIRT)
+endif
 ifeq ($(TEST_UPGRADE),true)
 	TEST_FILTER += && (upgrade)
 else
@@ -1049,7 +1068,7 @@ test-e2e: test-e2e-setup install-ginkgo $(if $(MUST_GATHER_REPO),build-must-gath
 	-artifact_dir=$(ARTIFACT_DIR) \
 	-kvm_emulation=$(KVM_EMULATION) \
 	-hco_upstream=$(HCO_UPSTREAM) \
-	-hco_community=$(TEST_VIRT) \
+	-hco_community=$(HCO_COMMUNITY) \
 	-hco_index_tag=$(HCO_INDEX_TAG) \
 	-skipMustGather=$(SKIP_MUST_GATHER) \
 	$(HCP_EXTERNAL_ARGS) \

api/v1alpha1/dataprotectionapplication_types.go

@@ -683,6 +683,13 @@ type KubevirtDatamoverConfig struct {
 	// +optional
 	MaxIncrementalBackups *int32 `json:"maxIncrementalBackups,omitempty"`
 
+	// MaxConcurrentDataMovers is the maximum number of concurrent active
+	// DataUploads/DataDownloads per direction (DU and DD counted independently).
+	// 0 means unlimited (default behavior).
+	// +kubebuilder:validation:Minimum=0
+	// +optional
+	MaxConcurrentDataMovers *int32 `json:"maxConcurrentDataMovers,omitempty"`
+
 	// StaleDataUploadThreshold is the duration after which a DataUpload in an
 	// active phase is considered stale and will no longer block younger
 	// DataUploads for the same VM. Default is 2h.

build/ci-Dockerfile

@@ -10,26 +10,47 @@ RUN chmod +x tests/e2e/scripts/analyze_failures.sh
 
 # Install kubectl (multi-arch)
 ARG TARGETARCH
-RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH}/kubectl" && \
+# --retry: curl's default retry set already covers exit 6 (could not resolve
+# host) -- the exact transient DNS failure hit live in CI on this same class
+# of fetch (see the virtctl step below) -- along with connect/read/5xx errors.
+RUN curl --retry 5 --retry-delay 5 -LO "https://dl.k8s.io/release/$(curl --retry 5 --retry-delay 5 -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH}/kubectl" && \
     chmod +x kubectl && \
     mv kubectl /usr/local/bin/
 
 # Install virtctl for KubeVirt VM operations in E2E tests
-RUN export KV_VERSION=$(curl -s https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt) && \
-    curl -L -o virtctl "https://github.com/kubevirt/kubevirt/releases/download/${KV_VERSION}/virtctl-${KV_VERSION}-linux-${TARGETARCH}" && \
+# --retry: confirmed live in CI -- this exact curl failed with "Could not
+# resolve host: github.com" (transient DNS blip in the build environment),
+# failing the whole image build over a network hiccup unrelated to any code
+# change. curl's default retry-on-error set already covers that failure mode
+# (exit 6); no extra flags needed beyond --retry/--retry-delay.
+RUN export KV_VERSION=$(curl --retry 5 --retry-delay 5 -s https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt) && \
+    curl --retry 5 --retry-delay 5 -L -o virtctl "https://github.com/kubevirt/kubevirt/releases/download/${KV_VERSION}/virtctl-${KV_VERSION}-linux-${TARGETARCH}" && \
     chmod +x virtctl && \
     mv virtctl /usr/local/bin/
 
 # Install Claude CLI (native binary, no Node.js dependency)
-RUN curl -fsSL https://claude.ai/install.sh | bash && \
+RUN curl --retry 5 --retry-delay 5 -fsSL https://claude.ai/install.sh | bash && \
     ln -sf ~/.local/bin/claude /usr/local/bin/claude && \
     claude --version
 
 # Clone openshift/velero source code for failure analysis
 # Uses oadp-dev branch to match OADP operator development
-RUN git clone --depth 1 --branch oadp-dev \
-    https://github.com/openshift/velero.git \
-    /go/src/github.com/openshift/velero
+# Retried like the curl fetches above -- same category of transient
+# network/DNS blip, just via git instead of curl (no built-in --retry flag).
+# rm -rf before each attempt: a clone that fails partway through can leave a
+# non-empty target directory behind, which would make the next attempt fail
+# with "destination path already exists" instead of actually retrying.
+# The trailing `test -d .../.git` is load-bearing, not decoration: without it,
+# `&& break || sleep 5` makes the loop's own exit status the last `sleep`'s
+# (always 0), so RUN succeeds even after all 3 clone attempts fail -- silently
+# shipping an image with no velero checkout instead of failing the build here.
+RUN for i in 1 2 3; do \
+      rm -rf /go/src/github.com/openshift/velero && \
+      git clone --depth 1 --branch oadp-dev \
+        https://github.com/openshift/velero.git \
+        /go/src/github.com/openshift/velero && break || sleep 5; \
+    done; \
+    test -d /go/src/github.com/openshift/velero/.git
 
 RUN go mod download && \
     mkdir -p $(go env GOCACHE) && \

bundle/manifests/oadp-operator.clusterserviceversion.yaml

@@ -1249,6 +1249,13 @@ spec:
           - get
           - list
           - delete
+        - apiGroups:
+          - ""
+          resources:
+          - events
+          verbs:
+          - create
+          - patch
         - apiGroups:
           - ""
           resources:
@@ -1330,6 +1337,7 @@ spec:
           verbs:
           - get
           - list
+          - update
           - watch
         - apiGroups:
           - velero.io

bundle/manifests/oadp.openshift.io_dataprotectionapplications.yaml

@@ -231,6 +231,14 @@ spec:
                     kubevirtDatamover:
                       description: KubevirtDatamover configures the kubevirt-datamover-controller for VM backup/restore.
                       properties:
+                        maxConcurrentDataMovers:
+                          description: |-
+                            MaxConcurrentDataMovers is the maximum number of concurrent active
+                            DataUploads/DataDownloads per direction (DU and DD counted independently).
+                            0 means unlimited (default behavior).
+                          format: int32
+                          minimum: 0
+                          type: integer
                         maxIncrementalBackups:
                           description: |-
                             MaxIncrementalBackups is the maximum number of incremental backups per VM

config/crd/bases/oadp.openshift.io_dataprotectionapplications.yaml

@@ -231,6 +231,14 @@ spec:
                     kubevirtDatamover:
                       description: KubevirtDatamover configures the kubevirt-datamover-controller for VM backup/restore.
                       properties:
+                        maxConcurrentDataMovers:
+                          description: |-
+                            MaxConcurrentDataMovers is the maximum number of concurrent active
+                            DataUploads/DataDownloads per direction (DU and DD counted independently).
+                            0 means unlimited (default behavior).
+                          format: int32
+                          minimum: 0
+                          type: integer
                         maxIncrementalBackups:
                           description: |-
                             MaxIncrementalBackups is the maximum number of incremental backups per VM

config/kubevirt-datamover-controller_rbac/role.yaml

@@ -4,6 +4,13 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -85,6 +92,7 @@ rules:
   verbs:
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - velero.io

internal/controller/kubevirt_datamover_controller.go

@@ -255,6 +255,10 @@ func ensureKubevirtDatamoverRequiredSpecs(
 			args = append(args, fmt.Sprintf("--max-incremental-backups=%d",
 				*dpa.Spec.Configuration.KubevirtDatamover.MaxIncrementalBackups))
 		}
+		if dpa.Spec.Configuration.KubevirtDatamover.MaxConcurrentDataMovers != nil {
+			args = append(args, fmt.Sprintf("--max-concurrent-data-movers=%d",
+				*dpa.Spec.Configuration.KubevirtDatamover.MaxConcurrentDataMovers))
+		}
 		if dpa.Spec.Configuration.KubevirtDatamover.StaleDataUploadThreshold != nil {
 			args = append(args, fmt.Sprintf("--stale-dataupload-threshold=%s",
 				dpa.Spec.Configuration.KubevirtDatamover.StaleDataUploadThreshold.Duration.String()))