From b4855e6990d13aa871d734c235e18ebb64fb5c6f Mon Sep 17 00:00:00 2001
From: OpenShift CI Bot <ci-bot@redhat.com>
Date: Fri, 28 Aug 2026 09:10:56 +0000
Subject: [PATCH] fix(operator): retry on conflict in syncMachineConfigNodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The MachineConfigNode controller's resync loop performs a
Get-then-Update on MCN objects without any conflict retry. When
concurrent writes from the daemon or drain controller change the
object's resourceVersion between Get and Update, the update fails
with an optimistic-concurrency conflict. This error propagates through
syncAll to syncDegradedStatus, which surfaces a transient
Degraded=True condition with reason MachineConfigNodeFailed — tripping
the CVO invariant monitor in CI even though the condition self-clears.

Wrap the ApplyMachineConfigNode call with retry.OnError using the
existing IsApplyErrorRetriable predicate (which already handles
conflicts, RPC errors, and timeouts) and DefaultRetry backoff. This
matches the retry pattern already used by applyManifests and other
sync functions in the same file.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 lib/resourceapply/machineconfig_test.go | 38 +++++++++++++++++++++++++
 pkg/operator/sync.go                    |  8 ++++--
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/lib/resourceapply/machineconfig_test.go b/lib/resourceapply/machineconfig_test.go
index 3103fc78d9..703d02de66 100644
--- a/lib/resourceapply/machineconfig_test.go
+++ b/lib/resourceapply/machineconfig_test.go
@@ -10,8 +10,10 @@ import (
 	"github.com/openshift/client-go/machineconfiguration/clientset/versioned/fake"
 	"github.com/openshift/machine-config-operator/test/helpers"
 	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/diff"
 	clienttesting "k8s.io/client-go/testing"
 )
@@ -296,3 +298,39 @@ func TestApplyMachineConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestApplyMachineConfigNodeConflictReturnsRetriableError(t *testing.T) {
+	existing := &mcfgv1.MachineConfigNode{
+		ObjectMeta: metav1.ObjectMeta{Name: "node-1", ResourceVersion: "100"},
+		Spec: mcfgv1.MachineConfigNodeSpec{
+			Pool: mcfgv1.MCOObjectReference{Name: "worker"},
+		},
+	}
+	required := &mcfgv1.MachineConfigNode{
+		ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
+		Spec: mcfgv1.MachineConfigNodeSpec{
+			Pool: mcfgv1.MCOObjectReference{Name: "master"},
+		},
+	}
+
+	client := fake.NewSimpleClientset(existing)
+	conflictErr := apierrors.NewConflict(
+		schema.GroupResource{Group: "machineconfiguration.openshift.io", Resource: "machineconfignodes"},
+		"node-1",
+		fmt.Errorf("the object has been modified; please apply your changes to the latest version and try again"),
+	)
+	client.PrependReactor("update", "machineconfignodes", func(action clienttesting.Action) (bool, runtime.Object, error) {
+		return true, nil, conflictErr
+	})
+
+	_, _, err := ApplyMachineConfigNode(client.MachineconfigurationV1(), required)
+	if err == nil {
+		t.Fatal("expected conflict error, got nil")
+	}
+	if !apierrors.IsConflict(err) {
+		t.Fatalf("expected conflict error, got: %v", err)
+	}
+	if !IsApplyErrorRetriable(err) {
+		t.Fatal("conflict error should be retriable via IsApplyErrorRetriable")
+	}
+}
diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go
index 7ce6191150..ed1a15cdad 100644
--- a/pkg/operator/sync.go
+++ b/pkg/operator/sync.go
@@ -894,8 +894,12 @@ func (optr *Operator) syncMachineConfigNodes(_ *renderConfig, _ *configv1.Cluste
 			return err
 		}
 		p := mcoResourceRead.ReadMachineConfigNodeV1OrDie(mcsBytes)
-		mcn, _, err := mcoResourceApply.ApplyMachineConfigNode(optr.client.MachineconfigurationV1(), p)
-		if err != nil {
+		var mcn *mcfgv1.MachineConfigNode
+		if err := retry.OnError(retry.DefaultRetry, mcoResourceApply.IsApplyErrorRetriable, func() error {
+			var applyErr error
+			mcn, _, applyErr = mcoResourceApply.ApplyMachineConfigNode(optr.client.MachineconfigurationV1(), p)
+			return applyErr
+		}); err != nil {
 			return err
 		}
 		// if this is the first time we are applying the MCN and the node is ready, set the config version probably