fix(iouring): stop a ring SEND completion parking the worker on a running async handler's detachMu (celeris#750)

.github/scripts/mutant-587-unlock-zc-first-cqe.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 """celeris#587 detector control: the MUTANT, applied in CI and never committed.
 
-Deletes the cs.detachMu acquire in handleSend's CQE_F_MORE branch
+Releases cs.detachMu at the top of handleSend's CQE_F_MORE branch
 (engine/iouring/worker.go), i.e. the SEND_ZC first completion then writes
 cs.sending / cs.zcNotifPending / cs.zcSentBytes with no lock, while the
 inline-egress guard on the dispatch goroutine reads them under the lock.
+handleSend takes the lock once, at its top (celeris#750), and the branch
+defers its release; the mutant releases it at once instead.
 
 TestSendZCWindowGuardUnderRace, run under -race against the mutated tree,
 MUST fail with a "WARNING: DATA RACE" report. If it does not, the race
@@ -22,24 +24,29 @@
 
 ANCHOR = "\tif cqeHasMore(c.Flags) {\n"
 LOCK = (
-    "\t\tif mu := cs.detachMu; mu != nil {\n"
-    "\t\t\tmu.Lock()\n"
+    "\t\tif mu != nil {\n"
     "\t\t\tdefer mu.Unlock()\n"
     "\t\t}\n"
 )
+MUTANT = (
+    "\t\t// MUTANT celeris#587: detachMu released before the F_MORE writes\n"
+    "\t\tif mu != nil {\n"
+    "\t\t\tmu.Unlock()\n"
+    "\t\t}\n"
+)
 
 src = open(PATH).read()
 if src.count(ANCHOR) != 1:
     print(f"mutant-587: expected exactly one F_MORE branch anchor, found {src.count(ANCHOR)}", file=sys.stderr)
     sys.exit(2)
 start = src.index(ANCHOR) + len(ANCHOR)
-# The lock must be the first statement block of the branch (after its
-# comment lines), within a short distance of the anchor.
+# The deferred release must be the first statement block of the branch
+# (after its comment lines), within a short distance of the anchor.
 window = src[start:start + 600]
 if window.count(LOCK) != 1:
-    print("mutant-587: the detachMu acquire was not found at the top of the F_MORE branch", file=sys.stderr)
+    print("mutant-587: the deferred detachMu release was not found at the top of the F_MORE branch", file=sys.stderr)
     sys.exit(2)
 i = start + window.index(LOCK)
-mutated = src[:i] + "\t\t// MUTANT celeris#587: detachMu acquire deleted\n" + src[i + len(LOCK):]
+mutated = src[:i] + MUTANT + src[i + len(LOCK):]
 open(PATH, "w").write(mutated)
-print(f"mutant-587: deleted the detachMu acquire in handleSend's CQE_F_MORE branch ({PATH})")
+print(f"mutant-587: released detachMu before the writes of handleSend's CQE_F_MORE branch ({PATH})")

.github/workflows/ci.yml

@@ -947,7 +947,8 @@ jobs:
       #   2. CELERIS_IOURING_SEND_ZC=off: every process PASS with every ZC
       #      witness at 0 (the witnesses track SEND_ZC, not traffic), no race;
       #   3. the MUTANT (.github/scripts/mutant-587-unlock-zc-first-cqe.py
-      #      deletes the detachMu acquire in handleSend's CQE_F_MORE branch):
+      #      releases detachMu before the writes of handleSend's CQE_F_MORE
+      #      branch):
       #      EVERY process FAIL, and every process with its own
       #      "WARNING: DATA RACE" report and "race detected during execution
       #      of test" line. A process that fails for any other reason does

engine/iouring/conn.go

@@ -273,10 +273,26 @@ type connState struct {
 	closeOwed bool
 	// relinkOwed (guarded by asyncInMu) is set by the dirty-list pass when it
 	// gives the conn up because its dispatch goroutine holds detachMu across
-	// a handler (celeris#704). The goroutine hands cs back through the detach
-	// queue at the top of its next loop, after the handler's own flush, and
-	// drainDetachQueue puts it on the dirty list again.
+	// a handler (celeris#704), and by a send completion held for the same
+	// reason (heldSends, celeris#750). The goroutine hands cs back through
+	// the detach queue at the top of its next loop, after the handler's own
+	// flush, and drainDetachQueue applies the held completions and puts it
+	// on the dirty list again.
 	relinkOwed bool
+	// heldSends (worker-thread only) are the ring SEND completions of this
+	// conn that arrived while its dispatch goroutine held detachMu across a
+	// handler, in arrival order (a SEND_ZC gives two). handleSend applies a
+	// completion under detachMu, and waiting for the lock parked the worker,
+	// and every connection of its ring, until the handler returned
+	// (celeris#750, a fifth celeris#704 site). A completion cannot be dropped:
+	// it is held, with relinkOwed set, and replayHeldSends applies it when the
+	// goroutine hands the conn back, or when the conn is closed, before
+	// anything else acts on the conn. Until then cs.sending (or
+	// zcNotifPending) stays set, so no other SEND starts and every raw write
+	// waits (celeris#751), and the dirty-list pass gives the conn up, which
+	// it would spin on otherwise (flushDirty). The kernel's side of each is
+	// done: kernelInflight was settled when it was dispatched.
+	heldSends []completionEntry
 	// closeErr (worker-thread only) is the error handleRecv's peer-FIN or
 	// recv-error branch owes a detached middleware (OnError) when it met a
 	// running handler holding detachMu. The branch used to deliver it under
@@ -539,6 +555,7 @@ func releaseConnState(cs *connState) {
 	cs.asyncParked = false
 	cs.closeOwed = false
 	cs.relinkOwed = false
+	cs.heldSends = cs.heldSends[:0]
 	cs.closeErr = nil
 	cs.transplantPending.Store(false)
 	cs.sweepKick = nil

engine/iouring/send_completion_stall_fields_linux_test.go

@@ -0,0 +1,9 @@
+//go:build linux
+
+package iouring
+
+// The connState field the celeris#750 tests read.
+
+// heldSends750 reports how many of cs's send completions are held for its
+// dispatch goroutine's hand-back. Worker thread (the test's).
+func heldSends750(cs *connState) int { return len(cs.heldSends) }