From 3ef618b02b08d0d3eedb78398a4d2d1de20717c0 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 04:54:51 +0000 Subject: [PATCH 1/6] SOLR-6213: run the leader rejoin on its own thread so repeated rejoins do not grow the stack --- .../cloud/ShardLeaderElectionContext.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index 7512867a794d..f1f874b0efe3 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -18,6 +18,7 @@ import java.lang.invoke.MethodHandles; import java.util.EnumSet; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.lucene.search.MatchAllDocsQuery; @@ -33,6 +34,8 @@ import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.logging.MDCLoggingContext; @@ -56,6 +59,10 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase { private volatile boolean isClosed = false; + private final ExecutorService rejoinExecutor = + ExecutorUtil.newMDCAwareSingleLazyThreadExecutor( + new SolrNamedThreadFactory("leaderRejoin"), 10, TimeUnit.SECONDS); + public ShardLeaderElectionContext( LeaderElector leaderElector, final String shardId, @@ -74,6 +81,7 @@ public ShardLeaderElectionContext( public void close() { super.close(); this.isClosed = true; + rejoinExecutor.shutdownNow(); syncStrategy.close(); } @@ -482,7 +490,7 @@ private boolean waitForReplicasToComeUp(int timeoutms) throws InterruptedExcepti private void rejoinLeaderElection(SolrCore core) throws InterruptedException, KeeperException { // remove our ephemeral and re join the election - if (cc.isShutDown()) { + if (cc.isShutDown() || isClosed) { log.debug("Not rejoining election because CoreContainer is closed"); return; } @@ -491,6 +499,16 @@ private void rejoinLeaderElection(SolrCore core) throws InterruptedException, Ke cancelElection(); - leaderElector.joinElection(this, true); + rejoinExecutor.execute(this::joinElectionAfterRejoin); + } + + private void joinElectionAfterRejoin() { + try { + leaderElector.joinElection(this, true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (KeeperException e) { + log.error("Failed to rejoin the election", e); + } } } From 4be5d1a69b94f2075149da69b150b99cc4dec25f Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 05:42:33 +0000 Subject: [PATCH 2/6] SOLR-6213: let a rejoin that races with close stop without an error, and keep an in-flight join intact --- .../solr/cloud/ShardLeaderElectionContext.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index f1f874b0efe3..b594885335ea 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -20,6 +20,7 @@ import java.util.EnumSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; @@ -81,7 +82,7 @@ public ShardLeaderElectionContext( public void close() { super.close(); this.isClosed = true; - rejoinExecutor.shutdownNow(); + rejoinExecutor.shutdown(); syncStrategy.close(); } @@ -499,10 +500,17 @@ private void rejoinLeaderElection(SolrCore core) throws InterruptedException, Ke cancelElection(); - rejoinExecutor.execute(this::joinElectionAfterRejoin); + try { + rejoinExecutor.execute(this::joinElectionAfterRejoin); + } catch (RejectedExecutionException e) { + log.debug("Not rejoining election because the context is closed"); + } } private void joinElectionAfterRejoin() { + if (isClosed) { + return; + } try { leaderElector.joinElection(this, true); } catch (InterruptedException e) { From 27ad9870cc9413a19681d61eace9bb2be246a9e6 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 06:34:51 +0000 Subject: [PATCH 3/6] SOLR-6213: log a runtime failure of the rejoin instead of leaving it to the thread's uncaught handler --- .../java/org/apache/solr/cloud/ShardLeaderElectionContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index b594885335ea..855eab008ebc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -515,7 +515,7 @@ private void joinElectionAfterRejoin() { leaderElector.joinElection(this, true); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - } catch (KeeperException e) { + } catch (KeeperException | RuntimeException e) { log.error("Failed to rejoin the election", e); } } From bcea4f23c1944cf0383193c66fccdb9e6e624567 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 07:04:56 +0000 Subject: [PATCH 4/6] SOLR-6213: close the election context when its core is unregistered The context is removed from the map there and only its election node was cancelled, so its rejoin thread and sync executor were never shut down. --- solr/core/src/java/org/apache/solr/cloud/ZkController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 5db9f8d7256a..0efd0ce2768e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -2044,6 +2044,7 @@ public void unregister(String coreName, CoreDescriptor cd, boolean removeCoreFro if (context != null) { context.cancelElection(); + context.close(); } } CloudDescriptor cloudDescriptor = cd.getCloudDescriptor(); From 1fdc74f35021063d94cc6bb41a86892876ee5abb Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 09:38:06 -0600 Subject: [PATCH 5/6] SOLR-6213: changelog entry --- changelog/unreleased/SOLR-6213-rejoin-thread.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/SOLR-6213-rejoin-thread.yml diff --git a/changelog/unreleased/SOLR-6213-rejoin-thread.yml b/changelog/unreleased/SOLR-6213-rejoin-thread.yml new file mode 100644 index 000000000000..d27b12d69b94 --- /dev/null +++ b/changelog/unreleased/SOLR-6213-rejoin-thread.yml @@ -0,0 +1,10 @@ +title: > + Leader election rejoins after a failed sync or leader publish now run on their own thread instead + of nesting on the caller's thread, so repeated rejoins no longer grow the stack toward + StackOverflowError. +type: fixed +authors: + - name: Nick Shanin +links: + - name: SOLR-6213 + url: https://issues.apache.org/jira/browse/SOLR-6213 From 2193e0104a3fa5c6235d4f843f1d6930afeb65d1 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 16:25:36 -0400 Subject: [PATCH 6/6] SOLR-6213: serialize rejoin cancellation and retain election regression tests --- .../unreleased/SOLR-6213-rejoin-thread.yml | 2 + .../cloud/ShardLeaderElectionContext.java | 51 ++-- .../org/apache/solr/cloud/ZkController.java | 2 +- .../cloud/ShardLeaderElectionRejoinTest.java | 238 ++++++++++++++++++ 4 files changed, 273 insertions(+), 20 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cloud/ShardLeaderElectionRejoinTest.java diff --git a/changelog/unreleased/SOLR-6213-rejoin-thread.yml b/changelog/unreleased/SOLR-6213-rejoin-thread.yml index d27b12d69b94..69e0ffa105d0 100644 --- a/changelog/unreleased/SOLR-6213-rejoin-thread.yml +++ b/changelog/unreleased/SOLR-6213-rejoin-thread.yml @@ -2,6 +2,8 @@ title: > Leader election rejoins after a failed sync or leader publish now run on their own thread instead of nesting on the caller's thread, so repeated rejoins no longer grow the stack toward StackOverflowError. + Closing and cancelling a replica's election waits for an in-flight rejoin to finish so it + cannot leave a stale election participant behind. type: fixed authors: - name: Nick Shanin diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index 855eab008ebc..f98335216182 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -60,6 +60,9 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase { private volatile boolean isClosed = false; + // Cancellation must see the election node created by an in-flight rejoin. + private final Object rejoinLock = new Object(); + private final ExecutorService rejoinExecutor = ExecutorUtil.newMDCAwareSingleLazyThreadExecutor( new SolrNamedThreadFactory("leaderRejoin"), 10, TimeUnit.SECONDS); @@ -96,14 +99,16 @@ private boolean shouldAbort() { @Override public void cancelElection() throws InterruptedException, KeeperException { - String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP); - try (SolrCore core = cc.getCore(coreName)) { - if (core != null) { - core.getCoreDescriptor().getCloudDescriptor().setLeader(false); + synchronized (rejoinLock) { + String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP); + try (SolrCore core = cc.getCore(coreName)) { + if (core != null) { + core.getCoreDescriptor().getCloudDescriptor().setLeader(false); + } } - } - super.cancelElection(); + super.cancelElection(); + } } @Override @@ -194,7 +199,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup if (zkShardTerms.registered(coreNodeName) && !zkShardTerms.canBecomeLeader(coreNodeName)) { if (!waitForEligibleBecomeLeaderAfterTimeout( zkShardTerms, coreNodeName, leaderVoteWait)) { - rejoinLeaderElection(core); + rejoinLeaderElection(); return; } else { // only log an error if this replica win the election @@ -251,7 +256,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup } } if (!success) { - rejoinLeaderElection(core); + rejoinLeaderElection(); return; } } @@ -324,7 +329,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup // we could not publish ourselves as leader - try and rejoin election try { - rejoinLeaderElection(core); + rejoinLeaderElection(); } catch (SessionExpiredException exc) { throw new SolrException( ErrorCode.SERVER_ERROR, @@ -489,7 +494,7 @@ private boolean waitForReplicasToComeUp(int timeoutms) throws InterruptedExcepti return false; } - private void rejoinLeaderElection(SolrCore core) throws InterruptedException, KeeperException { + void rejoinLeaderElection() throws InterruptedException, KeeperException { // remove our ephemeral and re join the election if (cc.isShutDown() || isClosed) { log.debug("Not rejoining election because CoreContainer is closed"); @@ -508,15 +513,23 @@ private void rejoinLeaderElection(SolrCore core) throws InterruptedException, Ke } private void joinElectionAfterRejoin() { - if (isClosed) { - return; - } - try { - leaderElector.joinElection(this, true); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (KeeperException | RuntimeException e) { - log.error("Failed to rejoin the election", e); + synchronized (rejoinLock) { + if (isClosed) { + return; + } + try { + try { + leaderElector.joinElection(this, true); + } finally { + if (isClosed) { + cancelElection(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (KeeperException | RuntimeException e) { + log.error("Failed to rejoin the election", e); + } } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 0efd0ce2768e..19037e39832e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -2043,8 +2043,8 @@ public void unregister(String coreName, CoreDescriptor cd, boolean removeCoreFro ElectionContext context = electionContexts.remove(new ContextKey(collection, coreNodeName)); if (context != null) { - context.cancelElection(); context.close(); + context.cancelElection(); } } CloudDescriptor cloudDescriptor = cd.getCloudDescriptor(); diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardLeaderElectionRejoinTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardLeaderElectionRejoinTest.java new file mode 100644 index 000000000000..f5f4dbfb9c7d --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/ShardLeaderElectionRejoinTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.solr.SolrTestCase; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.CoreContainer; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** Exercises rejoin scheduling and cancellation with real ZooKeeper election participants. */ +public class ShardLeaderElectionRejoinTest extends SolrTestCase { + private static final int WAIT_SECONDS = 30; + private static final int REJOINS = 20; + private static final String ELECTION_PATH = "/collections/rejoin/leader_elect/shard1/election"; + + private ZkTestServer server; + private SolrZkClient zkClient; + private ZkStateReader zkStateReader; + private CoreContainer cc; + private ZkController zkController; + + @Before + public void startZooKeeper() throws Exception { + assumeWorkingMockito(); + server = new ZkTestServer(createTempDir("rejoin-zk")); + server.run(); + zkClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(WAIT_SECONDS, TimeUnit.SECONDS) + .build(); + zkStateReader = new ZkStateReader(zkClient); + zkClient.makePath("/collections/rejoin", true); + cc = mock(CoreContainer.class, RETURNS_DEEP_STUBS); + when(cc.getCore("core")).thenReturn(null); + zkController = mock(ZkController.class); + when(zkController.getZkClient()).thenReturn(zkClient); + when(zkController.getZkStateReader()).thenReturn(zkStateReader); + } + + @After + public void stopZooKeeper() throws Exception { + try { + if (zkStateReader != null) { + zkStateReader.close(); + } + } finally { + try { + if (zkClient != null) { + zkClient.close(); + } + } finally { + if (server != null) { + server.shutdown(); + } + } + } + } + + @Test + public void testRepeatedRejoinDoesNotGrowStack() throws Exception { + List depths = new CopyOnWriteArrayList<>(); + CountDownLatch finished = new CountDownLatch(1); + LeaderElector elector = + new LeaderElector(zkClient) { + @Override + protected void runIamLeaderProcess(ElectionContext context, boolean replacement) + throws KeeperException, InterruptedException { + depths.add(Thread.currentThread().getStackTrace().length); + if (depths.size() < REJOINS) { + ((ShardLeaderElectionContext) context).rejoinLeaderElection(); + } else { + finished.countDown(); + } + } + }; + ShardLeaderElectionContext context = newContext(elector); + try { + context.rejoinLeaderElection(); + assertTrue("Rejoins did not finish", finished.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertEquals(REJOINS, depths.size()); + int min = depths.stream().mapToInt(Integer::intValue).min().orElseThrow(); + int max = depths.stream().mapToInt(Integer::intValue).max().orElseThrow(); + assertTrue("Stack grew by " + (max - min) + " frames", max - min < REJOINS); + } finally { + context.close(); + context.cancelElection(); + } + assertTrue(zkClient.getChildren(ELECTION_PATH, null).isEmpty()); + } + + @Test + public void testCancellationWaitsForRunningRejoin() throws Exception { + CountDownLatch joining = new CountDownLatch(1); + CountDownLatch allowJoin = new CountDownLatch(1); + CountDownLatch joined = new CountDownLatch(1); + CountDownLatch cancelling = new CountDownLatch(1); + LeaderElector elector = + new LeaderElector(zkClient) { + @Override + public int joinElection(ElectionContext context, boolean replacement) + throws KeeperException, InterruptedException { + joining.countDown(); + if (!allowJoin.await(WAIT_SECONDS, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to create election node"); + } + try { + return super.joinElection(context, replacement); + } finally { + joined.countDown(); + } + } + + @Override + protected void runIamLeaderProcess(ElectionContext context, boolean replacement) {} + }; + ShardLeaderElectionContext context = newContext(elector); + ExecutorService canceller = + ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrNamedThreadFactory("cancelElection")); + try { + context.rejoinLeaderElection(); + assertTrue("Rejoin did not start", joining.await(WAIT_SECONDS, TimeUnit.SECONDS)); + context.close(); + Future cancelled = + canceller.submit( + () -> { + cancelling.countDown(); + context.cancelElection(); + return null; + }); + assertTrue(cancelling.await(WAIT_SECONDS, TimeUnit.SECONDS)); + try { + cancelled.get(1, TimeUnit.SECONDS); + fail("Cancellation returned before the running rejoin could create its node"); + } catch (TimeoutException expected) { + // Cancellation must remain blocked until the running join releases the lifecycle lock. + } finally { + allowJoin.countDown(); + } + cancelled.get(WAIT_SECONDS, TimeUnit.SECONDS); + assertTrue(joined.await(WAIT_SECONDS, TimeUnit.SECONDS)); + assertTrue( + "Closed context left an election node", + zkClient.getChildren(ELECTION_PATH, null).isEmpty()); + } finally { + allowJoin.countDown(); + context.close(); + ExecutorUtil.shutdownAndAwaitTermination(canceller); + if (joining.getCount() == 0) { + assertTrue(joined.await(WAIT_SECONDS, TimeUnit.SECONDS)); + } + context.cancelElection(); + } + + AtomicInteger elected = new AtomicInteger(); + LeaderElector next = + new LeaderElector(zkClient) { + @Override + protected void runIamLeaderProcess(ElectionContext context, boolean replacement) { + elected.incrementAndGet(); + } + }; + ShardLeaderElectionContext nextContext = newContext(next); + try { + next.joinElection(nextContext, true); + assertEquals("A stale participant blocked the next candidate", 1, elected.get()); + } finally { + nextContext.close(); + nextContext.cancelElection(); + } + } + + @Test + public void testClosedContextDoesNotRejoin() throws Exception { + AtomicInteger joins = new AtomicInteger(); + LeaderElector elector = + new LeaderElector(zkClient) { + @Override + public int joinElection(ElectionContext context, boolean replacement) { + joins.incrementAndGet(); + return 0; + } + }; + ShardLeaderElectionContext context = newContext(elector); + context.close(); + context.rejoinLeaderElection(); + assertEquals(0, joins.get()); + assertTrue(zkClient.getChildren(ELECTION_PATH, null).isEmpty()); + } + + private ShardLeaderElectionContext newContext(LeaderElector elector) throws Exception { + ShardLeaderElectionContext context = + new ShardLeaderElectionContext( + elector, + "shard1", + "rejoin", + "core_node1", + new ZkNodeProps(ZkStateReader.CORE_NAME_PROP, "core"), + zkController, + cc); + elector.setup(context); + return context; + } +}