From 6f72e0cac151d139626299a680f90a7c9c123ceb Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Fri, 23 Oct 2026 12:31:05 -0400 Subject: [PATCH 1/4] SOLR-18569: Wait on init for ephemeral node-role cleanup Solr has long had code in ZkController to check whether the ephemeral `/live_node` entry from a previous process is still around, and wait for ZK to deleted it before proceeding. But `/live_node` entries aren't the only ephemeral nodes that Solr expects to create on startup and that can linger after process exit. The `/node_roles/` tree also contains ephemeral nodes. This commit changes ZkController's init code so that it now waits for these nodes as well. In practice, this prevents Solr from potentially stalling on start up because of an unexpected or unhandled `NodeExists` exception. --- ...wait-for-node-role-deletion-on-startup.yml | 8 ++ .../org/apache/solr/cloud/ZkController.java | 78 ++++++++++----- .../apache/solr/cloud/ZkControllerTest.java | 99 +++++++++++++++++++ 3 files changed, 160 insertions(+), 25 deletions(-) create mode 100644 changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml diff --git a/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml new file mode 100644 index 000000000000..b0f6f9c4b416 --- /dev/null +++ b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml @@ -0,0 +1,8 @@ +title: > + Solr startup no longer hangs if ephemeral "node-role" znodes from a previous process are still present. +type: fixed +authors: + - name: Jason Gerlowski +links: + - name: SOLR-18569 + url: https://issues.apache.org/jira/browse/SOLR-18569 diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 5db9f8d7256a..04d2b4c04725 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1171,39 +1171,67 @@ private void init() { } } - private void checkForExistingEphemeralNode() throws KeeperException, InterruptedException { + /** + * Waits for any ephemeral znodes left by a previous process with this node name to disappear. + * + *

Covers both the {@code /live_nodes} entry and this node's entries under any {@code + * /node_roles//}. Throws if any are still present after twice the ZK session timeout. + */ + // package-private for testing + void checkForExistingEphemeralNode() throws KeeperException, InterruptedException { String nodeName = getNodeName(); - String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName; - if (!zkClient.exists(nodePath)) { - return; + List candidatePaths = new ArrayList<>(); + candidatePaths.add(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName); + for (NodeRoles.Role role : NodeRoles.Role.values()) { + for (String mode : role.supportedModes()) { + candidatePaths.add(NodeRoles.getZNodeForRoleMode(role, mode) + "/" + nodeName); + } } - final CountDownLatch deletedLatch = new CountDownLatch(1); - Stat stat = - zkClient.exists( - nodePath, - event -> { - if (Watcher.Event.EventType.None.equals(event.getType())) { - return; - } - if (Watcher.Event.EventType.NodeDeleted.equals(event.getType())) { - deletedLatch.countDown(); - } - }); + // Set a deletion watch on each znode that exists; the latch counts down as each is deleted + final List watchedPaths = new ArrayList<>(); + final List latches = new ArrayList<>(); + for (String path : candidatePaths) { + if (!zkClient.exists(path)) { + continue; + } + final CountDownLatch deletedLatch = new CountDownLatch(1); + Stat stat = + zkClient.exists( + path, + event -> { + if (Watcher.Event.EventType.None.equals(event.getType())) { + return; + } + if (Watcher.Event.EventType.NodeDeleted.equals(event.getType())) { + deletedLatch.countDown(); + } + }); + if (stat == null) { + // znode suddenly disappeared but that's okay + continue; + } + watchedPaths.add(path); + latches.add(deletedLatch); + } - if (stat == null) { - // znode suddenly disappeared but that's okay + if (watchedPaths.isEmpty()) { return; } - boolean deleted = - deletedLatch.await(zkClient.getZkSessionTimeout() * 2L, TimeUnit.MILLISECONDS); - if (!deleted) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "A previous ephemeral live node still exists. " - + "Solr cannot continue. Please ensure that no other Solr process using the same port is running already."); + // A single overall deadline, shared by all of the znodes + final long deadlineNanos = + System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(zkClient.getZkSessionTimeout() * 2L); + for (int i = 0; i < latches.size(); i++) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (!latches.get(i).await(Math.max(remainingNanos, 0), TimeUnit.NANOSECONDS)) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "An ephemeral node from a previous process representing this Solr instance still exists: " + + watchedPaths.get(i) + + ". Solr cannot continue. Please ensure that no other Solr process using the same port is running already."); + } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index 0d993da70ac8..eccdcc8b86df 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -59,6 +59,7 @@ import org.apache.solr.core.CloudConfig; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.NodeRoles; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrXmlConfig; import org.apache.solr.handler.admin.CoreAdminHandler; @@ -222,6 +223,104 @@ public void testLiveNodeDataStored() throws Exception { } } + @Test + public void testStartupWaitsForLeftoverEphemeralNodesToDisappear() throws Exception { + final String staleRoleEntry = ephemeralNodeRolePath(NodeRoles.Role.OVERSEER); + runWithLeftoverEphemeralNodes( + "testStartupWaitsForLeftoverEphemeralNodes", + staleRoleEntry, + (zkController, otherClient, leftoverPaths) -> { + // Remove the leftovers shortly after the check starts waiting on them + Thread deleter = + new Thread( + () -> { + try { + Thread.sleep(1000); + for (String path : leftoverPaths) { + otherClient.delete(path, -1); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + deleter.start(); + try { + // Returns normally once everything is gone + zkController.checkForExistingEphemeralNode(); + } finally { + deleter.join(); + } + for (String path : leftoverPaths) { + assertFalse(path + " should be gone", otherClient.exists(path)); + } + }); + } + + @Test + public void testStartupFailsIfLeftoverEphemeralNodeNeverDisappears() throws Exception { + final String staleRoleEntry = ephemeralNodeRolePath(NodeRoles.Role.OVERSEER); + runWithLeftoverEphemeralNodes( + "testStartupFailsIfLeftoverEphemeralNode", + staleRoleEntry, + (zkController, otherClient, leftoverPaths) -> { + // Only the role entry is left behind, so the message must name it specifically + SolrException e = + expectThrows(SolrException.class, zkController::checkForExistingEphemeralNode); + assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, e.code()); + assertTrue( + "Message should name the leftover path: " + e.getMessage(), + e.getMessage().contains(staleRoleEntry)); + }); + } + + /** The path of this test node's entry under a mode of 'role' that the node isn't using itself */ + private static String ephemeralNodeRolePath(NodeRoles.Role role) { + String unusedMode = NodeRoles.MODE_PREFERRED; // default roles use 'allowed' + assertTrue(role.supportedModes().contains(unusedMode)); + return NodeRoles.getZNodeForRoleMode(role, unusedMode) + "/" + TEST_NODE_NAME; + } + + private static final String TEST_NODE_NAME = "127.0.0.1:8983_solr"; + + @FunctionalInterface + private interface LeftoverNodesCheck { + void run(ZkController zkController, SolrZkClient otherClient, List leftoverPaths) + throws Exception; + } + + /** + * Starts a ZkController, removes its own ephemeral entries, then has a different ZK client create + * the given ephemeral paths (simulating a previous process with the same node name that hasn't + * fully gone away) before running 'check'. + */ + private void runWithLeftoverEphemeralNodes( + String dirName, String leftoverPath, LeftoverNodesCheck check) throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir(dirName)); + try { + server.run(); + CoreContainer cc = getCoreContainer(); + // Short session timeout, as the check waits for twice the session timeout + CloudConfig cloudConfig = + new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).setZkClientTimeout(3000).build(); + ZkController zkController = new ZkController(cc, server.getZkAddress(), 3000, cloudConfig); + try (SolrZkClient otherClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + assertEquals(TEST_NODE_NAME, zkController.getNodeName()); + zkController.removeEphemeralLiveNode(); + otherClient.create(leftoverPath, null, CreateMode.EPHEMERAL); + check.run(zkController, otherClient, List.of(leftoverPath)); + } finally { + zkController.close(); + cc.shutdown(); + } + } finally { + server.shutdown(); + } + } + @LogLevel(value = "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.overseer=DEBUG") @Test public void testPublishAndWaitForDownStates() throws Exception { From 34526140c9fb7d9c71379eb1c2fa4919b6b179c7 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 26 Oct 2026 10:06:23 -0400 Subject: [PATCH 2/4] Address review comments, rd 1 --- solr/core/src/java/org/apache/solr/cloud/ZkController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 04d2b4c04725..0abc95cbd9b8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1176,6 +1176,8 @@ private void init() { * *

Covers both the {@code /live_nodes} entry and this node's entries under any {@code * /node_roles//}. Throws if any are still present after twice the ZK session timeout. + * + * @throws SolrException if ephemeral nodes were not all removed by the end of the waiting period */ // package-private for testing void checkForExistingEphemeralNode() throws KeeperException, InterruptedException { From ccce03a4907b67e02b552bffe603b7040bf511c6 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 26 Oct 2026 10:21:14 -0400 Subject: [PATCH 3/4] SOLR-18569: Assert leftover nodes are gone before joining the deleter (#6) Co-authored-by: Nick Shanin --- .../org/apache/solr/cloud/ZkControllerTest.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index eccdcc8b86df..ea339d7126d9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -231,6 +231,7 @@ public void testStartupWaitsForLeftoverEphemeralNodesToDisappear() throws Except staleRoleEntry, (zkController, otherClient, leftoverPaths) -> { // Remove the leftovers shortly after the check starts waiting on them + AtomicReference deleterFailure = new AtomicReference<>(); Thread deleter = new Thread( () -> { @@ -240,19 +241,22 @@ public void testStartupWaitsForLeftoverEphemeralNodesToDisappear() throws Except otherClient.delete(path, -1); } } catch (Exception e) { - throw new RuntimeException(e); + deleterFailure.set(e); } }); deleter.start(); try { // Returns normally once everything is gone zkController.checkForExistingEphemeralNode(); + for (String path : leftoverPaths) { + assertFalse( + path + " should be gone when the check returns", otherClient.exists(path)); + } } finally { - deleter.join(); - } - for (String path : leftoverPaths) { - assertFalse(path + " should be gone", otherClient.exists(path)); + deleter.join(TimeUnit.SECONDS.toMillis(30)); } + assertFalse("Deleter thread did not finish", deleter.isAlive()); + assertNull("Deleter thread failed", deleterFailure.get()); }); } From 15efe08d06ed788da79b7be5e55dc79e97ca4fa7 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 26 Oct 2026 10:26:16 -0400 Subject: [PATCH 4/4] Update changelog attribution --- .../SOLR-18569-wait-for-node-role-deletion-on-startup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml index b0f6f9c4b416..686f6fb8c810 100644 --- a/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml +++ b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml @@ -3,6 +3,7 @@ title: > type: fixed authors: - name: Jason Gerlowski + - name: Nick Shanin links: - name: SOLR-18569 url: https://issues.apache.org/jira/browse/SOLR-18569