From 52e95e064099a5279c47de42254c4c6f5329c2ac Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 26 Oct 2026 12:05:19 -0400 Subject: [PATCH 1/2] SOLR-18569: Wait on init for ephemeral node-role cleanup (#5211) Solr has long had code in ZkController to check whether the ephemeral /live_node entry from a previous process is still around, and wait for ZK to deleted it before proceeding. But /live_node entries aren't the only ephemeral nodes that Solr expects to create on startup and that can linger after process exit. The /node_roles/ tree also contains ephemeral nodes. Currently, if any of these exist at startup-time, Solr will hit a NodeExists exception from ZK and fail to start up. This commit changes ZkController's init code so that it now waits for these nodes as well. In practice, this prevents Solr from potentially stalling on start up because of an unexpected or unhandled NodeExists exception. Co-authored-by: Nick Shanin --- ...wait-for-node-role-deletion-on-startup.yml | 9 ++ .../org/apache/solr/cloud/ZkController.java | 83 +++++++++---- .../apache/solr/cloud/ZkControllerTest.java | 114 ++++++++++++++++++ 3 files changed, 180 insertions(+), 26 deletions(-) create mode 100644 changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml diff --git a/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml new file mode 100644 index 000000000000..686f6fb8c810 --- /dev/null +++ b/changelog/unreleased/SOLR-18569-wait-for-node-role-deletion-on-startup.yml @@ -0,0 +1,9 @@ +title: > + Solr startup no longer hangs if ephemeral "node-role" znodes from a previous process are still present. +type: fixed +authors: + - name: Jason Gerlowski + - name: Nick Shanin +links: + - name: SOLR-18569 + url: https://issues.apache.org/jira/browse/SOLR-18569 diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 72467e0fc519..728f09fe0c52 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1092,43 +1092,74 @@ private void init() { } } - private void checkForExistingEphemeralNode() throws KeeperException, InterruptedException { + /** + * Waits for any ephemeral znodes left by a previous process with this node name to disappear. + * + *

Covers both the {@code /live_nodes} entry and this node's entries under any {@code + * /node_roles//}. Throws if any are still present after twice the ZK session timeout. + * + * @throws SolrException if ephemeral nodes were not all removed by the end of the waiting period + */ + // package-private for testing + void checkForExistingEphemeralNode() throws KeeperException, InterruptedException { if (zkRunOnly) { return; } String nodeName = getNodeName(); - String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName; - if (!zkClient.exists(nodePath, true)) { - return; + List candidatePaths = new ArrayList<>(); + candidatePaths.add(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName); + for (NodeRoles.Role role : NodeRoles.Role.values()) { + for (String mode : role.supportedModes()) { + candidatePaths.add(NodeRoles.getZNodeForRoleMode(role, mode) + "/" + nodeName); + } } - final CountDownLatch deletedLatch = new CountDownLatch(1); - Stat stat = - zkClient.exists( - nodePath, - event -> { - if (Watcher.Event.EventType.None.equals(event.getType())) { - return; - } - if (Watcher.Event.EventType.NodeDeleted.equals(event.getType())) { - deletedLatch.countDown(); - } - }, - true); + // Set a deletion watch on each znode that exists; the latch counts down as each is deleted + final List watchedPaths = new ArrayList<>(); + final List latches = new ArrayList<>(); + for (String path : candidatePaths) { + if (!zkClient.exists(path, true)) { + continue; + } + final CountDownLatch deletedLatch = new CountDownLatch(1); + Stat stat = + zkClient.exists( + path, + event -> { + if (Watcher.Event.EventType.None.equals(event.getType())) { + return; + } + if (Watcher.Event.EventType.NodeDeleted.equals(event.getType())) { + deletedLatch.countDown(); + } + }, + true); + if (stat == null) { + // znode suddenly disappeared but that's okay + continue; + } + watchedPaths.add(path); + latches.add(deletedLatch); + } - if (stat == null) { - // znode suddenly disappeared but that's okay + if (watchedPaths.isEmpty()) { return; } - boolean deleted = - deletedLatch.await(zkClient.getZooKeeper().getSessionTimeout() * 2L, TimeUnit.MILLISECONDS); - if (!deleted) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "A previous ephemeral live node still exists. " - + "Solr cannot continue. Please ensure that no other Solr process using the same port is running already."); + // A single overall deadline, shared by all of the znodes + final long deadlineNanos = + System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(zkClient.getZooKeeper().getSessionTimeout() * 2L); + for (int i = 0; i < latches.size(); i++) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (!latches.get(i).await(Math.max(remainingNanos, 0), TimeUnit.NANOSECONDS)) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "An ephemeral node from a previous process representing this Solr instance still exists: " + + watchedPaths.get(i) + + ". Solr cannot continue. Please ensure that no other Solr process using the same port is running already."); + } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index 2d45de8ebb38..e8cd41fb10ed 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -56,6 +56,7 @@ import org.apache.solr.core.CloudConfig; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.NodeRoles; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrXmlConfig; import org.apache.solr.handler.admin.CoreAdminHandler; @@ -255,6 +256,119 @@ public void testLiveNodeDataStored() throws Exception { } } + @Test + public void testStartupWaitsForLeftoverEphemeralNodesToDisappear() throws Exception { + final String staleRoleEntry = ephemeralNodeRolePath(NodeRoles.Role.OVERSEER); + runWithLeftoverEphemeralNodes( + "testStartupWaitsForLeftoverEphemeralNodes", + staleRoleEntry, + (zkController, otherClient, leftoverPaths) -> { + // Remove the leftovers shortly after the check starts waiting on them + AtomicReference deleterFailure = new AtomicReference<>(); + Thread deleter = + new Thread( + () -> { + try { + Thread.sleep(1000); + for (String path : leftoverPaths) { + otherClient.delete(path, -1, true); + } + } catch (Exception e) { + deleterFailure.set(e); + } + }); + deleter.start(); + try { + // Returns normally once everything is gone + zkController.checkForExistingEphemeralNode(); + for (String path : leftoverPaths) { + assertFalse( + path + " should be gone when the check returns", otherClient.exists(path, true)); + } + } finally { + deleter.join(TimeUnit.SECONDS.toMillis(30)); + } + assertFalse("Deleter thread did not finish", deleter.isAlive()); + assertNull("Deleter thread failed", deleterFailure.get()); + }); + } + + @Test + public void testStartupFailsIfLeftoverEphemeralNodeNeverDisappears() throws Exception { + final String staleRoleEntry = ephemeralNodeRolePath(NodeRoles.Role.OVERSEER); + runWithLeftoverEphemeralNodes( + "testStartupFailsIfLeftoverEphemeralNode", + staleRoleEntry, + (zkController, otherClient, leftoverPaths) -> { + // Only the role entry is left behind, so the message must name it specifically + SolrException e = + expectThrows(SolrException.class, zkController::checkForExistingEphemeralNode); + assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, e.code()); + assertTrue( + "Message should name the leftover path: " + e.getMessage(), + e.getMessage().contains(staleRoleEntry)); + }); + } + + /** The path of this test node's entry under a mode of 'role' that the node isn't using itself */ + private static String ephemeralNodeRolePath(NodeRoles.Role role) { + String unusedMode = NodeRoles.MODE_PREFERRED; // default roles use 'allowed' + assertTrue(role.supportedModes().contains(unusedMode)); + return NodeRoles.getZNodeForRoleMode(role, unusedMode) + "/" + TEST_NODE_NAME; + } + + private static final String TEST_NODE_NAME = "127.0.0.1:8983_solr"; + + @FunctionalInterface + private interface LeftoverNodesCheck { + void run(ZkController zkController, SolrZkClient otherClient, List leftoverPaths) + throws Exception; + } + + /** + * Starts a ZkController, removes its own ephemeral entries, then has a different ZK client create + * the given ephemeral paths (simulating a previous process with the same node name that hasn't + * fully gone away) before running 'check'. + */ + private void runWithLeftoverEphemeralNodes( + String dirName, String leftoverPath, LeftoverNodesCheck check) throws Exception { + ZkTestServer server = new ZkTestServer(createTempDir(dirName)); + try { + server.run(); + CoreContainer cc = getCoreContainer(); + // Short session timeout, as the check waits for twice the session timeout + CloudConfig cloudConfig = + new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "solr") + .setZkClientTimeout(3000) + .build(); + ZkController zkController = new ZkController(cc, server.getZkAddress(), 3000, cloudConfig); + try (SolrZkClient otherClient = + new SolrZkClient.Builder() + .withUrl(server.getZkAddress()) + .withTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + assertEquals(TEST_NODE_NAME, zkController.getNodeName()); + zkController.removeEphemeralLiveNode(); + // 9x's removeEphemeralLiveNode doesn't remove the node-role entries; clear them here + for (NodeRoles.Role role : NodeRoles.Role.values()) { + for (String mode : role.supportedModes()) { + String rolePath = NodeRoles.getZNodeForRoleMode(role, mode) + "/" + TEST_NODE_NAME; + if (otherClient.exists(rolePath, true)) { + otherClient.delete(rolePath, -1, true); + } + } + } + otherClient.create(leftoverPath, null, CreateMode.EPHEMERAL, true); + check.run(zkController, otherClient, List.of(leftoverPath)); + } finally { + zkController.close(); + cc.shutdown(); + } + } finally { + server.shutdown(); + } + } + @LogLevel(value = "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.overseer=DEBUG") @Test public void testPublishAndWaitForDownStates() throws Exception { From d1f4b9d0a9426503d55f880bcf00d1143c543aaa Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 26 Oct 2026 13:45:49 -0400 Subject: [PATCH 2/2] SOLR-18569: Cleanup /node_role/ ephem entries on shutdown Logic already existed on 'main' and 'branch_10x', but wasn't present in our 9x codeline. --- .../java/org/apache/solr/cloud/ZkController.java | 13 +++++++++++++ .../org/apache/solr/cloud/ZkControllerTest.java | 9 --------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 728f09fe0c52..05157a55f4c6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1320,6 +1320,19 @@ public void removeEphemeralLiveNode() throws KeeperException, InterruptedExcepti } catch (NoNodeException e) { } + cc.nodeRoles + .getRoles() + .forEach( + (role, mode) -> { + try { + zkClient.delete( + NodeRoles.getZNodeForRoleMode(role, mode) + "/" + nodeName, -1, true); + } catch (KeeperException e) { + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); } public String getNodeName() { diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index e8cd41fb10ed..6599a12a0bd4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -349,15 +349,6 @@ private void runWithLeftoverEphemeralNodes( .build()) { assertEquals(TEST_NODE_NAME, zkController.getNodeName()); zkController.removeEphemeralLiveNode(); - // 9x's removeEphemeralLiveNode doesn't remove the node-role entries; clear them here - for (NodeRoles.Role role : NodeRoles.Role.values()) { - for (String mode : role.supportedModes()) { - String rolePath = NodeRoles.getZNodeForRoleMode(role, mode) + "/" + TEST_NODE_NAME; - if (otherClient.exists(rolePath, true)) { - otherClient.delete(rolePath, -1, true); - } - } - } otherClient.create(leftoverPath, null, CreateMode.EPHEMERAL, true); check.run(zkController, otherClient, List.of(leftoverPath)); } finally {