diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index fe61b3c49234a..154fd57192136 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -287,6 +287,48 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --groups cache-group1,cach control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1 ---- +== Deleting Snapshot + +You can delete a snapshot using the `control.sh|bat` script. + +The deletion is performed on all *online* server nodes of the cluster. +[NOTE] +==== +The snapshot integrity, topology and correctness aren't checked. Snapshot data on offline server nodes aren't deleted. +==== + +[tabs] +-- +tab:Unix[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.sh --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder. +control.sh --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots +---- + +tab:Windows[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.bat --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder. +control.bat --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots +---- +-- + +=== Delete operation limitations + +The delete operation is subject to the following limitations: + +* The deletion is rejected if any concurrent snapshot operation (create, restore, check, or a delete with the same name) is + active for the snapshot. +* The operation is irreversible. It cannot be undone, and the deleted snapshot cannot be restored. +* The command prompts for a confirmation before the deletion because the operation is irreversible and cannot be undone. + == Getting Snapshot Operation Status The status of the current snapshot operation in the cluster can be obtained using the `control.sh|bat` script or JMX interface: diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 0fc350ab7c8fc..99882f4ccc6ff 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -81,6 +81,7 @@ import org.apache.ignite.internal.management.performancestatistics.PerformanceStatisticsCommand; import org.apache.ignite.internal.management.property.PropertyCommand; import org.apache.ignite.internal.management.snapshot.SnapshotCommand; +import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand; import org.apache.ignite.internal.management.snapshot.SnapshotRestoreCommand; import org.apache.ignite.internal.management.tx.TxCommand; import org.apache.ignite.internal.management.tx.TxCommandArg; @@ -529,6 +530,8 @@ else if (cmd.getClass() == EncryptionChangeCacheKeyCommand.class) cmdText = F.concat(cmdText, "cacheGroup1"); else if (cmd.getClass() == SnapshotRestoreCommand.class) cmdText = F.concat(cmdText, "snp1"); + else if (cmd.getClass() == SnapshotDeleteCommand.class) + cmdText = F.concat(cmdText, "snp1"); else if (cmd.getClass() == MetaUpdateCommand.class) return; else if (cmd.getClass() == MetaRemoveCommand.class) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java index 6528d919f6105..91a1d67d0dfc0 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java @@ -27,6 +27,7 @@ import org.apache.ignite.util.GridCommandHandlerCheckpointTest; import org.apache.ignite.util.GridCommandHandlerClusterByClassTest; import org.apache.ignite.util.GridCommandHandlerClusterByClassWithSSLTest; +import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest; import org.apache.ignite.util.GridCommandHandlerIncompatibleSslConfigTest; import org.apache.ignite.util.GridCommandHandlerIndexingCheckSizeTest; import org.apache.ignite.util.GridCommandHandlerIndexingClusterByClassTest; @@ -74,6 +75,7 @@ GridCommandHandlerCheckIndexesInlineSizeTest.class, GridCommandHandlerMetadataTest.class, GridCommandHandlerCheckIncrementalSnapshotTest.class, + GridCommandHandlerDeleteSnapshotTest.class, GridCommandHandlerLegacyClientTest.class, KillCommandsControlShTest.class, diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java new file mode 100644 index 0000000000000..967f7b141bf33 --- /dev/null +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.util; + +import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import static java.nio.file.Files.newDirectoryStream; +import static org.apache.ignite.cluster.ClusterState.ACTIVE; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assume.assumeTrue; + +/** Test for the command '--snapshot delete'. */ +@RunWith(Parameterized.class) +public class GridCommandHandlerDeleteSnapshotTest extends GridCommandHandlerAbstractTest { + /** Value: -1 - do not use, 1 - server node, 0 - client node. */ + @Parameter(1) + public int extraNodeIsServer = -1; + + /** */ + @Parameter(2) + public boolean addIncrements; + + /** */ + @Parameter(3) + public boolean changeBaseline; + + /** */ + @Parameter(4) + public boolean customPath; + + /** */ + @Parameter(5) + public boolean separatedWorkDir; + + /** */ + @Parameters(name = "client={0},useExtraNode={1},inc={2},chBaseln={3},cstSnpPath={4},ownWorkDir={5}") + public static Collection parameters() { + return GridTestUtils.cartesianProduct( + commandHandlers(), + F.asList(-1, 1, 0), // Use extra node (do not use at all, server node, client node); + F.asList(false, true), // Add increments to the test snapshot; + F.asList(false, true), // Change baseline; + F.asList(false, true), // Use custom snapshot path; + F.asList(true, false) // Separated (own) work directory. + ); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + /** Handy if test running is interrupted and {@link #afterTest()} isn't invoked. */ + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void cleanPersistenceDir() throws Exception { + super.cleanPersistenceDir(); + + // Also cleans separated snapshot working directories and custom snapshot pacthes. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + if (separatedWorkDir) + cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); + + return cfg; + } + + /** */ + @Test + public void testSnapshotDelete() throws Exception { + // A custom snapshot path actually puts snapshots in a shared directory. This skews the results when dedicated + // work directories are set. + assumeTrue(!customPath || !separatedWorkDir); + + int entriesCnt = 4000; + int initNodes = 3; + + walCompactionEnabled(addIncrements); + + IgniteEx ig = (IgniteEx)startGridsMultiThreaded(initNodes); + + if (changeBaseline) { + ig.cluster().baselineAutoAdjustEnabled(false); + + ig.cluster().setBaselineTopology(ig.cluster().topologyVersion()); + } + + ig.cluster().state(ACTIVE); + + createCacheAndPreload(ig, entriesCnt); + + File cstSnpsRoot = customPath ? new File(U.defaultWorkDirectory(), "ex_snapshots") : null; + File snpDir = new File(customPath ? cstSnpsRoot : ig.context().pdsFolderResolver().fileTree().snapshotsRoot(), "testSnapshot"); + + snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, false, false) + .get(getTestTimeout()); + + if (addIncrements) { + for (int i = 0; i < 3; ++i) { + int dataIdx = entriesCnt + entriesCnt / 4 * i; + + try (IgniteDataStreamer streamer = ig.dataStreamer(DEFAULT_CACHE_NAME)) { + for (int d = dataIdx; d < dataIdx + entriesCnt / 4; ++d) + streamer.addData(i, i); + } + + snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, true, false) + .get(getTestTimeout()); + } + } + + // Optionally restarts with the same servers number, but changed baseline. The snapshot is kept on the same + // previous nodes independently of the baseline. + if (changeBaseline) { + ig.destroyCache(DEFAULT_CACHE_NAME); + awaitPartitionMapExchange(); + + stopAllGrids(); + + ig = (IgniteEx)startGridsMultiThreaded(initNodes - 1); + + ig.cluster().setBaselineTopology(ig.cluster().topologyVersion()); + + startGrid(initNodes - 1); + + assertEquals(initNodes - 1, ig.cluster().currentBaselineTopology().size()); + assertEquals(initNodes, ig.cluster().nodes().size()); + } + + // Optionally adds extra server or client node. + if (extraNodeIsServer == 1) + startGrid(initNodes); + else if (extraNodeIsServer == 0) + startGrid(CLIENT_NODE_NAME_PREFIX); + + injectTestSystemOut(); + + // Tests missing snapshot deletion. + if (customPath) { + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath(), "wrongSnapshot")); + + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath() + "_wrongPath", "testSnapshot")); + } + else + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "wrongSnapshot")); + + String out = testOut.toString(); + + assertFalse(out.contains("Snapshot removed on the following nodes")); + assertFalse(out.contains("the following nodes didn't find any snapshot data, nothing to delete")); + assertTrue(out.contains("Snapshot not found on current server nodes")); + + testOut.reset(); + assertTrue(testOut.toString().isEmpty()); + + if (customPath) { + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath(), "testSnapshot")); + } + else + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "testSnapshot")); + + out = testOut.toString(); + + if (separatedWorkDir) { + // When the nodes use own separated work directory, we expect a strict result. + assertTrue(out.contains("Snapshot removed on the following nodes [cnt=%d]:".formatted(initNodes))); + + if (extraNodeIsServer == 1) + assertTrue(out.contains("the following nodes didn't find any snapshot data, nothing to delete [cnt=1]:")); + else if (extraNodeIsServer == 0) + assertFalse(out.contains("the following nodes didn't find any snapshot data, nothing to delete")); + } + else { + // When nodes use a shared work directory, there is a race for the delete operation. One node can get faster + // than others and remove snapshot completely quickly. The others might not find snapshot files. We can be + // only sure that at least one node removes snapshot. + assertTrue(out.contains("Snapshot removed on the following nodes [cnt=")); + } + + assertFalse(out.contains("Snapshot not found on current server nodes")); + + assertTrue(waitForCondition(() -> !snpDir.exists(), getTestTimeout())); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java index f7537f0b20f79..95d28ef5ec2b9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java @@ -162,6 +162,8 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckPartitionHashesResponse; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcessRequest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckResponse; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesFailureMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesRequestMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerResult; @@ -437,6 +439,8 @@ public CoreMessagesProvider() { register(SnapshotFilesFailureMessage.class); register(IncrementalSnapshotVerifyResult.class); register(IncrementalSnapshotAwareMessage.class); + register(SnapshotDeleteRequest.class); + register(SnapshotDeleteResponse.class); // [6300 - 6400]: Services messages. Most of them originally come from Discovery. msgIdx = 6300; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java index deb5416b8e28b..8979958bd8fb6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java @@ -28,7 +28,8 @@ public SnapshotCommand() { new SnapshotCancelCommand(), new SnapshotCheckCommand(), new SnapshotRestoreCommand(), - new SnapshotStatusCommand() + new SnapshotStatusCommand(), + new SnapshotDeleteCommand() ); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java new file mode 100644 index 0000000000000..e7262ce26942c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import java.util.Collection; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcess; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Snapshot deletion command. + * + * @see SupportedFeatureRegistry#SNAPSHOT_DELETE_FEATURE + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteCommand extends AbstractSnapshotCommand { + /** {@inheritDoc} */ + @Override public String description() { + return "Deletes snapshot and all its increments from all the online server nodes"; + } + + /** {@inheritDoc} */ + @Override public Class argClass() { + return SnapshotDeleteCommandArg.class; + } + + /** {@inheritDoc} */ + @Override public Class taskClass() { + return SnapshotDeleteTask.class; + } + + /** {@inheritDoc} */ + @Override public void printResult(SnapshotDeleteCommandArg arg, SnapshotDeleteProcessResult res, Consumer printer) { + boolean found = false; + + if (!F.isEmpty(res.uncompletedNodes())) { + found = true; + + printer.accept("WARNING, the following nodes found snapshot data but might not remove it completely " + + nodeIdsStrLst(res.uncompletedNodes())); + + printer.accept(""); + } + + if (!F.isEmpty(res.completedNodes())) { + found = true; + + printer.accept("Snapshot removed on the following nodes " + nodeIdsStrLst(res.completedNodes())); + printer.accept(""); + } + + if (found) { + if (!F.isEmpty(res.emptyNodes())) { + printer.accept("NOTE, the following nodes didn't find any snapshot data, nothing to delete " + + nodeIdsStrLst(res.emptyNodes())); + } + } + else { + if (!F.isEmpty(res.emptyNodes())) + printer.accept("Snapshot not found on current server nodes."); + else + printer.accept("Unknown result."); + } + } + + /** */ + private static String nodeIdsStrLst(Collection uuids) { + return "[cnt=" + uuids.size() + "]: " + uuids.stream().map(UUID::toString).collect(Collectors.joining(", ")); + } + + /** {@inheritDoc} */ + @Override public String confirmationPrompt(SnapshotDeleteCommandArg arg) { + return "This will delete snapshot '" + arg.snapshotName() + + "' and all its increments from all online server nodes." + + U.nl() + U.nl() + + "WARNING: the snapshot integrity, topology and correctness aren't checked." + + " Snapshot data on offline server nodes aren't deleted." + + U.nl() + U.nl() + + "The operation is irreversible."; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java new file mode 100644 index 0000000000000..2b7063de333bb --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.management.api.Argument; +import org.apache.ignite.internal.management.api.Positional; + +/** */ +public class SnapshotDeleteCommandArg extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0; + + /** */ + @Order(0) + @Positional + @Argument(description = "Snapshot name") + String snapshotName; + + /** */ + @Order(1) + @Argument(example = "path", optional = true, + description = "Path to the directory where the snapshot is located. " + + "If not specified, the default configured snapshot directory will be used") + String src; + + /** */ + public String snapshotName() { + return snapshotName; + } + + /** */ + public void snapshotName(String snapshotName) { + this.snapshotName = snapshotName; + } + + /** */ + public String src() { + return src; + } + + /** */ + public void src(String src) { + this.src = src; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java new file mode 100644 index 0000000000000..86bac6972bd2f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult; +import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; + +/** + * @see IgniteSnapshotManager#deleteSnapshot(String, String) + */ +@GridInternal +public class SnapshotDeleteTask extends VisorOneNodeTask { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorJob job(SnapshotDeleteCommandArg arg) { + return new SnapshotDeleteJob(arg, debug); + } + + /** */ + private static class SnapshotDeleteJob extends SnapshotJob { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Snapshot delete task argument. + * @param debug Flag indicating whether debug information should be printed into node log. + */ + protected SnapshotDeleteJob(SnapshotDeleteCommandArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected SnapshotDeleteProcessResult run(SnapshotDeleteCommandArg arg) { + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + return snpMgr.deleteSnapshot(arg.snapshotName(), arg.src()).get(); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index a9f1f42ca872b..1857aabd1f965 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -34,7 +34,7 @@ /** * Information about partitions of a single node. Sent in response to {@link GridDhtPartitionsSingleRequest} and during * processing partitions exchange future.
- * Has to be completelly restored after receiving from another node. + * Has to be completely restored after receiving from another node. * * @see #afterReceive() */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java index 159eeb77db7c9..96c03fd4573a1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java @@ -333,12 +333,12 @@ public NodeFileTree(File root, String folderName) { this.folderName = folderName; - binaryMeta = new File(binaryMetaRoot, folderName); - nodeStorage = rootRelative(DB_DIR); - checkpoint = new File(nodeStorage, CHECKPOINT_DIR); - wal = rootRelative(DFLT_WAL_PATH); - walArchive = rootRelative(DFLT_WAL_ARCHIVE_PATH); - walCdc = rootRelative(DFLT_WAL_CDC_PATH); + all.add(binaryMeta = new File(binaryMetaRoot, folderName)); + all.add(nodeStorage = rootRelative(DB_DIR)); + all.add(checkpoint = new File(nodeStorage, CHECKPOINT_DIR)); + all.add(wal = rootRelative(DFLT_WAL_PATH)); + all.add(walArchive = rootRelative(DFLT_WAL_ARCHIVE_PATH)); + all.add(walCdc = rootRelative(DFLT_WAL_CDC_PATH)); extraStorages = Collections.emptyMap(); } @@ -381,29 +381,31 @@ protected NodeFileTree(IgniteConfiguration cfg, File root, String folderName, bo this.folderName = folderName; - binaryMeta = new File(binaryMetaRoot, folderName); + all.add(binaryMeta = new File(binaryMetaRoot, folderName)); DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration(); if (CU.isPersistenceEnabled(cfg) || CU.isCdcEnabled(cfg)) { // Snapshots MUST use root relative node storage path. - nodeStorage = (dsCfg.getStoragePath() == null || isSnapshot) + all.add(nodeStorage = (dsCfg.getStoragePath() == null || isSnapshot) ? rootRelative(DB_DIR) - : resolveDirectory(dsCfg.getStoragePath()); - checkpoint = new File(nodeStorage, CHECKPOINT_DIR); - wal = resolveDirectory(dsCfg.getWalPath()); - walArchive = resolveDirectory(dsCfg.getWalArchivePath()); - walCdc = resolveDirectory(dsCfg.getCdcWalPath()); + : resolveDirectory(dsCfg.getStoragePath())); + all.add(checkpoint = new File(nodeStorage, CHECKPOINT_DIR)); + all.add(wal = resolveDirectory(dsCfg.getWalPath())); + all.add(walArchive = resolveDirectory(dsCfg.getWalArchivePath())); + all.add(walCdc = resolveDirectory(dsCfg.getCdcWalPath())); } else { - nodeStorage = rootRelative(DB_DIR); - checkpoint = new File(nodeStorage, CHECKPOINT_DIR); - wal = rootRelative(DFLT_WAL_PATH); - walArchive = rootRelative(DFLT_WAL_ARCHIVE_PATH); - walCdc = rootRelative(DFLT_WAL_CDC_PATH); + all.add(nodeStorage = rootRelative(DB_DIR)); + all.add(checkpoint = new File(nodeStorage, CHECKPOINT_DIR)); + all.add(wal = rootRelative(DFLT_WAL_PATH)); + all.add(walArchive = rootRelative(DFLT_WAL_ARCHIVE_PATH)); + all.add(walCdc = rootRelative(DFLT_WAL_CDC_PATH)); } extraStorages = extraStorages(dsCfg); + + all.addAll(extraStorages.values()); } /** @return Node storage directory. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java index 33df1ed636040..75a9562c4e4a3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java @@ -19,6 +19,10 @@ import java.io.File; import java.nio.file.Paths; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.configuration.IgniteConfiguration; @@ -63,6 +67,9 @@ public class SharedFileTree { /** Path to the snapshot root directory. */ private final File snpsRoot; + /** All knows working pathes. */ + protected final Set all = new HashSet<>(); + /** * @param root Root directory. * @param snpsRoot Snapshot path. @@ -70,13 +77,13 @@ public class SharedFileTree { protected SharedFileTree(File root, String snpsRoot) { A.notNull(root, "Root directory"); - this.root = root; - this.snpsRoot = resolveDirectory(snpsRoot); + all.add(this.root = root); + all.add(this.snpsRoot = resolveDirectory(snpsRoot)); String rootStr = root.getAbsolutePath(); - marshaller = Paths.get(rootStr, DB_DIR, MARSHALLER_DIR).toFile(); - binaryMetaRoot = Paths.get(rootStr, DB_DIR, BINARY_METADATA_DIR).toFile(); + all.add(marshaller = Paths.get(rootStr, DB_DIR, MARSHALLER_DIR).toFile()); + all.add(binaryMetaRoot = Paths.get(rootStr, DB_DIR, BINARY_METADATA_DIR).toFile()); } /** @@ -132,6 +139,11 @@ public File snapshotsRoot() { return snpsRoot; } + /** @return All known main pathes used. */ + public Collection all() { + return Collections.unmodifiableSet(all); + } + /** * Creates {@link #binaryMetaRoot()} directory. * @return Created directory. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index c6e25c2baa3ca..98a9d8e6b2afb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -30,6 +30,7 @@ import java.nio.channels.FileChannel; import java.nio.file.FileVisitResult; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; @@ -60,6 +61,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -344,6 +346,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Snapshot validation distributed process. */ private final SnapshotCheckProcess checkSnpProc; + /** Distributed process to delete cluster snapshot. */ + private final SnapshotDeleteProcess deleteSnpProc; + /** Check previously performed snapshot operation and delete uncompleted files if we need. */ private final DistributedProcess endSnpProc; @@ -446,6 +451,8 @@ public IgniteSnapshotManager(GridKernalContext ctx) { checkSnpProc = new SnapshotCheckProcess(ctx); + deleteSnpProc = new SnapshotDeleteProcess(ctx); + // Manage remote snapshots. snpRmtMgr = new SequentialRemoteSnapshotManager(); } @@ -664,6 +671,7 @@ public IgniteSnapshotManager(GridKernalContext ctx) { restoreCacheGrpProc.interrupt(stopErr); checkSnpProc.interrupt(stopErr); + deleteSnpProc.interrupt(stopErr); // Try stop all snapshot processing if not yet. for (AbstractSnapshotFutureTask sctx : locSnpTasks.values()) @@ -701,46 +709,107 @@ public IgniteSnapshotManager(GridKernalContext ctx) { /** * @param snpDir Snapshot dir. */ - public void deleteSnapshot(File snpDir) { + public void deleteLocalSnapshot(File snpDir) { if (!snpDir.exists()) return; if (!snpDir.isDirectory()) return; - deleteSnapshot(new SnapshotFileTree( + var sft = new SnapshotFileTree( cctx.kernalContext(), snpDir.getName(), snpDir.getParent(), ft.folderName(), - pdsSettings.consistentId().toString())); + pdsSettings.consistentId().toString() + ); + + deleteLocalSnapshot(sft, ft.folderName(), null); } - /** */ - public void deleteSnapshot(SnapshotFileTree sft) { + /** + * Deletes local shapshot data. + * + * @param sft Snapshot file tree + * @param nodeFolderName Exact node's data subdirectory name usually taken from the consistent id. + * @param existsFlag Flag to set {@code true} if any snapshot file or directory was found (existed). If {@code null}, ignored. + * @return {@code True}, if data is found and completely deleted; + * {@code False}, if nothing found or if data is found but might not be deleted completely. + */ + public boolean deleteLocalSnapshot(SnapshotFileTree sft, String nodeFolderName, @Nullable AtomicBoolean existsFlag) { + if (existsFlag != null) + existsFlag.set(sft.root().exists()); + + if (!sft.root().exists()) + return false; + + AtomicBoolean res = new AtomicBoolean(true); + + for (var dir : F.asList(sft.binaryMeta(), sft.binaryMetaRoot(), sft.marshaller(), sft.db())) { + if (!dir.exists()) + continue; + + var nodeDir = new File(dir, nodeFolderName); + + if (nodeDir.exists()) + deleteSnapshotDataCompletely(nodeDir, res); + + // Recheck for the case of concurrent deletion. + try { + Files.delete(dir.toPath()); + } + catch (NoSuchFileException ne) { + // No-op: someone else deleted. + } + catch (Exception e) { + res.set(false); + } + } + + deleteSnapshotDataCompletely(sft.meta(), res); + try { - U.delete(sft.binaryMeta()); - sft.allStorages().forEach(U::delete); - U.delete(sft.meta()); + Files.delete(sft.root().toPath()); + } catch (Exception e) { + res.set(false); + } + + return res.get(); + } - deleteDirectory(sft.binaryMetaRoot()); - deleteDirectory(sft.marshaller()); + /** + * Deletes file/directory and sets existence and deletion failure flags. + * + * @param f File/directory to delete. If {@code null}, does nothing. + * @param failRes Is set to {@code False} if at least one existed file or directory was denied to delete. + */ + private void deleteSnapshotDataCompletely(@Nullable File f, AtomicBoolean failRes) { + if (f == null || !f.exists()) + return; - // Delete parent dir which is {snapshot_root}/db if empty. - sft.marshaller().getParentFile().delete(); - // Delete root dir which is {snapshot_root} if empty. - sft.root().delete(); + try { + if (f.isDirectory()) { + if (!deleteDirectoryWithContent(f)) + failRes.set(false); + } + else + Files.delete(f.toPath()); } - catch (IOException e) { - throw new IgniteException(e); + catch (Exception e) { + failRes.set(false); } } /** Concurrently traverse the directory and delete all files. */ - private void deleteDirectory(File dir) throws IOException { - Files.walkFileTree(dir.toPath(), new SimpleFileVisitor() { + private boolean deleteDirectoryWithContent(File dir) throws IOException { + AtomicBoolean res = new AtomicBoolean(true); + + Files.walkFileTree(dir.toPath(), new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - U.delete(file); + var f = file.toFile(); + + if (f.exists() && !U.delete(file) && f.exists()) + res.set(false); return FileVisitResult.CONTINUE; } @@ -751,7 +820,10 @@ private void deleteDirectory(File dir) throws IOException { } @Override public FileVisitResult postVisitDirectory(Path dir, IOException e) { - dir.toFile().delete(); + var f = dir.toFile(); + + if (f.exists() && !f.delete() && f.exists()) + res.set(false); if (log.isInfoEnabled() && e != null) log.info("Snapshot directory cleaned with an exception [dir=" + dir + ", e=" + e.getMessage() + ']'); @@ -759,6 +831,8 @@ private void deleteDirectory(File dir) throws IOException { return FileVisitResult.CONTINUE; } }); + + return res.get(); } /** @@ -813,6 +887,11 @@ private IgniteInternalFuture initLocalSnapshotStartSt "re-encryption process is not finished yet.")); } + if (cctx.snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) { + return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Snapshot " + + "'%s' is being deleted.".formatted(req.snapshotName()))); + } + List grpIds = new ArrayList<>(F.viewReadOnly(req.groups(), CU::cacheId)); Collection comprGrpIds = F.view(grpIds, i -> { CacheGroupDescriptor desc = cctx.cache().cacheGroupDescriptor(i); @@ -1282,7 +1361,7 @@ private IgniteInternalFuture initLocalSnapshotEndStag if (snpStartReq.incremental()) U.delete(snpOp.snapshotFileTree().incrementalSnapshotFileTree(snpStartReq.incrementIndex()).root()); else - deleteSnapshot(snpOp.snapshotFileTree()); + deleteLocalSnapshot(snpOp.snapshotFileTree(), cctx.kernalContext().pdsFolderResolver().fileTree().folderName(), null); } else if (!F.isEmpty(endReq.warnings())) { // Pass the warnings further to the next stage for the case when snapshot started from not coordinator. @@ -1446,6 +1525,27 @@ public boolean isSnapshotChecking(String snpName) { return checkSnpProc.isSnapshotChecking(snpName); } + /** + * @return {@code True} if a snapshot {@code snpName} delete operation is in progress. + */ + public boolean isSnapshotDeleting(String snpName, @Nullable String snpPath) { + return deleteSnpProc.isSnapshotDeleting(snpName, snpPath); + } + + /** + * Deletes the cluster-wide snapshot with the given name. + *

+ * The operation is rejected if a concurrent snapshot operation (create, restore, check, etc...) is in progress + * for the snapshot. + * + * @param name Snapshot name. + * @param snpPath Snapshot directory path. If {@code null}, the default configured snapshot directory will be used. + * @return Future which will be completed when the snapshot is deleted on all the baseline nodes. + */ + public IgniteFuture deleteSnapshot(String name, @Nullable String snpPath) { + return deleteSnpProc.start(name, snpPath); + } + /** * Sets the streamer warning flag to current snapshot process if it is active. */ @@ -2032,7 +2132,7 @@ public IgniteFutureImpl createSnapshot( if (!incremental && snpExists) { throw new IgniteException("Create snapshot request has been rejected. " + - "Snapshot with given name already exists on local node."); + "Snapshot with given name already exists on local node or the path is not empty."); } if (incremental) { @@ -2263,7 +2363,7 @@ public IgniteFutureImpl restoreSnapshot( if (SnapshotFileTree.incrementSnapshotDir(snpDir)) U.delete(snpDir); else - deleteSnapshot(snpDir); + deleteLocalSnapshot(snpDir); } if (log.isInfoEnabled()) { @@ -3957,7 +4057,7 @@ public LocalSnapshotSender(SnapshotFileTree sft) { log.info("The Local snapshot sender closed. All resources released [dbNodeSnpDir=" + sft.nodeStorage() + ']'); } else { - deleteSnapshot(sft); + deleteLocalSnapshot(sft, cctx.kernalContext().pdsFolderResolver().fileTree().folderName(), null); if (log.isDebugEnabled()) log.debug("Local snapshot sender closed due to an error occurred: " + th.getMessage()); @@ -4317,7 +4417,7 @@ public CancelSnapshotCallable(UUID reqId, String snpName) { } /** {@inheritDoc} */ - @Override public Boolean call() throws Exception { + @Override public Boolean call() { if (reqId != null) return ignite.context().cache().context().snapshotMgr().cancelLocalSnapshotOperations(reqId); else { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 2f19878617a15..1337ff8541e99 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridKernalContext; @@ -491,8 +492,13 @@ private IgniteInternalFuture prepareAndCheckMetas(UUID ig } if (!ctx.req.requestId().equals(req.requestId())) { - return new GridFinishedFuture<>(new IllegalStateException("Validation of snapshot '" + req.snapshotName() - + "' has already started [ctx=" + ctx + ']')); + return new GridFinishedFuture<>(new IgniteIllegalStateException("Validation of snapshot '" + req.snapshotName() + + "' has already started [req=" + req + ']')); + } + + if (kctx.cache().context().snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) { + return new GridFinishedFuture<>(new IgniteIllegalStateException("Snapshot '" + req.snapshotName() + + "' is being deleted [req=" + req + ']')); } // Excludes non-baseline initiator. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java new file mode 100644 index 0000000000000..46349aaafc406 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree; +import org.apache.ignite.internal.util.distributed.DistributedProcess; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.lang.IgniteReducer; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_SNAPSHOT; + +/** + * Distributed process to delete a cluster snapshot. The operation is rejected if any concurrent snapshot operation is + * active. + */ +public class SnapshotDeleteProcess { + /** Reject operation messages. */ + private static final String OP_REJECT_MSG = "Snapshot deletion was rejected. "; + + /** */ + private static final String SNP_PATH_ERR_PREF = "Provided snapshot path "; + + /** Kernal context. */ + private final GridKernalContext kctx; + + /** Logger. */ + private final IgniteLogger log; + + /** */ + private volatile boolean interrupted; + + /** Cluster-wide operation futures per request id on certain node. */ + private final Map> clusterOpFuts = new ConcurrentHashMap<>(); + + /** Process requests per snapshot name on each server node. */ + private final Set requests = ConcurrentHashMap.newKeySet(); + + /** The distributed process. */ + private final DistributedProcess distrProc; + + /** + * @param ctx Kernal context. + */ + public SnapshotDeleteProcess(GridKernalContext ctx) { + this.kctx = ctx; + + log = ctx.log(getClass()); + + distrProc = new DistributedProcess<>(ctx, DELETE_SNAPSHOT, this::deletePhase, this::reducePhase); + } + + /** + * Starts the cluster snapshot delete process. + * + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path (optional). + * @return Future that will be completed when the snapshot is deleted. + */ + public IgniteFuture start(String snpName, @Nullable String snpPath) { + UUID reqId = UUID.randomUUID(); + + var clusterOpFut = new GridFutureAdapter(); + + clusterOpFut.listen(fut -> clusterOpFuts.remove(reqId)); + + try { + synchronized (clusterOpFuts) { + if (interrupted || kctx.isStopping()) + throw new NodeStoppingException("Failed to start snapshot delete process: node is stopping."); + + clusterOpFuts.put(reqId, clusterOpFut); + } + + SnapshotDeleteRequest req = new SnapshotDeleteRequest(reqId, snpName, snpPath); + + distrProc.start(reqId, req); + } + catch (Throwable t) { + log.error("Failed to start distributed delete snapshot process [snpName=" + snpName + ", snpPath=" + snpPath + ']', t); + + clusterOpFut.onDone(t); + } + + return new IgniteFutureImpl<>(clusterOpFut); + } + + /** */ + private IgniteInternalFuture deletePhase(UUID ignored, SnapshotDeleteRequest req) { + if (interrupted || kctx.isStopping()) { + return new GridFinishedFuture<>(new NodeStoppingException(OP_REJECT_MSG + + " Node is stopping [req=" + req + ']')); + } + + if (kctx.cluster().get().localNode().isClient()) + return new GridFinishedFuture<>(new SnapshotDeleteResponse(null)); + + kctx.security().authorize(ADMIN_SNAPSHOT); + + IgniteSnapshotManager snpMgr = kctx.cache().context().snapshotMgr(); + + var curCreateRq = snpMgr.currentCreateRequest(); + + if (curCreateRq != null && curCreateRq.snpName.equals(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + "Snapshot with this name is being created [req=" + req + ']')); + } + + if (snpMgr.isRestoring(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + "Snapshot with this name is being restored [req=" + req + ']')); + } + + if (snpMgr.isSnapshotChecking(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + "Snapshot with this name is being checked [req=" + req + ']')); + } + + if (!kctx.rollingUpgrade().features().isActive(SNAPSHOT_DELETE_FEATURE)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + + "The snapshot deletion feature isn't activated yet [req=" + req + ']')); + } + + File path = kctx.pdsFolderResolver().fileTree().snapshotsRoot(); + + if (!F.isEmpty(req.snpPath)) { + File reqPath = new File(req.snpPath); + + path = reqPath.isAbsolute() + ? reqPath + : new File(path, req.snpPath); + + String pathValidationErr = validateAbsoluteSnapshotRoot(path); + + if (pathValidationErr != null) { + return new GridFinishedFuture<>(new IllegalArgumentException(OP_REJECT_MSG + + SNP_PATH_ERR_PREF + pathValidationErr + " [req=" + req + ']')); + } + } + + try { + if (!requests.add(req)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException("Deletion of the snapshot has already " + + "started [req=" + req + ']')); + } + + SnapshotFileTree snpFiles = new SnapshotFileTree(kctx, req.snpName, path.getAbsolutePath()); + + // We need to find and read snapshot metas to ensure the content is a snapshot. Also, the metas contain + // initial cluster topology and actual snasphot folder names. + List locMetas = kctx.cache().context().snapshotMgr().readSnapshotMetadatas(snpFiles); + + if (locMetas.isEmpty()) { + log.warning("Snapshot deletion won't process, no snapshot metadata found [req=" + req + ']'); + + return new GridFinishedFuture<>(new SnapshotDeleteResponse(SnapshotDeleteResponse.SnapshotDeleteStatus.NOT_FOUND)); + } + + // Future to delete snapshot contents according to snapshot metadatas. + GridCompoundFuture resultFut = + new GridCompoundFuture<>(new MetaFuturesReducer()); + + resultFut.listen(fut->requests.remove(req)); + + File path0 = path; + + for(var meta : locMetas) { + GridFutureAdapter perMetaFut = new GridFutureAdapter<>(); + + kctx.pools().getSnapshotExecutorService().submit(() -> { + try { + AtomicBoolean foundFlag = new AtomicBoolean(); + + // Read file tree of the snapshot. + var byMetaSft = new SnapshotFileTree(kctx, req.snpName, path0.getAbsolutePath(), meta.folderName(), + meta.consId); + + boolean deleted = snpMgr.deleteLocalSnapshot(byMetaSft, meta.folderName(), foundFlag); + + SnapshotDeleteResponse.SnapshotDeleteStatus res; + + if (foundFlag.get()) { + if (deleted && log.isInfoEnabled()) + log.info("Snapshot successfully deleted [req=" + req + ']'); + else if (!deleted) + log.warning("Snapshot deleted not completely [req=" + req + ']'); + + res = deleted + ? SnapshotDeleteResponse.SnapshotDeleteStatus.DELETED + : SnapshotDeleteResponse.SnapshotDeleteStatus.PARTLY_DELETED; + } + else { + if (log.isInfoEnabled()) + log.info("Snapshot not found to delete [req=" + req + ']'); + + res = SnapshotDeleteResponse.SnapshotDeleteStatus.NOT_FOUND; + } + + perMetaFut.onDone(res); + } catch (Throwable e) { + perMetaFut.onDone(e); + } + }); + + resultFut.add(perMetaFut); + } + + resultFut.markInitialized(); + + if (log.isInfoEnabled()) + log.info("Deletion of snapshot initialized [req=" + req + ']'); + + return resultFut; + } + catch (Throwable t) { + requests.remove(req); + + log.warning("An error occurred during snapshot deletion [req=" + req + ']', t); + + return new GridFinishedFuture<>(t); + } + } + + /** */ + private @Nullable String validateAbsoluteSnapshotRoot(@Nullable File path) { + if (path == null) + return null; + + assert path.isAbsolute(); + + var ignFileTree = kctx.pdsFolderResolver().fileTree(); + + for (var ignPath : ignFileTree.all()) { + if (contains(ignPath, path)) { + return ignPath.equals(ignFileTree.snapshotsRoot()) + ? null + : "belongs to a an Ignite's directory"; + } + } + + if (!path.exists()) + return "doesn't exist"; + + if (!path.isDirectory()) + return "is not a directory"; + + return null; + } + + /** */ + private void reducePhase(UUID reqId, Map results, Map errors) { + var clusterOpFut = clusterOpFuts.get(reqId); + + if (clusterOpFut == null) + return; + + assert clusterOpFut != null; + + try { + var errP = F.isEmpty(errors) ? null : F.first(errors.entrySet()); + + if (errP != null) { + log.warning("Snapshot deletion finished with an error [reqId=" + reqId + ", nodeId=" + + errP.getKey() + ", err='" + errP.getValue().getMessage() + "']", errP.getValue()); + + clusterOpFut.onDone(errP.getValue()); + + return; + } + + var completedNodes = new ArrayList(results.size()); + var uncompletedNodes = new ArrayList(results.size()); + var emptyNodes = new ArrayList(results.size()); + + results.forEach((nodeId, nodeRes) -> { + if (nodeRes.res != null) { + switch (nodeRes.res) { + case NOT_FOUND: + emptyNodes.add(nodeId); + break; + case DELETED: + completedNodes.add(nodeId); + break; + case PARTLY_DELETED: + uncompletedNodes.add(nodeId); + break; + default: + throw new IgniteIllegalStateException("Unknown snapshot deletion node result, [nodeRes=" + + nodeRes + ", nodeId=" + nodeId + ']'); + } + } + }); + + clusterOpFut.onDone(new SnapshotDeleteProcessResult( + completedNodes.isEmpty() ? null : completedNodes, + uncompletedNodes.isEmpty() ? null : uncompletedNodes, + emptyNodes.isEmpty() ? null : emptyNodes + )); + } + catch (Throwable t) { + clusterOpFut.onDone(t); + } + } + + /** */ + public boolean isSnapshotDeleting(String snpName, @Nullable String snpPath) { + return requests.contains(new SnapshotDeleteRequest(null, snpName, snpPath)); + } + + /** + * @param err The interrupt reason. + */ + void interrupt(Throwable err) { + synchronized (clusterOpFuts) { + interrupted = true; + } + + clusterOpFuts.forEach((reqId, clusterOpFut) -> clusterOpFut.onDone(err)); + + clusterOpFuts.clear(); + } + + /** */ + public static boolean contains(File root, File candidate) { + Path root0 = root.toPath().toAbsolutePath().normalize(); + Path candidate0 = candidate.toPath().toAbsolutePath().normalize(); + + return candidate0.startsWith(root0); + } + + /** */ + private static class MetaFuturesReducer implements IgniteReducer { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** */ + private @Nullable SnapshotDeleteResponse.SnapshotDeleteStatus res = null; + + /** {@inheritDoc} */ + @Override public boolean collect(@Nullable SnapshotDeleteResponse.SnapshotDeleteStatus status) { + synchronized (this) { + if (res == null || res == status) + res = status; + else + res = SnapshotDeleteResponse.SnapshotDeleteStatus.PARTLY_DELETED; + } + + return true; + } + + /** {@inheritDoc} */ + @Override public SnapshotDeleteResponse reduce() { + return new SnapshotDeleteResponse(res == null ? SnapshotDeleteResponse.SnapshotDeleteStatus.NOT_FOUND : res); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java new file mode 100644 index 0000000000000..95a904dc6d55b --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.util.Collection; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** Result of {@link SnapshotDeleteProcess}. */ +public final class SnapshotDeleteProcessResult extends IgniteDataTransferObject { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Nodes which found snapshot data and completely removed it. */ + @Order(0) + @Nullable Collection completedNodes; + + /** Nodes which found snapshot data but didn't remove it completely. */ + @Order(1) + @Nullable Collection uncompletedNodes; + + /** Server nodes which didn't find any snapshot data. */ + @Order(2) + @Nullable Collection emptyNodes; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteProcessResult() { + // No-op. + } + + /** */ + public SnapshotDeleteProcessResult( + @Nullable Collection completedNodes, + @Nullable Collection uncompletedNodes, + @Nullable Collection emptyNodes + ) { + this.completedNodes = completedNodes; + this.uncompletedNodes = uncompletedNodes; + this.emptyNodes = emptyNodes; + } + + /** */ + public @Nullable Collection completedNodes() { + return completedNodes; + } + + /** */ + public @Nullable Collection uncompletedNodes() { + return uncompletedNodes; + } + + /** */ + public @Nullable Collection emptyNodes() { + return emptyNodes; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java new file mode 100644 index 0000000000000..37577c1b2eb60 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Cluster snapshot delete distributed process request. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteRequest implements Message { + /** Request ID. */ + @Order(0) + UUID reqId; + + /** Snapshot name. */ + @Order(1) + String snpName; + + /** Snapshot directory path. */ + @Order(2) + @Nullable String snpPath; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteRequest() { + // No-op. + } + + /** + * @param reqId Request ID. + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. + */ + SnapshotDeleteRequest(UUID reqId, String snpName, @Nullable String snpPath) { + this.reqId = reqId; + this.snpName = snpName; + this.snpPath = snpPath; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) + return false; + + SnapshotDeleteRequest other = (SnapshotDeleteRequest)o; + + return snpName.equals(other.snpName) && Objects.equals(snpPath, other.snpPath); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(snpName, snpPath); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SnapshotDeleteRequest.class, this, super.toString()); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java new file mode 100644 index 0000000000000..917d23da2d3cb --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Single-node result of the snapshot deletion distributed process. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteResponse implements Message { + /** {@code null} for client node. */ + @Order(0) + @Nullable SnapshotDeleteResponse.SnapshotDeleteStatus res; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteResponse() { + // No-op. + } + + /** {@code null} for client node. */ + SnapshotDeleteResponse(@Nullable SnapshotDeleteResponse.SnapshotDeleteStatus res) { + this.res = res; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SnapshotDeleteResponse.class, this); + } + + /** */ + public enum SnapshotDeleteStatus { + /** Snapshot found and completely deleted. */ + DELETED, + + /** Snapshot found but some files or directories might not be deleted (locked). */ + PARTLY_DELETED, + + /** Snapshot not found. */ + NOT_FOUND; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java index 643838ac3c445..4a8fea2b43b73 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java @@ -35,7 +35,10 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; import org.apache.ignite.internal.util.typedef.F; -/** Snapshot task to verify snapshot metadata on the baseline nodes for given snapshot name. */ +/** + * Snapshot task to verify snapshot metadata on the baseline nodes for given snapshot name. + * TODO : Revise in https://issues.apache.org/jira/browse/IGNITE-29062 + */ public class SnapshotMetadataVerificationTask implements Supplier> { /** */ private final IgniteEx ignite; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 77f56f4d4cc7f..f4fe684eba584 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -659,6 +659,9 @@ private IgniteInternalFuture prepare(UUID igno if (snpMgr.isSnapshotCreating()) throw new IgniteCheckedException(OP_REJECT_MSG + "A cluster snapshot operation is in progress."); + if (snpMgr.isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) + throw new IgniteException(OP_REJECT_MSG + "A snapshot '" + req.snapshotName() + "' delete operation is in progress."); + if (ctx.encryption().isMasterKeyChangeInProgress()) { return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Master key changing " + "process is not finished yet.")); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java index 7b3e55b85d3c4..a14e52331cda0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java @@ -93,4 +93,7 @@ public class SupportedFeatureRegistry { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); + + /** */ + public static final IgniteFeature SNAPSHOT_DELETE_FEATURE = new IgniteCoreFeature(1); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java index 3d0ce062f4f75..9c428e0fa748c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java @@ -519,5 +519,10 @@ public enum DistributedProcessType { * Cluster version finalization abort process. */ RU_ABORT_VERSION_FINALIZATION, + + /** + * Delete snapshot procedure. + */ + DELETE_SNAPSHOT } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java index 8857a4ddd656e..2c960ec62d49c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java @@ -36,7 +36,10 @@ import java.util.Queue; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -98,6 +101,8 @@ import org.apache.ignite.lang.IgniteFutureCancelledException; import org.apache.ignite.lang.IgniteFutureTimeoutException; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi; @@ -173,6 +178,9 @@ protected Function valueBuilder() { return valBuilder; } + /** */ + protected @Nullable AbstractTestPluginProvider pluginProvider; + /** Enable encryption of all caches in {@code IgniteConfiguration} before start. */ @Parameterized.Parameter public boolean encryption; @@ -182,7 +190,7 @@ protected Function valueBuilder() { public boolean onlyPrimary; /** Parameters. */ - @Parameterized.Parameters(name = "encryption={0}, onlyPrimay={1}") + @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}") public static Collection params() { List res = new ArrayList<>(); @@ -217,6 +225,9 @@ protected static Collection encryptionParameters() { if (cfg.isClientMode()) return cfg; + if (pluginProvider != null) + cfg.setPluginProviders(pluginProvider); + return cfg.setConsistentId(igniteInstanceName) .setDataStorageConfiguration(new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() @@ -283,6 +294,17 @@ public void afterTestSnapshot() throws Exception { cleanPersistenceDir(); } + /** {@inheritDoc} */ + @Override protected void cleanPersistenceDir() throws Exception { + super.cleanPersistenceDir(); + + // Clean all: also separated snapshot working directories and custom snapshot pathes. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + /** * @param evts Events to check. * @throws IgniteInterruptedCheckedException If interrupted. @@ -814,6 +836,85 @@ public static void doSnapshotCancellationTest( assertEquals("Snapshot directory must be empty due to snapshot cancelled", 0, snpDir.list().length); } + /** Tests concurrent snapshot deletion. */ + protected void doTestConcurrentSnapshotDeleteOperation( + ExRunnable prepareCluster, + ExRunnable concurrentOp, + @Nullable Function errValidator, + boolean rerunAtTheEnd + ) throws Exception { + CountDownLatch delProcInitLatch = new CountDownLatch(1); + CountDownLatch delProcProceedLatch = new CountDownLatch(1); + + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public boolean deleteLocalSnapshot( + SnapshotFileTree sft, + String nodeFolderName, + @Nullable AtomicBoolean existsFlag + ) { + delProcInitLatch.countDown(); + + try { + assertTrue(delProcProceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + + return super.deleteLocalSnapshot(sft, nodeFolderName, existsFlag); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + prepareCluster.run(); + + var delFut = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null); + + assertTrue(delProcInitLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + try { + concurrentOp.run(); + + if (errValidator != null) + throw new IllegalStateException("Exception is not thrown."); + } + catch (Exception e) { + if (errValidator == null || !errValidator.apply(e)) + throw new IllegalStateException("Unexpected exception: " + e.getMessage(), e); + } + + delProcProceedLatch.countDown(); + + var delRes = delFut.get(getTestTimeout()); + + assertFalse(delRes.completedNodes.isEmpty()); + + if (!rerunAtTheEnd) + return; + + assertThrowsAnyCause( + null, + () -> { + concurrentOp.run(); + + return null; + }, + IllegalArgumentException.class, + "Snapshot does not exists " + ); + } + /** * @param sft Snapshot file tree. * @param parts Collection of pairs group and appropriate cache partition to be snapshot. @@ -994,6 +1095,13 @@ public void waitBlockedSize(int size, long timeout) throws IgniteInterruptedChec } } + /** */ + @FunctionalInterface + protected interface ExRunnable { + /** */ + void run() throws Exception; + } + /** */ protected static class Value { /** */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index 306d44224a4d6..268ff54775232 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -46,6 +46,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.BaselineNode; @@ -124,6 +125,7 @@ import static org.apache.ignite.testframework.GridTestUtils.cartesianProduct; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; /** * Cluster-wide snapshot check procedure tests. @@ -1201,6 +1203,37 @@ public void testConcurrentFullCheckAndFullRestoreDeclined() throws Exception { ); } + /** */ + @Test + public void testConcurrentSnapshotDeleteAndCheckOperations() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> prepareGridsAndSnapshot(4, 3, 1, false), + () -> snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, null).get(), + e -> e.getMessage().contains("Snapshot '%s' is being deleted".formatted(SNAPSHOT_NAME)), + true + ); + } + + /** */ + @Test + public void testConcurrentSnapshotDeleteAndCheckOperationsWithDifferentPath() throws Exception { + // The test uses thread blocking. + assumeTrue(snpThrdPoolSz > 1); + + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + prepareGridsAndSnapshot(4, 3, 1, false); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout()); + }, + () -> snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, snpPath).get(), + null, + false + ); + } + /** Tests that concurrent snapshot full check is declined when the same snapshot is being fully restored (checked). */ @Test public void testConcurrentTheSameSnpFullCheckWhenFullyRestoringDeclined() throws Exception { @@ -1409,8 +1442,8 @@ private void prepareGridsAndSnapshot(int servers, int baseLineCnt, int clients, /** * Tests concurrent snapshot operations related to the snapshot checking. * - * @param originatorOp First snapshot operation on an originator node. - * @param trierOp Second concurrent snapshot operation on a trier node. + * @param firstOp First snapshot operation on an originator node. + * @param secondOp Second concurrent snapshot operation on a trier node. * @param firstDelay First distributed process full message of {@code originatorOp} to delay on the coordinator * to launch {@code trierOp}. * @param secondDelay Second distributed process full message of {@code originatorOp} to delay on the coordinator @@ -1422,8 +1455,8 @@ private void prepareGridsAndSnapshot(int servers, int baseLineCnt, int clients, * @param cleaner If not {@code null}, is executed at the end. */ private void doTestConcurrentSnpCheckOperations( - Supplier> originatorOp, - Supplier> trierOp, + Supplier> firstOp, + Supplier> secondOp, DistributedProcess.DistributedProcessType firstDelay, @Nullable DistributedProcess.DistributedProcessType secondDelay, boolean expectFailure, @@ -1440,17 +1473,17 @@ private void doTestConcurrentSnpCheckOperations( && ((FullMessage)msg).type() == firstDelay.ordinal() && (waitForBothFirstDelays || firstDelayed.compareAndSet(false, true))); - IgniteFuture fut = originatorOp.get(); + IgniteFuture fut = firstOp.get(); discoSpi(grid(0)).waitBlocked(getTestTimeout()); - IgniteFuture fut2 = trierOp.get(); + IgniteFuture fut2 = secondOp.get(); if (expectFailure) { assertThrowsAnyCause( log, fut2::get, - IllegalStateException.class, + IgniteIllegalStateException.class, "Validation of snapshot '" + SNAPSHOT_NAME + "' has already started" ); @@ -1473,7 +1506,7 @@ private void doTestConcurrentSnpCheckOperations( assertThrowsAnyCause( log, fut2::get, - IllegalStateException.class, + IgniteIllegalStateException.class, "Validation of snapshot '" + SNAPSHOT_NAME + "' has already started" ); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java new file mode 100644 index 0000000000000..6c608f25a07a5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; + +import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; + +/** */ +public class IgniteClusterSnapshotDeleteRollingUpgradeTest extends AbstractRollingUpgradeTest { + /** */ + private static final int ALL_GRIDS = 4; + + /** */ + private static final int CLIENTS = 1; + + /** */ + private static final String SNP_NAME = "testSnapshot"; + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName, String ver) throws Exception { + var cfg = super.getConfiguration(igniteInstanceName, ver); + + cfg.setDataStorageConfiguration( + new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE) + ) + ); + + return cfg; + } + + /** */ + @Test + public void testConcurrentUnfinishedRU() throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.19.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + int testNodeIx = ALL_GRIDS - CLIENTS - 1; + + createCacheAndSnapshot(testNodeIx); + + ru(grid(testNodeIx)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.19.1"); + } + + spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && + snm.type() == RU_PREPARE_VERSION_FINALIZATION.ordinal()); + + var finalizeFut = GridTestUtils.runAsync(() -> ru(testNodeIx).finalizeClusterVersion()); + + assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); + + ensureSnapshotDeletionFailed(); + + spi(grid(testNodeIx)).stopBlock(); + + assertFalse(spi(grid(testNodeIx)).hasBlockedMessages()); + + finalizeFut.get(getTestTimeout()); + + for (int i = 0; i < ALL_GRIDS; i++) + assertFalse(ru(grid(i)).isVersionUpgradeEnabled()); + + assertFalse(F.isEmpty(snp(1).deleteSnapshot(SNP_NAME, null).get(getTestTimeout()).completedNodes)); + } + + /** */ + @Test + public void testNodeNotSupportingSnapshotDeleteFeature() throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.19.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + createCacheAndSnapshot(1); + + ensureSnapshotDeletionFailed(); + + ru(grid(0)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.19.1"); + + ensureSnapshotDeletionFailed(); + } + + ru(grid(1)).finalizeClusterVersion(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertFalse(ru(grid(i)).isVersionUpgradeEnabled()); + + assertFalse(F.isEmpty(snp(i).deleteSnapshot(SNP_NAME, null).get().completedNodes)); + + if (i < ALL_GRIDS - 1) + createSnapshot(i); + } + } + + /** */ + private void createCacheAndSnapshot(int gridIdx) { + int partsCnt = 32; + int keysCnt = partsCnt * 10; + + grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.REPLICATED) + .setBackups(1) + .setAffinity(new RendezvousAffinityFunction().setPartitions(32)) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + try (var ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < keysCnt; i++) + ds.addData(i, i); + } + + createSnapshot(gridIdx); + } + + /** */ + private void createSnapshot(int gridIdx) { + snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout()); + } + + /** */ + private void ensureSnapshotDeletionFailed() { + for (int i = 0; i < ALL_GRIDS; i++) { + int i0 = i; + + assertThrowsAnyCause( + null, + () -> snp(i0).deleteSnapshot(SNP_NAME, null).get(), + IgniteIllegalStateException.class, + "The snapshot deletion feature isn't activated yet" + ); + } + } + + /** */ + private IgniteSnapshotManager snp(int gridIdx) { + return grid(gridIdx).context().cache().context().snapshotMgr(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java new file mode 100644 index 0000000000000..5befd3af8adfd --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java @@ -0,0 +1,739 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree; +import org.apache.ignite.internal.util.distributed.DistributedProcess; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; + +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PRELOAD; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_INCREMENTAL_SNAPSHOT_START; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.START_SNAPSHOT; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; + +/** */ +@RunWith(Parameterized.class) +public class IgniteClusterSnapshotDeleteTest extends AbstractSnapshotSelfTest { + /** */ + private boolean separatedWorkDir; + + /** */ + @Parameter(2) + public boolean incremental = true; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + var cfg = super.getConfiguration(igniteInstanceName); + + if (separatedWorkDir) + cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); + + return cfg; + } + + /** Parameters. */ + @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}, incremental={2}") + public static Collection runParams() { + Collection res = new ArrayList<>(); + + for (boolean incremental : F.asList(false, true)) { + for (Object[] src0 : params()) { + Object[] res0 = new Object[src0.length + 1]; + System.arraycopy(src0, 0, res0, 0, src0.length); + + res0[src0.length] = incremental; + + res.add(res0); + } + } + + return res; + } + + /** {@inheritDoc} */ + @Override public void afterTestSnapshot() throws Exception { + super.afterTestSnapshot(); + + G.allGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override public void beforeTestSnapshot() throws Exception { + super.beforeTestSnapshot(); + + /** Handy if test running is interrupted and {@link #afterTestSnapshot()} isn't invoked. */ + cleanPersistenceDir(); + } + + /** Tests snapshot deletion when one node finds snapshot but fails to delete its data. */ + @Test + public void testUncompletedNodes() throws Exception { + separatedWorkDir = true; + + // Simulates a deletion error on some node. + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public boolean deleteLocalSnapshot( + SnapshotFileTree sft, + String nodeFolderName, + @Nullable AtomicBoolean existsFlag + ) { + if (ctx.localNode().id().equals(grid(1).localNode().id())) { + existsFlag.set(true); + + return false; + } + + return super.deleteLocalSnapshot(sft, nodeFolderName, existsFlag); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + var delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertTrue(F.isEmpty(delSnpRes.emptyNodes)); + assertFalse(F.isEmpty(delSnpRes.uncompletedNodes)); + assertTrue(delSnpRes.uncompletedNodes.contains(grid(1).localNode().id())); + } + + /** */ + @Test + public void testDeleteNotSnapshotSharedDirectory() throws Exception { + doTestDeleteNotSnapshot(false); + } + + /** */ + @Test + public void testDeleteNotSnapshotDedicatedDirectories() throws Exception { + doTestDeleteNotSnapshot(true); + } + + /** */ + protected void doTestDeleteNotSnapshot(boolean separatedWorkDir) throws Exception { + this.separatedWorkDir = separatedWorkDir; + + startGridsWithCache(3, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg); + + snp(grid(1)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + var snpSft = new SnapshotFileTree(grid(1).context(), SNAPSHOT_NAME, null); + + // Ensure that all the snapshot node folders exist. + assertTrue(snpSft.binaryMeta().exists()); + assertTrue(new SnapshotFileTree(grid(0).context(), SNAPSHOT_NAME, null, folderName(0), consistentId(0)) + .binaryMeta().exists()); + assertTrue(new SnapshotFileTree(grid(2).context(), SNAPSHOT_NAME, null, folderName(2), consistentId(2)) + .binaryMeta().exists()); + + assertTrue(snpSft.meta().exists()); + assertTrue(U.delete(snpSft.meta())); + assertFalse(snpSft.meta().exists()); + + var delSnpRes = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + // Check the result. + if(separatedWorkDir) { + // One node doesn't find meta, decided not a snapshot. + assertTrue(F.isEmpty(delSnpRes.uncompletedNodes)); + assertEquals(2, delSnpRes.completedNodes.size()); + assertEquals(1, delSnpRes.emptyNodes.size()); + assertTrue(delSnpRes.emptyNodes.contains(grid(1).localNode().id())); + } else { + // All nodes may see some metas, may try to delete snapshot by the metas, but see that the snapshot directory isn't empty. + // Some node may not get any meta to process. But node of the nodes can say the snapshot 100% deleted. + assertTrue(F.isEmpty(delSnpRes.completedNodes)); + + int cnt = 0; + + if (!F.isEmpty(delSnpRes.emptyNodes)) + cnt += delSnpRes.emptyNodes.size(); + + if (!F.isEmpty(delSnpRes.uncompletedNodes)) + cnt += delSnpRes.uncompletedNodes.size(); + + assertEquals(3, cnt); + } + + assertTrue(snpSft.binaryMeta().exists()); + assertFalse(new SnapshotFileTree(grid(0).context(), SNAPSHOT_NAME, null, folderName(0), consistentId(0)) + .binaryMeta().exists()); + assertFalse(new SnapshotFileTree(grid(2).context(), SNAPSHOT_NAME, null, folderName(2), consistentId(2)) + .binaryMeta().exists()); + } + + /** */ + private String consistentId(int gridIdx) { + return grid(gridIdx).configuration().getConsistentId().toString(); + } + + /** */ + private String folderName(int gridIdx) { + return grid(gridIdx).context().pdsFolderResolver().fileTree().folderName(); + } + + /** Test delete snapshot directly in Ignite. */ + @Test + public void testDeletionInIgnite() throws Exception { + /** No need to multiply this test. */ + assumeFalse(onlyPrimary || encryption || incremental); + + startGridsMultiThreaded(3); + + List dirsToTest = new ArrayList<>(30); + + var ignFileTree = grid(0).context().pdsFolderResolver().fileTree(); + + dirsToTest.addAll(ignFileTree.all().stream().filter(f -> f.compareTo(ignFileTree.snapshotsRoot()) != 0) + .map(File::getAbsolutePath).toList()); + + IgniteSnapshotManager snpMgr = snp(grid(0)); + var fileSep = File.separator; + var belongsToErrMsg = "belongs to a an Ignite's directory"; + + for (var dir : dirsToTest) { + List tests = new ArrayList<>(20); + + tests.add(dir + fileSep); + tests.add(dir + fileSep + fileSep); + tests.add(dir.replaceAll(fileSep, fileSep + fileSep)); + tests.add(dir + fileSep + "unexisting"); + + for (var test : tests) { + if (log.isInfoEnabled()) + log.info("Testing path: " + test); + + assertThrowsAnyCause( + null, + () -> snpMgr.deleteSnapshot(SNAPSHOT_NAME, test).get(getTestTimeout()), + IllegalArgumentException.class, + belongsToErrMsg + ); + } + + tests.clear(); + tests.add(dir + File.pathSeparator); + tests.add(dir + "_unexisting"); + + for (var test : tests) { + if (log.isInfoEnabled()) + log.info("Testing path: " + test); + + try { + snpMgr.deleteSnapshot(SNAPSHOT_NAME, test).get(getTestTimeout()); + + throw new IllegalStateException("An exception wasn't thrown."); + } + catch (Exception e) { + var m = e.getMessage(); + + if (!X.hasCause(e, IllegalArgumentException.class) + || (!m.contains(belongsToErrMsg) && !m.contains("Provided snapshot path doesn't exist")) + ) + throw new IllegalStateException("Unexpected exception.", e); + } + } + } + + var snpRoot = ignFileTree.snapshotsRoot(); + + var delRes = snpMgr.deleteSnapshot(SNAPSHOT_NAME, snpRoot.getAbsolutePath()).get(getTestTimeout()); + + assertTrue(F.isEmpty(delRes.completedNodes)); + assertTrue(F.isEmpty(delRes.uncompletedNodes)); + assertEquals(3, delRes.emptyNodes.size()); + + delRes = snpMgr.deleteSnapshot(SNAPSHOT_NAME, new File(snpRoot, "unexisting").getAbsolutePath()).get(getTestTimeout()); + + assertTrue(F.isEmpty(delRes.completedNodes)); + assertTrue(F.isEmpty(delRes.uncompletedNodes)); + assertEquals(3, delRes.emptyNodes.size()); + } + + /** Tests snapshot deletion when one node has no snapshot data. */ + @Test + public void testEmptyNodes() throws Exception { + separatedWorkDir = true; + + startGridsWithCache(2, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + startGrid(G.allGrids().size()); + + var delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertFalse(F.isEmpty(delSnpRes.emptyNodes)); + assertTrue(delSnpRes.emptyNodes.contains(grid(G.allGrids().size() - 1).localNode().id())); + assertTrue(F.isEmpty(delSnpRes.uncompletedNodes)); + } + + /** Tests snapshot deletion repeat after an offline node restarts. */ + @Test + public void testDeletionRepeatAfterOfflineNodeStarts() throws Exception { + separatedWorkDir = true; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + int stoppedNodeIdx = G.allGrids().size() - 1; + + UUID stoppedNodeId = grid(stoppedNodeIdx).localNode().id(); + + stopGrid(stoppedNodeIdx); + + var delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(2, delSnpRes.completedNodes.size()); + assertFalse(delSnpRes.completedNodes.contains(stoppedNodeId)); + + assertTrue(F.isEmpty(delSnpRes.uncompletedNodes)); + assertTrue(F.isEmpty(delSnpRes.emptyNodes)); + + startGrid(stoppedNodeIdx); + + stoppedNodeId = grid(stoppedNodeIdx).localNode().id(); + + delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(1, delSnpRes.completedNodes.size()); + assertTrue(delSnpRes.completedNodes.contains(stoppedNodeId)); + + assertTrue(F.isEmpty(delSnpRes.uncompletedNodes)); + assertEquals(2, delSnpRes.emptyNodes.size()); + } + + /** Test snapshot deletion process when one node leaves. */ + @Test + public void testNodeStopsInTheMiddle() throws Exception { + // Incremental snapshots don't support only-primary and encryption mode. + assumeTrue(!incremental || !(onlyPrimary || encryption)); + + separatedWorkDir = true; + + CountDownLatch beginLatch = new CountDownLatch(1); + CountDownLatch proceedLatch = new CountDownLatch(1); + + // Simulates a deletion error on some node. + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public boolean deleteLocalSnapshot( + SnapshotFileTree sft, + String nodeFolderName, + @Nullable AtomicBoolean existsFlag + ) { + if (ctx.localNode().id().equals(grid(1).localNode().id())) { + beginLatch.countDown(); + + try { + assertTrue(proceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + } + + return super.deleteLocalSnapshot(sft, nodeFolderName, existsFlag); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + var delFut = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null); + + assertTrue(beginLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + UUID stoppedGridId = grid(1).localNode().id(); + + stopGrid(1); + + proceedLatch.countDown(); + + var delRes = delFut.get(getTestTimeout()); + + assertEquals(2, delRes.completedNodes.size()); + assertFalse(delRes.completedNodes.contains(stoppedGridId)); + + startGrid(1); + + delRes = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(1, delRes.completedNodes.size()); + assertTrue(delRes.completedNodes.contains(grid(1).localNode().id())); + } + + /** Tests that a concurrent deletion of a snapshot with the same name but different path is allowed. */ + @Test + public void testConcurrentDeleteOfTheSameSnapshotDifferentPath() throws Exception { + // Incremental snapshots don't support encryption and only-primary mode. + assumeTrue(!incremental || !(encryption || onlyPrimary)); + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(TIMEOUT); + + if (incremental) + addIncrementalSnapshot(null); + + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(snpPath); + + TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi(); + + commSpi1.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage msg0 && msg0.type() == DELETE_SNAPSHOT.ordinal()); + + var delFut0 = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null); + var delFut1 = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, snpPath); + + commSpi1.waitForBlocked(2, getTestTimeout()); + + commSpi1.stopBlock(); + + var delRes0 = delFut0.get(getTestTimeout()); + var delRes1 = delFut1.get(getTestTimeout()); + + assertFalse((delRes0.completedNodes().isEmpty())); + assertFalse(delRes1.completedNodes().isEmpty()); + } + + /** Tests that a concurrent deletion of the same snapshot is declined. */ + @Test + public void testConcurrentDeleteOfTheSameSnapshot() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE, false), + () -> snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()), + e -> e.getMessage().contains("Deletion of the snapshot has already started"), + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot check operation is in progress. */ + @Test + public void testSnapshotDeleteWhenCheckInProgress() throws Exception { + // Incremental snapshots don't support encryption. + assumeTrue(!incremental || !encryption); + + doTestConcurrentSnapshotDelete( + () -> new IgniteFutureImpl<>(snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, null, incremental ? 1 : 0)), + F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS), + true, + null, + "Snapshot with this name is being checked", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot create operation is in progress. */ + @Test + public void testSnapshotDeleteWhenCreateInProgress() throws Exception { + // Incremental snapshots don't support encryption and only-primary mode. + assumeTrue(!incremental || !(encryption || onlyPrimary)); + + doTestConcurrentSnapshotDelete( + () -> snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, incremental, onlyPrimary), + F.asList(START_SNAPSHOT, END_SNAPSHOT), + false, + () -> { + snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + if (incremental) + snp(grid(0)).createSnapshot(SNAPSHOT_NAME).get(getTestTimeout()); + }, + "Snapshot with this name is being created", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore begins. */ + @Test + public void testSnapshotDeleteWhenRestoreBegins() throws Exception { + // Incremental snapshots don't support encryption. + assumeTrue(!incremental || !encryption); + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS), + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + "Snapshot with this name is being checked", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore is in progress. */ + @Test + public void testSnapshotDeleteWhenRestoreInProgress() throws Exception { + // Incremental snapshots don't support encryption. + assumeTrue(!incremental || !encryption); + + var restoreMsgs = F.asList( + RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE, + RESTORE_CACHE_GROUP_SNAPSHOT_PRELOAD, + RESTORE_CACHE_GROUP_SNAPSHOT_START + ); + + if (incremental) { + restoreMsgs = new ArrayList<>(restoreMsgs); + restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START); + } + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + restoreMsgs, + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + "Snapshot with this name is being restored", + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore is in progress but fails. */ + @Test + public void testSnapshotDeleteWhenRestoreProgressFails() throws Exception { + // An in-the-middle failure won't allow to start restoring the incrementals. + assumeFalse(incremental); + + var restoreMsgs = F.asList(RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK); + + if (incremental) { + restoreMsgs = new ArrayList<>(restoreMsgs); + restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START); + } + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + restoreMsgs, + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + + SnapshotFileTree sft = snapshotFileTree(grid(1), SNAPSHOT_NAME); + + String failingFilePath = sft.partitionFile(dfltCacheCfg, primaries[0]).getAbsolutePath() + .replace(sft.nodeStorage().getAbsolutePath(), ""); + + grid(1).context().cache().context().snapshotMgr().ioFactory((file, modes) -> { + FileIO delegate = new RandomAccessFileIOFactory().create(file, modes); + + if (file.getPath().endsWith(failingFilePath)) + throw new RuntimeException("Test exception"); + + return delegate; + }); + }, + "Snapshot with this name is being restored", + true + ); + } + + /** + * @param firstOp First cluster-wide snapshot operation. + * @param msgsToWatch {@link SingleNodeMessage#type()} relating to {@code firstOp} to block on one node. + * @param precreateSnp If {@code true}, creates snapshot after the cluster start. + * @param prepareIteration If not {@code null}, is invoked in the beginning of test iteration at each {@code msgsToWatch}. + * @param concurrentMsgErr Test of failed concurrent to {@code firstOp} delete snapshot operation to watch. + * @param ignoreFirstOpFailure If {@code true}, possible failure of {@code firstOp} is ignored. + */ + protected void doTestConcurrentSnapshotDelete( + Supplier> firstOp, + Collection msgsToWatch, + boolean precreateSnp, + @Nullable Runnable prepareIteration, + String concurrentMsgErr, + boolean ignoreFirstOpFailure + ) throws Exception { + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + if (precreateSnp) { + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(TIMEOUT); + + if (incremental) + addIncrementalSnapshot(null); + } + + TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi(); + + for (var nodeResMsgType : msgsToWatch) { + if (log.isInfoEnabled()) + log.info("Iteration with message-to-wait-for type: " + nodeResMsgType); + + if (prepareIteration != null) + prepareIteration.run(); + + commSpi1.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage msg0 && msg0.type() == nodeResMsgType.ordinal()); + + var firstFut = firstOp.get(); + + commSpi1.waitForBlocked(1, getTestTimeout()); + + assertThrowsAnyCause( + null, + () -> { + snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + return null; + }, + IgniteIllegalStateException.class, + concurrentMsgErr + ); + + commSpi1.stopBlock(); + + if (ignoreFirstOpFailure) { + try { + firstFut.get(getTestTimeout()); + } + catch (Exception e) { + if (log.isDebugEnabled()) + log.debug("The first operation failed but a failure is expected. Failure: " + e.getMessage()); + } + } + else + firstFut.get(getTestTimeout()); + } + } + + /** */ + private void addIncrementalSnapshot(@Nullable String path) { + try (var ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = CACHE_KEYS_RANGE; i < CACHE_KEYS_RANGE + CACHE_KEYS_RANGE / 4; i++) + ds.addData(i, i); + } + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, path, true, onlyPrimary).get(getTestTimeout()); + } + + /** {@inheritDoc} */ + @Override protected void awaitPartitionMapExchange() { + try { + super.awaitPartitionMapExchange(); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 78ec729ec931b..84dd1f57c05d8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -333,6 +333,45 @@ public void testCreateSnapshotDuringRestore() throws Exception { assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE); } + /** Tests that snapshot restore is declined when the same snapshot is being deleted. */ + @Test + public void testConcurrentSnapshotDeleteAndRestoreOperations() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE), + () -> snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null).get(), + e -> e.getMessage().contains("Snapshot '%s' is being deleted".formatted(SNAPSHOT_NAME)), + true + ); + } + + /** */ + @Test + public void testConcurrentSnapshotDeleteAndRestoreOperationsWithDifferentPath() throws Exception { + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithSnapshot(3, CACHE_KEYS_RANGE); + + grid(0).createCache(DEFAULT_CACHE_NAME); + + try (var ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < CACHE_KEYS_RANGE; ++i) + ds.addData(i, i); + } + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout()); + + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + () -> snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, snpPath, null).get(), + null, + false + ); + } + /** * Ensures that the cache doesn't start if one of the baseline nodes fails. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index 570761dac227f..1270c77394d93 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -615,6 +615,43 @@ public void testSnapshotExistsException() throws Exception { waitForEvents(EVT_CLUSTER_SNAPSHOT_STARTED, EVT_CLUSTER_SNAPSHOT_FAILED); } + /** + * Tests that snapshot create detects concurrent deletion, or detects still existing snapshot or successfully + * proceeds if snapshot already deleted. + */ + @Test + public void testConcurrentSnapshotDeleteOperation() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(); + }, + () -> snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(), + e -> e.getMessage().contains("Snapshot with given name already exists"), + false + ); + } + + /** + * Tests that a concurrent deletion of a same-named snapshot is allowed if it has a different path. + */ + @Test + public void testConcurrentSnapshotDeleteOperationWithDifferentPath() throws Exception { + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(); + }, + () -> snp(grid(2)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(), + null, + false + ); + } + /** @throws Exception If fails. */ @Test public void testClusterSnapshotCleanedOnLeft() throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java index 38056daa0b428..c6108f4a28fb8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java @@ -21,4 +21,7 @@ public class TestIgniteReleaseFeatures_2_19_1 { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = TestIgniteReleaseFeatures_2_19_0.ROLLING_UPGRADE_FEATURE; + + /** */ + public static final IgniteFeature SNAPSHOT_DELETE_FEATURE = SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java index 5673ed2b71af3..857e317599640 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java @@ -21,6 +21,8 @@ import java.util.Collection; import java.util.List; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckTest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteRollingUpgradeTest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotHandlerTest; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.DynamicSuite; @@ -48,6 +50,8 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotCheckTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotHandlerTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteRollingUpgradeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output index dd6ff4e43caf5..c122892ee1f66 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output @@ -269,6 +269,13 @@ This utility can do the following commands: Get the status of the current snapshot operation: control.(sh|bat) --snapshot status + Deletes snapshot and all its increments from all the online server nodes: + control.(sh|bat) --snapshot delete snapshot_name [--src path] + + Parameters: + snapshot_name - Snapshot name. + --src path - Path to the directory where the snapshot is located. If not specified, the default configured snapshot directory will be used. + Change cluster tag to new value: control.(sh|bat) --change-tag newTagValue [--yes] diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output index 76f0f8c7e0e37..30a77600c6efa 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output @@ -269,6 +269,13 @@ This utility can do the following commands: Get the status of the current snapshot operation: control.(sh|bat) --snapshot status + Deletes snapshot and all its increments from all the online server nodes: + control.(sh|bat) --snapshot delete snapshot_name [--src path] + + Parameters: + snapshot_name - Snapshot name. + --src path - Path to the directory where the snapshot is located. If not specified, the default configured snapshot directory will be used. + Change cluster tag to new value: control.(sh|bat) --change-tag newTagValue [--yes]