diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java index 19b0a4a3f750c..8857a4ddd656e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java @@ -96,6 +96,7 @@ import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteFutureCancelledException; +import org.apache.ignite.lang.IgniteFutureTimeoutException; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -903,6 +904,18 @@ private void checkIncrementalSnapshotWalRecords(IgniteEx node, IncrementalSnapsh } } + /** Print thread dump if {@code IgniteFutureTimeoutException} is raised. */ + protected void runWithLoggedThreadDump(Runnable action) { + try { + action.run(); + } + catch (IgniteFutureTimeoutException ex) { + U.dumpThreads(log); + + throw ex; + } + } + /** * @param ignite Ignite instance to resolve discovery spi to. * @return BlockingCustomMessageDiscoverySpi instance. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java index 7b679de89ad4f..830aa605ef549 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotHandlerTest.java @@ -61,6 +61,9 @@ public class IgniteClusterSnapshotHandlerTest extends IgniteClusterSnapshotResto /** Custom snapshot handlers. */ private final List> handlers = new ArrayList<>(); + /** Timeout in milliseconds to await for snapshot operation being completed. */ + protected static final long TIMEOUT = 60_000; + /** Extensions plugin provider. */ private final PluginProvider pluginProvider = new AbstractTestPluginProvider() { @Override public String name() { @@ -84,7 +87,7 @@ public class IgniteClusterSnapshotHandlerTest extends IgniteClusterSnapshotResto /** {@inheritDoc} */ @Override protected Function valueBuilder() { - return Integer::new; + return Integer::valueOf; } /** @@ -142,11 +145,13 @@ public void testClusterSnapshotHandlers() throws Exception { IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); - GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); + runWithLoggedThreadDump(() -> + GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg)); changeMetadataRequestIdOnDisk(reqIdRef.get()); - ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT); + runWithLoggedThreadDump(() -> + ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT)); assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE); } @@ -212,7 +217,8 @@ public void testClusterSnapshotHandlerFailure() throws Exception { IgniteFuture fut = snp(ignite).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary); - GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); + runWithLoggedThreadDump(() -> + GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg)); failCreateFlag.set(false); @@ -224,11 +230,13 @@ public void testClusterSnapshotHandlerFailure() throws Exception { IgniteFuture fut0 = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); - GridTestUtils.assertThrowsAnyCause(log, () -> fut0.get(TIMEOUT), IgniteCheckedException.class, expMsg); + runWithLoggedThreadDump(() -> + GridTestUtils.assertThrowsAnyCause(log, () -> fut0.get(TIMEOUT), IgniteCheckedException.class, expMsg)); failRestoreFlag.set(false); - ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT); + runWithLoggedThreadDump(() -> + ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT)); assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE); } @@ -406,7 +414,8 @@ public void testHandlerSnapshotLocation() throws Exception { ignite.destroyCache(DEFAULT_CACHE_NAME); awaitPartitionMapExchange(); - snpMgr.restoreSnapshot(snpName, snpDir.getAbsolutePath(), null).get(TIMEOUT); + runWithLoggedThreadDump(() -> + snpMgr.restoreSnapshot(snpName, snpDir.getAbsolutePath(), null).get(TIMEOUT)); } finally { U.delete(snpDir); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 409c1fb13cbd8..78ec729ec931b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -95,6 +95,9 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR /** Reset consistent ID flag. */ private boolean resetConsistentId; + /** Timeout in milliseconds to await for snapshot operation being completed. */ + protected static final long TIMEOUT = 60_000; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -117,7 +120,8 @@ public void testRestoreWithEmptyPartitions() throws Exception { // Skip check because some partitions will be empty - keysCnt == parts/2. Ignite ignite = startGridsWithSnapshot(1, keysCnt, false, true); - ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT); + runWithLoggedThreadDump(() -> + ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT)); assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), keysCnt); } @@ -235,7 +239,8 @@ private void doRestoreAllGroups() throws Exception { TestRecordingCommunicationSpi.spi(g).record(SnapshotFilesRequestMessage.class); // Restore all cache groups. - grid(0).snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT); + runWithLoggedThreadDump(() -> + grid(0).snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(TIMEOUT)); awaitPartitionMapExchange(true, true, null, true); @@ -277,8 +282,9 @@ private void checkStartClusterSnapshotRestoreMultithreaded(IntSupplier nodeIdxSu IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> { try { - grid(nodeIdxSupplier.getAsInt()).snapshot().restoreSnapshot( - SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT); + runWithLoggedThreadDump(() -> + grid(nodeIdxSupplier.getAsInt()).snapshot().restoreSnapshot( + SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT)); successCnt.incrementAndGet(); } @@ -444,7 +450,8 @@ public void testClusterSnapshotRestoreOnSmallerTopology() throws Exception { resetBaselineTopology(); - grid(0).snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT); + runWithLoggedThreadDump(() -> + grid(0).snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT)); assertCacheKeys(grid(0).cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE); waitForEvents(EVT_CLUSTER_SNAPSHOT_RESTORE_STARTED, EVT_CLUSTER_SNAPSHOT_RESTORE_FINISHED); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java index d1a804ada61c7..cf359d313ef52 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotMXBeanTest.java @@ -53,6 +53,9 @@ public class IgniteSnapshotMXBeanTest extends AbstractSnapshotSelfTest { /** Snapshot group name. */ private static final String SNAPSHOT_GROUP = "Snapshot"; + /** Timeout in milliseconds to await for snapshot operation being completed. */ + protected static final long TIMEOUT = 60_000; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { return super.getConfiguration(igniteInstanceName) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java index c7f1316077635..f819ba3925542 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckWithIndexesTest.java @@ -36,6 +36,9 @@ * Cluster-wide snapshot test check command with indexes. */ public class IgniteClusterSnapshotCheckWithIndexesTest extends AbstractSnapshotSelfTest { + /** Timeout in milliseconds to await for snapshot operation being completed. */ + protected static final long TIMEOUT = 60_000; + /** @throws Exception If fails. */ @Test public void testClusterSnapshotCheckEmptyCache() throws Exception { @@ -87,7 +90,9 @@ public void testClusterSnapshotCheckWithNodeFilter() throws Exception { cache2.put(i, new Account(i, i)); } - createAndCheckSnapshot(grid(0), SNAPSHOT_NAME, null, TIMEOUT); + runWithLoggedThreadDump(() -> + createAndCheckSnapshot(grid(0), SNAPSHOT_NAME, null, TIMEOUT) + ); IdleVerifyResult res = grid(0).context().cache().context().snapshotMgr() .checkSnapshot(SNAPSHOT_NAME, null).get().idleVerifyResult(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java index b8ebe10a7d202..9ccda9c73b614 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreWithIndexingTest.java @@ -57,6 +57,9 @@ public class IgniteClusterSnapshotRestoreWithIndexingTest extends IgniteClusterS /** Number of cache keys to pre-create at node start. */ private static final int CACHE_KEYS_RANGE = 10_000; + /** Timeout in milliseconds to await for snapshot operation being completed. */ + protected static final long TIMEOUT = 60_000; + /** {@inheritDoc} */ @Override protected CacheConfiguration txCacheConfig(CacheConfiguration ccfg) { return super.txCacheConfig(ccfg).setSqlIndexMaxInlineSize(255).setSqlSchema("PUBLIC") @@ -74,7 +77,8 @@ public void testBasicClusterSnapshotRestore() throws Exception { IgniteEx client = startGridsWithSnapshot(2, CACHE_KEYS_RANGE, true); - grid(0).snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT); + runWithLoggedThreadDump(() -> + grid(0).snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT)); // Only primary mode leads to index rebuild on restore. // Must wait until index rebuild finish so subsequent checks will pass. @@ -101,7 +105,8 @@ public void testBasicClusterSnapshotRestoreWithMetadata() throws Exception { forceCheckpoint(); - ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT); + runWithLoggedThreadDump(() -> + ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT)); // Only primary mode leads to index rebuild on restore. // Must wait until index rebuild finish so subsequent checks will pass. @@ -126,7 +131,8 @@ public void testClusterSnapshotRestoreOnBiggerTopology() throws Exception { startGridsWithCache(nodesCnt - 2, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg); - grid(0).snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT); + runWithLoggedThreadDump(() -> + grid(0).snapshot().createSnapshot(SNAPSHOT_NAME).get(TIMEOUT)); startGrid(nodesCnt - 2); @@ -152,8 +158,8 @@ public void testClusterSnapshotRestoreOnBiggerTopology() throws Exception { forceCheckpoint(); // Restore from an empty node. - ignite.snapshot().restoreSnapshot( - SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT); + runWithLoggedThreadDump(() -> ignite.snapshot().restoreSnapshot( + SNAPSHOT_NAME, Collections.singleton(DEFAULT_CACHE_NAME)).get(TIMEOUT)); awaitPartitionMapExchange();