From 4049e94e6d9d30bde100e0250a5b0426b2529984 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 1 Sep 2026 23:12:00 +0500 Subject: [PATCH 1/2] IGNITE-29026 Introduce extension points in compatibility testcontainers --- .../ru/IgniteRebalanceOnUpgradeTest.java | 37 ++- .../IgniteClusterContainer.java | 38 ++- .../testcontainers/IgniteContainer.java | 232 ++++++++++++------ 3 files changed, 219 insertions(+), 88 deletions(-) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java index e21ca5160f569..fb28ce19e61e1 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/ru/IgniteRebalanceOnUpgradeTest.java @@ -56,14 +56,14 @@ /** Smoke test for rolling upgrade with persistence. */ public class IgniteRebalanceOnUpgradeTest extends GridCommonAbstractTest { /** Consistent ID's. */ - private static final List CONSISTENT_IDS = List.of( + protected static final List CONSISTENT_IDS = List.of( "ad26bff6-5ff5-49f1-9a61-425a827953ed", "c1099d16-e7d7-49f4-925c-53329286c444", "7b880b69-8a9e-4b84-b555-250d365e2e67" ); /** Source image name, overridable via {@code -Dru.source.image.name}. */ - private static final String SOURCE_IMAGE_NAME = System.getProperty("ru.source.image.name"); + protected static final String SOURCE_IMAGE_NAME = System.getProperty("ru.source.image.name"); /** Upgrade mode. */ private static final UpgradeMode UPGRADE_MODE = UpgradeMode.valueOf(System.getProperty("ru.upgrade.mode", @@ -114,7 +114,7 @@ public static void afterClass() { /** Basic RU test. */ @Test public void testRollingUpgrade() throws Exception { - try (IgniteClusterContainer cluster = new IgniteClusterContainer(SOURCE_IMAGE_NAME, CONSISTENT_IDS)) { + try (IgniteClusterContainer cluster = cluster()) { cluster.start(); ClientCacheConfiguration cfg = new ClientCacheConfiguration() @@ -144,8 +144,13 @@ public void testRollingUpgrade() throws Exception { } } + /** @return Source cluster container. */ + protected IgniteClusterContainer cluster() throws Exception { + return new IgniteClusterContainer(SOURCE_IMAGE_NAME, CONSISTENT_IDS); + } + /** Verify data via local host-JVM nodes. */ - private void verifyViaLocalNodes() { + protected void verifyViaLocalNodes() { IgniteCache targetCache = nodes.get(0).cache(CACHE_NAME); for (int i = 0; i < 1000; i++) @@ -157,7 +162,7 @@ private void verifyViaLocalNodes() { } /** Verify data via thin client connected to upgraded Docker nodes. */ - private void verifyViaDockerNodes(IgniteClusterContainer cluster) { + protected void verifyViaDockerNodes(IgniteClusterContainer cluster) { IgniteContainer con = cluster.containers().get(0); con.checkNodeCount(cluster.containers().size()); @@ -173,7 +178,7 @@ private void verifyViaDockerNodes(IgniteClusterContainer cluster) { } /** */ - private void upgradeCluster(IgniteClusterContainer srcCluster) throws Exception { + protected void upgradeCluster(IgniteClusterContainer srcCluster) throws Exception { List srcContainers = srcCluster.containers(); if (UPGRADE_MODE == UpgradeMode.LOCAL) { @@ -194,7 +199,7 @@ private void upgradeCluster(IgniteClusterContainer srcCluster) throws Exception } /** Stop container, start a local host-JVM node with the same consistent ID. */ - private void upgradeLocally(IgniteContainer con, int idx) throws Exception { + protected void upgradeLocally(IgniteContainer con, int idx) throws Exception { // Address containers use to reach this (host JVM) node: // - Linux: Docker bridge gateway IP (e.g. 172.24.0.1) — always reachable from containers, // and the host can bind to it. The host's LAN IP is unreliable (on Debian/Ubuntu @@ -235,7 +240,7 @@ private void upgradeLocally(IgniteContainer con, int idx) throws Exception { } /** */ - private IgniteConfiguration configuration(String nodeId, String workDir, Collection addrs0, String ip, int idx) { + protected IgniteConfiguration configuration(String nodeId, String workDir, Collection addrs0, String ip, int idx) { DataRegionConfiguration dataRegionCfg = new DataRegionConfiguration() .setName("testRegion") .setInitialSize(1024L * 1024 * 1024) @@ -295,15 +300,23 @@ private IgniteConfiguration configuration(String nodeId, String workDir, Collect } /** */ - private IgniteClient client(String addr) { + protected IgniteClient client(String addr) { if (client == null) - client = Ignition.startClient(new ClientConfiguration().setAddresses(addr)); + client = Ignition.startClient(clientConfiguration(addr)); return client; } + /** + * @param addr Server address. + * @return Thin client configuration. + */ + protected ClientConfiguration clientConfiguration(String addr) { + return new ClientConfiguration().setAddresses(addr); + } + /** */ - private void closeClient() { + protected void closeClient() { if (client != null) { client.close(); @@ -312,7 +325,7 @@ private void closeClient() { } /** */ - private void stopLocalNodes() { + protected void stopLocalNodes() { for (IgniteEx node : nodes) { if (node != null) Ignition.stop(node.name(), false); diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java index ffeb0ad6c88aa..b7ec9218645d9 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.ignite.IgniteException; import org.testcontainers.containers.Network; import org.testcontainers.lifecycle.Startable; import org.testcontainers.lifecycle.Startables; @@ -30,21 +31,52 @@ public class IgniteClusterContainer implements Startable { private final List containers; /** Network. */ - private final Network net = Network.newNetwork(); + private final Network net; + + /** Image name. */ + private final String imageName; + + /** Consistent ID's. */ + private final List consistentIds; /** * @param imageName Image name. * @param consistentIds Consistent ID's. */ - public IgniteClusterContainer(String imageName, List consistentIds) throws Exception { + public IgniteClusterContainer(String imageName, List consistentIds) { + this.imageName = imageName; + this.consistentIds = consistentIds; + + net = Network.newNetwork(); containers = new ArrayList<>(consistentIds.size()); + } + /** + * @param imageName Image name. + * @param net Shared test network the container must be attached to. + * @param consistentIds Consistent ID's. + * @param idx Node index. + * @return The node container. + */ + protected IgniteContainer container(String imageName, Network net, List consistentIds, int idx) throws Exception { + return new IgniteContainer(imageName, net, "node" + (1 + idx), consistentIds.get(idx), idx); + } + + /** Builds the node containers. */ + protected void initContainers() throws Exception { for (int i = 0; i < consistentIds.size(); i++) - containers.add(new IgniteContainer(imageName, net, "node" + (1 + i), consistentIds.get(i), i)); + containers.add(container(imageName, net, consistentIds, i)); } /** {@inheritDoc} */ @Override public void start() { + try { + initContainers(); + } + catch (Exception e) { + throw new IgniteException(e); + } + Startables.deepStart(containers).join(); containers.get(0).activateCluster(containers.size()); diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java index c4de0cc88ae4b..e4d07910b85c3 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java @@ -22,7 +22,9 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; +import java.net.JarURLConnection; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; @@ -31,9 +33,12 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.jar.JarEntry; +import java.util.jar.JarFile; import java.util.jar.JarOutputStream; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -59,6 +64,7 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.containers.wait.strategy.WaitStrategy; import org.testcontainers.shaded.com.github.dockerjava.core.command.ExecStartResultCallback; import org.testcontainers.utility.DockerImageName; @@ -89,21 +95,6 @@ public class IgniteContainer extends GenericContainer { /** Logger. */ private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); - /** Ignite root directory in container. */ - private static final String ROOT_DIR_PATH = "/opt/ignite/apache-ignite/"; - - /** Ignite libs directory in container. */ - private static final String LIBS_DIR_PATH = ROOT_DIR_PATH + "libs/"; - - /** Ignite work directory in container. */ - private static final String WORK_DIR_PATH = ROOT_DIR_PATH + "work"; - - /** Config path in container. */ - private static final String CFG_PATH = ROOT_DIR_PATH + "config/test-config.xml"; - - /** Common config path in container. */ - private static final String COMMON_CFG_PATH = ROOT_DIR_PATH + "config/common-test-config.xml"; - /** */ private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); @@ -116,18 +107,10 @@ public class IgniteContainer extends GenericContainer { /** Base host port for the published thin-client port (node index added). */ private static final int CLIENT_HOST_PORT_BASE = 50800; - /** Custom classes (with their nested classes) used by node in containers. */ - private static final List TEST_CLASSES = List.of( - ContainerAddressResolver.class.getName(), - TestCompatibilityPluginProvider.class.getName(), - DisabledRollingUpgradeProcessor.class.getName(), - DisabledValidationProcessor.class.getName() - ); - /** Seconds to wait after SIGTERM before SIGKILL. */ private static final int SHUTDOWN_TIMEOUT_SEC = 30; - /** Jar holding {@link #TEST_CLASSES}, injected so the old image can load it. */ + /** Jar holding the {@link #testClasses() test classes}, injected so the old image can load it. */ private static volatile File testClassesJar; /** Cached tar archive of {@link #TARGET_LIBS_DIR} + test-classes.jar, built once and reused for all containers. */ @@ -142,29 +125,41 @@ public class IgniteContainer extends GenericContainer { /** Consistent ID. */ private final String consistentId; - /** Path to work directory. */ - private final String workDirPath; + /** Ignite root directory in container, computed from {@link #rootDirPath()}. */ + private final String rootDir; + + /** Ignite libs directory in container. */ + private final String libsDirPath; + + /** Config path in container. */ + private final String cfgPath; /** * @param imageName Image name. * @param net Network. * @param hostname Hostname. * @param consistentId Consistent ID. - * param idx Node index. + * @param idx Node index. */ public IgniteContainer(String imageName, Network net, String hostname, String consistentId, int idx) throws Exception { super(DockerImageName.parse(imageName)); this.hostname = hostname; this.consistentId = consistentId; - workDirPath = WORK_DIR_PATH + "/" + hostname; + rootDir = rootDirPath(); + libsDirPath = rootDir + "libs/"; + String workDirBase = rootDir + "work"; + cfgPath = rootDir + "config/test-config.xml"; int discoHostPort = DISCO_HOST_PORT_BASE + idx; int commHostPort = COMM_HOST_PORT_BASE + idx; - withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("CONFIG_URI", "file://" + cfgPath); + // Some entrypoints (e.g. bin/ignite.sh) resolve the config from DEFAULT_CONFIG rather than CONFIG_URI. + // Point it at the same config so both entrypoint styles load it; run.sh-based images ignore DEFAULT_CONFIG. + withEnv("DEFAULT_CONFIG", cfgPath); withEnv("IGNITE_QUIET", "false"); - withEnv("IGNITE_WORK_DIR", workDirPath); + withEnv("IGNITE_WORK_DIR", workDirBase + "/" + hostname); withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); withEnv("TZ", ZoneId.systemDefault().toString()); @@ -187,7 +182,7 @@ public IgniteContainer(String imageName, Network net, String hostname, String co if (!locWorkDir.exists()) locWorkDir.mkdirs(); - withFileSystemBind(LOCAL_WORK_DIR_PATH, WORK_DIR_PATH, BindMode.READ_WRITE); + withFileSystemBind(LOCAL_WORK_DIR_PATH, workDirBase, BindMode.READ_WRITE); // On Linux, run as the host user so bind-mounted directories (work dir, etc.) are owned by // the host user and can be cleaned up without root. Docker supports numeric UID:GID without @@ -200,9 +195,9 @@ public IgniteContainer(String imageName, Network net, String hostname, String co withCreateContainerCmdModifier(cmd -> cmd.withUser(uidGid)); } - withCopyFileToContainer(forClasspathResource("docker/common-test-config.xml"), COMMON_CFG_PATH); - withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); - withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), LIBS_DIR_PATH + "test-classes.jar"); + withCopyFileToContainer(forClasspathResource(commonConfigResource()), rootDir + "config/common-test-config.xml"); + withCopyFileToContainer(forClasspathResource(sourceConfigResource()), cfgPath); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), libsDirPath + "test-classes.jar"); withNetwork(net); withNetworkAliases(hostname); @@ -217,7 +212,12 @@ public IgniteContainer(String imageName, Network net, String hostname, String co addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); - waitingFor(Wait.forLogMessage(".*Node started.*", 1).withStartupTimeout(Duration.ofSeconds(600))); + waitingFor(waitStrategy()); + } + + /** @return Wait strategy for the node to become ready. */ + protected WaitStrategy waitStrategy() { + return Wait.forLogMessage(".*Node started.*", 1).withStartupTimeout(Duration.ofSeconds(600)); } /** {@inheritDoc} */ @@ -250,7 +250,7 @@ public void upgradeAndRestart() throws Exception { .withAttachStdout(true) .withAttachStderr(true) .withCmd("sh", "-c", - "rm -rf " + LIBS_DIR_PATH + "* && tar xf " + archivePathInContainer + " -C " + LIBS_DIR_PATH + "rm -rf " + libsDirPath + "* && tar xf " + archivePathInContainer + " -C " + libsDirPath + " && rm -f " + archivePathInContainer) .exec(); @@ -265,7 +265,7 @@ public void upgradeAndRestart() throws Exception { if (!Boolean.TRUE.equals(resp.isRunning()) && resp.getExitCodeLong() != null && resp.getExitCodeLong() != 0) throw new IllegalStateException("Failed to clean and extract libs: " + err); - copyFileToContainer(forClasspathResource("docker/target-test-config.xml"), CFG_PATH); + copyFileToContainer(forClasspathResource(targetConfigResource()), cfgPath); stopGraceful(); @@ -359,16 +359,12 @@ public String gatewayIp() { } /** */ - private String execControl(String... cmd) { - String[] fullCmd = new String[cmd.length + 1]; - - fullCmd[0] = ROOT_DIR_PATH + "bin/control.sh"; - - System.arraycopy(cmd, 0, fullCmd, 1, cmd.length); - + protected String execControl(String... cmd) { ExecResult result; try { + String[] fullCmd = command(cmd); + LOGGER.info("Running command: {}", Arrays.toString(fullCmd).replace(", ", " ")); result = execInContainer(fullCmd); @@ -383,8 +379,54 @@ private String execControl(String... cmd) { return result.getStdout(); } - /** @return Jar with {@link #TEST_CLASSES}, built once and reused for all containers. */ - private static File testClassesJar() throws IOException { + /** + * Builds the {@code control.sh} command line to be executed inside the container. + * + * @param cmd Control utility arguments (e.g. {@code --set-state ACTIVE --yes}). + * @return Full command whose first element is the absolute path to {@code control.sh}, followed by {@code cmd}. + */ + protected String[] command(String... cmd) { + String[] fullCmd = new String[cmd.length + 1]; + + fullCmd[0] = rootDir + "bin/control.sh"; + + System.arraycopy(cmd, 0, fullCmd, 1, cmd.length); + + return fullCmd; + } + + /** @return Classpath resource of the common (shared) node config copied into the container. */ + protected String commonConfigResource() { + return "docker/common-test-config.xml"; + } + + /** @return Classpath resource of the source (pre-upgrade) node config copied into the container. */ + protected String sourceConfigResource() { + return "docker/test-config.xml"; + } + + /** @return Classpath resource of the node config used on the target (upgraded) side during in-place Docker upgrade. */ + protected String targetConfigResource() { + return "docker/target-test-config.xml"; + } + + /** @return Ignite root directory inside the container, with a trailing slash. */ + protected String rootDirPath() { + return "/opt/ignite/apache-ignite/"; + } + + /** @return Custom classes (with their nested classes) used by the node in containers. */ + protected List testClasses() { + return List.of( + ContainerAddressResolver.class.getName(), + TestCompatibilityPluginProvider.class.getName(), + DisabledRollingUpgradeProcessor.class.getName(), + DisabledValidationProcessor.class.getName() + ); + } + + /** @return Jar with the {@link #testClasses() test classes}, built once and reused for all containers. */ + protected File testClassesJar() throws IOException { File jar = testClassesJar; if (jar != null) @@ -398,37 +440,23 @@ private static File testClassesJar() throws IOException { jar.deleteOnExit(); try (JarOutputStream out = new JarOutputStream(new FileOutputStream(jar))) { - for (String cls : TEST_CLASSES) { + for (String cls : testClasses()) { String clsPath = cls.replace('.', '/') + ".class"; - - URL url = IgniteContainer.class.getClassLoader().getResource(clsPath); - - if (url == null) - throw new IOException("Class not found on classpath: " + clsPath); - - File dir; - - try { - dir = new File(url.toURI()).getParentFile(); - } - catch (URISyntaxException e) { - throw new IOException(e); - } - String pkg = clsPath.substring(0, clsPath.lastIndexOf('/') + 1); String simple = cls.substring(cls.lastIndexOf('.') + 1); // Include the class and its nested classes (e.g. the provider's anonymous $1). - File[] clsFiles = dir.listFiles((d, name) -> - name.equals(simple + ".class") || name.startsWith(simple + '$')); + for (String resName : classResources(clsPath, pkg + simple + "$")) { + URL url = IgniteContainer.class.getClassLoader().getResource(resName); - if (clsFiles == null) - throw new IOException("Cannot list class directory: " + dir); + if (url == null) + throw new IOException("Class not found on classpath: " + resName); - for (File f : clsFiles) { - out.putNextEntry(new JarEntry(pkg + f.getName())); + out.putNextEntry(new JarEntry(resName)); - Files.copy(f.toPath(), out); + try (InputStream in = url.openStream()) { + in.transferTo(out); + } out.closeEntry(); } @@ -439,13 +467,71 @@ private static File testClassesJar() throws IOException { } } + /** + * Resolves the fully qualified resources (the top-level class plus its nested classes, e.g. {@code Outer$1}) + * for a class located either on the file system or inside a jar on the classpath. + * + * @param clsPath Resource path of the top-level class (package separator replaced with '/', ending in {@code .class}). + * @param nestedPrefix Package-based prefix of the nested classes, e.g. {@code org/apache/foo/Simple$}. + * @return Resource names of the class and its nested classes. + */ + private static Collection classResources(String clsPath, String nestedPrefix) throws IOException { + URL url = IgniteContainer.class.getClassLoader().getResource(clsPath); + + if (url == null) + throw new IOException("Class not found on classpath: " + clsPath); + + List res = new ArrayList<>(); + + res.add(clsPath); + + try { + if ("file".equals(url.getProtocol())) { + File dir = new File(url.toURI()).getParentFile(); + + String pkg = clsPath.substring(0, clsPath.lastIndexOf('/') + 1); + String simple = clsPath.substring(clsPath.lastIndexOf('/') + 1, clsPath.length() - ".class".length()); + + File[] clsFiles = dir.listFiles((d, name) -> + name.equals(simple + ".class") || name.startsWith(simple + '$')); + + if (clsFiles == null) + throw new IOException("Cannot list class directory: " + dir); + + for (File f : clsFiles) + res.add(pkg + f.getName()); + } + else if ("jar".equals(url.getProtocol())) { + JarURLConnection conn = (JarURLConnection)url.openConnection(); + + try (JarFile jar = conn.getJarFile()) { + Enumeration entries = jar.entries(); + + while (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + + if (name.startsWith(nestedPrefix) && name.endsWith(".class")) + res.add(name); + } + } + } + else + throw new IOException("Unsupported class resource protocol: " + url.getProtocol()); + } + catch (URISyntaxException e) { + throw new IOException(e); + } + + return res; + } + /** * Returns a cached tar archive (plain, no gzip) containing all files from {@link #TARGET_LIBS_DIR} * plus the test-classes jar. Built once and reused for all container upgrades. * * @return Path to the tar file on the host. */ - private static Path libsArchive() throws IOException { + protected Path libsArchive() throws IOException { Path archive = targetLibsArchive; if (archive != null) @@ -563,7 +649,7 @@ private static String hostUserUidGid() throws IOException, InterruptedException * (flush persistence, notify discovery neighbors, close socket connections) so that remaining nodes * don't trigger spurious "Failed to check connection to previous node" warnings during teardown.

*/ - private void stopGraceful() { + protected void stopGraceful() { if (!isRunning()) return; @@ -588,12 +674,12 @@ private void stopGraceful() { } /** @return Address the host JVM uses to reach this container's {@code port}. */ - private String address(int port) { + protected String address(int port) { return getHost() + ":" + getMappedPort(port); } /** @return This container's attachment to the single test Docker network. */ - private ContainerNetwork network() { + protected ContainerNetwork network() { return getContainerInfo().getNetworkSettings().getNetworks().values().iterator().next(); } } From 4db57df7e4fb493af9c2f3d703d9098bd27c2b83 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Fri, 11 Sep 2026 13:18:29 +0500 Subject: [PATCH 2/2] fix review comments --- .../IgniteClusterContainer.java | 29 +- .../testcontainers/IgniteContainer.java | 254 +++++++++++------- 2 files changed, 171 insertions(+), 112 deletions(-) diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java index b7ec9218645d9..16c6bf97192b3 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteClusterContainer.java @@ -31,13 +31,16 @@ public class IgniteClusterContainer implements Startable { private final List containers; /** Network. */ - private final Network net; + protected final Network net = Network.newNetwork(); /** Image name. */ - private final String imageName; + protected final String imageName; /** Consistent ID's. */ - private final List consistentIds; + protected final List consistentIds; + + /** Whether the cluster has been started, guarding against a second {@link #start()}. */ + private boolean started; /** * @param imageName Image name. @@ -47,29 +50,35 @@ public IgniteClusterContainer(String imageName, List consistentIds) { this.imageName = imageName; this.consistentIds = consistentIds; - net = Network.newNetwork(); containers = new ArrayList<>(consistentIds.size()); } /** - * @param imageName Image name. - * @param net Shared test network the container must be attached to. - * @param consistentIds Consistent ID's. + * Factory hook for the node container. Overrides only receive {@code idx}; the image name, network and + * consistent IDs are instance fields (see {@link #imageName}, {@link #net}, {@link #consistentIds}). + * * @param idx Node index. * @return The node container. */ - protected IgniteContainer container(String imageName, Network net, List consistentIds, int idx) throws Exception { + protected IgniteContainer container(int idx) throws Exception { return new IgniteContainer(imageName, net, "node" + (1 + idx), consistentIds.get(idx), idx); } /** Builds the node containers. */ protected void initContainers() throws Exception { for (int i = 0; i < consistentIds.size(); i++) - containers.add(container(imageName, net, consistentIds, i)); + containers.add(container(i)); } /** {@inheritDoc} */ @Override public void start() { + // Idempotent: either the cluster already started successfully, or container creation succeeded + // but startup (deepStart/activateCluster) failed on a previous attempt — in both cases the + // containers list is already populated and must not be built a second time (duplicate hostnames, + // consistent IDs and fixed host ports would make the baseline unreachable). + if (started || !containers.isEmpty()) + return; + try { initContainers(); } @@ -80,6 +89,8 @@ protected void initContainers() throws Exception { Startables.deepStart(containers).join(); containers.get(0).activateCluster(containers.size()); + + started = true; } /** {@inheritDoc} */ diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java index e4d07910b85c3..53f6f7605573a 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/testcontainers/IgniteContainer.java @@ -36,6 +36,8 @@ import java.util.Collection; import java.util.Enumeration; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -110,11 +112,11 @@ public class IgniteContainer extends GenericContainer { /** Seconds to wait after SIGTERM before SIGKILL. */ private static final int SHUTDOWN_TIMEOUT_SEC = 30; - /** Jar holding the {@link #testClasses() test classes}, injected so the old image can load it. */ - private static volatile File testClassesJar; + /** Jars for distinct {@link #testClasses() test classes} lists, injected so the old image can load them. */ + private static final Map, File> TEST_CLASSES_JARS = new ConcurrentHashMap<>(); - /** Cached tar archive of {@link #TARGET_LIBS_DIR} + test-classes.jar, built once and reused for all containers. */ - private static volatile Path targetLibsArchive; + /** Cached tar archives of {@link #TARGET_LIBS_DIR} + test-classes.jar, keyed by the test classes. */ + private static final Map, Path> TARGET_LIBS_ARCHIVES = new ConcurrentHashMap<>(); /** Cached "uid:gid" of the host user. */ private static volatile String hostUidGid; @@ -125,14 +127,17 @@ public class IgniteContainer extends GenericContainer { /** Consistent ID. */ private final String consistentId; + /** Node index, used for the fixed published host ports. */ + private final int idx; + /** Ignite root directory in container, computed from {@link #rootDirPath()}. */ - private final String rootDir; + private String rootDir; /** Ignite libs directory in container. */ - private final String libsDirPath; + private String libsDirPath; /** Config path in container. */ - private final String cfgPath; + private String cfgPath; /** * @param imageName Image name. @@ -146,73 +151,87 @@ public IgniteContainer(String imageName, Network net, String hostname, String co this.hostname = hostname; this.consistentId = consistentId; - rootDir = rootDirPath(); - libsDirPath = rootDir + "libs/"; - String workDirBase = rootDir + "work"; - cfgPath = rootDir + "config/test-config.xml"; - - int discoHostPort = DISCO_HOST_PORT_BASE + idx; - int commHostPort = COMM_HOST_PORT_BASE + idx; - - withEnv("CONFIG_URI", "file://" + cfgPath); - // Some entrypoints (e.g. bin/ignite.sh) resolve the config from DEFAULT_CONFIG rather than CONFIG_URI. - // Point it at the same config so both entrypoint styles load it; run.sh-based images ignore DEFAULT_CONFIG. - withEnv("DEFAULT_CONFIG", cfgPath); - withEnv("IGNITE_QUIET", "false"); - withEnv("IGNITE_WORK_DIR", workDirBase + "/" + hostname); - withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); - withEnv("TZ", ZoneId.systemDefault().toString()); - - // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host - // node, started with the same consistent id, inherits this node's persisted data. - String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; - - // Containers advertise published ports as external addresses via ContainerAddressResolver. - // This is needed on all platforms: - // - macOS/Windows: container-internal bridge IPs are not routable from the host - // - Linux: bridge IPs (172.x.x.x) may be blocked by host firewall (nftables/iptables) - // Published ports (127.0.0.1:5050x) are always reachable from the host via Docker port forwarding. - jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort - + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; - - withEnv("JVM_OPTS", jvmOpts); - - File locWorkDir = new File(LOCAL_WORK_DIR_PATH); - - if (!locWorkDir.exists()) - locWorkDir.mkdirs(); - - withFileSystemBind(LOCAL_WORK_DIR_PATH, workDirBase, BindMode.READ_WRITE); - - // On Linux, run as the host user so bind-mounted directories (work dir, etc.) are owned by - // the host user and can be cleaned up without root. Docker supports numeric UID:GID without - // the user existing in the container's /etc/passwd. - if (LINUX) { - String uidGid = hostUserUidGid(); - - LOGGER.info("Running container {} as host user uid/gid: {}", hostname, uidGid); - - withCreateContainerCmdModifier(cmd -> cmd.withUser(uidGid)); - } - - withCopyFileToContainer(forClasspathResource(commonConfigResource()), rootDir + "config/common-test-config.xml"); - withCopyFileToContainer(forClasspathResource(sourceConfigResource()), cfgPath); - withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), libsDirPath + "test-classes.jar"); - + this.idx = idx; + + // This constructor must NOT invoke the overridable hooks (rootDirPath(), *ConfigResource(), + // testClasses(), waitStrategy()): subclasses initialize their own instance state in their + // constructors, which run only after super(...) returns. Everything derived from those hooks + // is therefore deferred to configure(), which Testcontainers calls on start() — after every + // subclass constructor has finished. Only network membership and the fixed published ports + // (which depend solely on constructor parameters) are configured eagerly here. withNetwork(net); withNetworkAliases(hostname); withLogConsumer(frame -> System.out.println("[" + consistentId + "] " + frame.getUtf8String().trim())); - // Always publish fixed host ports so the host JVM node and thin client can reach each container at - // 127.0.0.1: via Testcontainers port forwarding. On Linux the bridge-internal IP (172.x) may - // be unreachable due to host firewall rules (firewalld/nftables) or Docker-in-VM setups (WSL2, - // VirtualBox), so published ports are the only reliable cross-platform approach. addFixedExposedPort(CLIENT_HOST_PORT_BASE + idx, ClientConnectorConfiguration.DFLT_PORT); - addFixedExposedPort(commHostPort, TcpCommunicationSpi.DFLT_PORT); - addFixedExposedPort(discoHostPort, TcpDiscoverySpi.DFLT_PORT); + addFixedExposedPort(COMM_HOST_PORT_BASE + idx, TcpCommunicationSpi.DFLT_PORT); + addFixedExposedPort(DISCO_HOST_PORT_BASE + idx, TcpDiscoverySpi.DFLT_PORT); + } - waitingFor(waitStrategy()); + /** {@inheritDoc} */ + @Override protected void configure() { + try { + // Resolved here, not in the constructor, so that the overridable hooks run only after every + // subclass constructor has initialized whatever instance state they may depend on. + rootDir = rootDirPath(); + libsDirPath = rootDir + "libs/"; + String workDirBase = rootDir + "work"; + cfgPath = rootDir + "config/test-config.xml"; + + int discoHostPort = DISCO_HOST_PORT_BASE + idx; + int commHostPort = COMM_HOST_PORT_BASE + idx; + + withEnv("CONFIG_URI", "file://" + cfgPath); + // Some entrypoints (e.g. bin/ignite.sh) resolve the config from DEFAULT_CONFIG rather than CONFIG_URI. + // Point it at the same config so both entrypoint styles load it; run.sh-based images ignore DEFAULT_CONFIG. + withEnv("DEFAULT_CONFIG", cfgPath); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_WORK_DIR", workDirBase + "/" + hostname); + withEnv("IGNITE_LOCAL_HOST", "0.0.0.0"); + withEnv("TZ", ZoneId.systemDefault().toString()); + + // node.consistent.id pins the node's consistent id (and thus its persistence folder) so the upgraded host + // node, started with the same consistent id, inherits this node's persisted data. + String jvmOpts = "-Xms512m -Xmx1g -Dnode.consistent.id=" + consistentId; + + // Containers advertise published ports as external addresses via ContainerAddressResolver. + // This is needed on all platforms: + // - macOS/Windows: container-internal bridge IPs are not routable from the host + // - Linux: bridge IPs (172.x.x.x) may be blocked by host firewall (nftables/iptables) + // Published ports (127.0.0.1:5050x) are always reachable from the host via Docker port forwarding. + jvmOpts += " -D" + EXT_ADDR_PROP_PREFIX + TcpDiscoverySpi.DFLT_PORT + "=127.0.0.1:" + discoHostPort + + " -D" + EXT_ADDR_PROP_PREFIX + TcpCommunicationSpi.DFLT_PORT + "=127.0.0.1:" + commHostPort; + + withEnv("JVM_OPTS", jvmOpts); + + File locWorkDir = new File(LOCAL_WORK_DIR_PATH); + + if (!locWorkDir.exists()) + locWorkDir.mkdirs(); + + withFileSystemBind(LOCAL_WORK_DIR_PATH, workDirBase, BindMode.READ_WRITE); + + // On Linux, run as the host user so bind-mounted directories (work dir, etc.) are owned by + // the host user and can be cleaned up without root. Docker supports numeric UID:GID without + // the user existing in the container's /etc/passwd. + if (LINUX) { + String uidGid = hostUserUidGid(); + + LOGGER.info("Running container {} as host user uid/gid: {}", hostname, uidGid); + + withCreateContainerCmdModifier(cmd -> cmd.withUser(uidGid)); + } + + withCopyFileToContainer(forClasspathResource(commonConfigResource()), rootDir + "config/common-test-config.xml"); + withCopyFileToContainer(forClasspathResource(sourceConfigResource()), cfgPath); + withCopyFileToContainer(forHostPath(testClassesJar().getAbsolutePath()), libsDirPath + "test-classes.jar"); + + waitingFor(waitStrategy()); + } + catch (IOException | InterruptedException e) { + throw new IgniteException("Failed to configure container " + hostname, e); + } } /** @return Wait strategy for the node to become ready. */ @@ -425,36 +444,40 @@ protected List testClasses() { ); } - /** @return Jar with the {@link #testClasses() test classes}, built once and reused for all containers. */ + /** + * @return Jar with the {@link #testClasses() test classes}, built per distinct class list and reused. + * The cache is keyed on the effective {@link #testClasses()} result so subclasses overriding it get + * their own jar instead of silently reusing the one built for the base class. + */ protected File testClassesJar() throws IOException { - File jar = testClassesJar; + // List.copyOf makes an immutable, value-comparable key. + List classes = List.copyOf(testClasses()); + + File jar = TEST_CLASSES_JARS.get(classes); if (jar != null) return jar; synchronized (IgniteContainer.class) { - if (testClassesJar != null) - return testClassesJar; + jar = TEST_CLASSES_JARS.get(classes); + + if (jar != null) + return jar; jar = File.createTempFile("test-classes", ".jar"); jar.deleteOnExit(); try (JarOutputStream out = new JarOutputStream(new FileOutputStream(jar))) { - for (String cls : testClasses()) { + for (String cls : classes) { String clsPath = cls.replace('.', '/') + ".class"; - String pkg = clsPath.substring(0, clsPath.lastIndexOf('/') + 1); - String simple = cls.substring(cls.lastIndexOf('.') + 1); // Include the class and its nested classes (e.g. the provider's anonymous $1). - for (String resName : classResources(clsPath, pkg + simple + "$")) { - URL url = IgniteContainer.class.getClassLoader().getResource(resName); + // Each entry carries the URL already resolved by classResources(), avoiding a second + // getResource() for the same top-level class (which would otherwise be hit twice). + for (ClassResource res : classResources(clsPath)) { + out.putNextEntry(new JarEntry(res.name)); - if (url == null) - throw new IOException("Class not found on classpath: " + resName); - - out.putNextEntry(new JarEntry(resName)); - - try (InputStream in = url.openStream()) { + try (InputStream in = res.url.openStream()) { in.transferTo(out); } @@ -463,43 +486,48 @@ protected File testClassesJar() throws IOException { } } - return testClassesJar = jar; + TEST_CLASSES_JARS.put(classes, jar); + + return jar; } } /** * Resolves the fully qualified resources (the top-level class plus its nested classes, e.g. {@code Outer$1}) - * for a class located either on the file system or inside a jar on the classpath. + * for a class located either on the file system or inside a jar on the classpath. Each returned element pairs + * the resource name with its already-resolved URL, so callers do not need to call {@code getResource()} again. * * @param clsPath Resource path of the top-level class (package separator replaced with '/', ending in {@code .class}). - * @param nestedPrefix Package-based prefix of the nested classes, e.g. {@code org/apache/foo/Simple$}. - * @return Resource names of the class and its nested classes. + * @return Pairs of resource name and resolved URL for the class and its nested classes. */ - private static Collection classResources(String clsPath, String nestedPrefix) throws IOException { - URL url = IgniteContainer.class.getClassLoader().getResource(clsPath); + private static Collection classResources(String clsPath) throws IOException { + String pkg = clsPath.substring(0, clsPath.lastIndexOf('/') + 1); + String simple = clsPath.substring(clsPath.lastIndexOf('/') + 1, clsPath.length() - ".class".length()); + String nestedPrefix = pkg + simple + "$"; + + ClassLoader cl = IgniteContainer.class.getClassLoader(); + + URL url = cl.getResource(clsPath); if (url == null) throw new IOException("Class not found on classpath: " + clsPath); - List res = new ArrayList<>(); - - res.add(clsPath); + List res = new ArrayList<>(); try { if ("file".equals(url.getProtocol())) { File dir = new File(url.toURI()).getParentFile(); - String pkg = clsPath.substring(0, clsPath.lastIndexOf('/') + 1); - String simple = clsPath.substring(clsPath.lastIndexOf('/') + 1, clsPath.length() - ".class".length()); - File[] clsFiles = dir.listFiles((d, name) -> - name.equals(simple + ".class") || name.startsWith(simple + '$')); + name.startsWith(simple + '$') && name.endsWith(".class")); if (clsFiles == null) throw new IOException("Cannot list class directory: " + dir); + res.add(new ClassResource(clsPath, url)); + for (File f : clsFiles) - res.add(pkg + f.getName()); + res.add(new ClassResource(pkg + f.getName(), cl.getResource(pkg + f.getName()))); } else if ("jar".equals(url.getProtocol())) { JarURLConnection conn = (JarURLConnection)url.openConnection(); @@ -510,8 +538,8 @@ else if ("jar".equals(url.getProtocol())) { while (entries.hasMoreElements()) { String name = entries.nextElement().getName(); - if (name.startsWith(nestedPrefix) && name.endsWith(".class")) - res.add(name); + if (name.equals(clsPath) || (name.startsWith(nestedPrefix) && name.endsWith(".class"))) + res.add(new ClassResource(name, cl.getResource(name))); } } } @@ -525,21 +553,39 @@ else if ("jar".equals(url.getProtocol())) { return res; } + /** A class resource: its name on the classpath paired with the already-resolved URL. */ + private static final class ClassResource { + /** Resource name on the classpath. */ + final String name; + + /** Resolved URL of the resource. */ + final URL url; + + /** @param name Resource name on the classpath. */ + ClassResource(String name, URL url) { + this.name = name; + this.url = url; + } + } + /** * Returns a cached tar archive (plain, no gzip) containing all files from {@link #TARGET_LIBS_DIR} - * plus the test-classes jar. Built once and reused for all container upgrades. + * plus the test-classes jar. Cached per distinct {@link #testClasses()} list, since the archive embeds + * {@link #testClassesJar()} whose content depends on it. * * @return Path to the tar file on the host. */ protected Path libsArchive() throws IOException { - Path archive = targetLibsArchive; + List classes = List.copyOf(testClasses()); + + Path archive = TARGET_LIBS_ARCHIVES.get(classes); if (archive != null) return archive; synchronized (IgniteContainer.class) { - if (targetLibsArchive != null) - return targetLibsArchive; + if (TARGET_LIBS_ARCHIVES.containsKey(classes)) + return TARGET_LIBS_ARCHIVES.get(classes); File targetLibsFile = TARGET_LIBS_DIR.toFile(); @@ -584,7 +630,9 @@ protected Path libsArchive() throws IOException { LOGGER.info("Libs archive built: {} ({} bytes)", archiveFile, archiveFile.length()); - return targetLibsArchive = archiveFile.toPath(); + TARGET_LIBS_ARCHIVES.put(classes, archiveFile.toPath()); + + return archiveFile.toPath(); } }