diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 1ceb22c9eefa9..69e317f1ea8d9 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -181,7 +181,9 @@ jobs: uses: actions/setup-java@v5 with: distribution: ${{ env.JDK_DISTRIBUTION }} - java-version: ${{ env.CI_JDK_MAJOR_VERSION }} + java-version: | + 17 + ${{ env.CI_JDK_MAJOR_VERSION }} - name: Setup Gradle uses: ./.github/actions/setup-gradle @@ -194,6 +196,9 @@ jobs: ./gradlew assemble rat spotlessCheck checkstyleMain checkstyleTest --no-configuration-cache + - name: Verify client and Functions API Java compatibility + run: ./gradlew :tests:pulsar-client-java-compatibility:test -PtestRetryCount=0 + - name: Check binary licenses run: ./gradlew checkBinaryLicense --no-configuration-cache diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f446bae4c1238..41c0f4fa2c95f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -94,8 +94,9 @@ concurrency model. ## Build infrastructure Apache Pulsar uses a **Gradle** build (migrated from Maven via PIP-463; some older tooling and docs -elsewhere still reference Maven). The wrapper `./gradlew` requires **JDK 21, 25 or 26** (bytecode targets -Java 17). See [`CONTRIBUTING.md` → Building](CONTRIBUTING.md#building) for the build and lint commands. +elsewhere still reference Maven). The wrapper `./gradlew` requires **JDK 21, 25 or 26** (server bytecode +targets Java 21; client/API bytecode targets Java 17). See +[`CONTRIBUTING.md` → Building](CONTRIBUTING.md#building) for the build and lint commands. - `settings.gradle.kts` — all modules, organized in dependency tiers (Tier 0 has no internal deps, higher tiers build on lower ones). @@ -122,6 +123,28 @@ preserved. Most importantly: Always use the Gradle project path (left of any `--tests`), e.g. `./gradlew :pulsar-client-original:test`. Check `settings.gradle.kts` when a path is ambiguous. +### Java compatibility boundaries + +`pulsar.java-conventions` defaults main sources to Java 21 and explicitly lists the Java 17 client +and public API dependency closure. This includes both client generations, admin/auth/crypto clients, +TLS/HTTP SPIs, shared common/package APIs, and the Functions/IO interfaces. Functions implementations, +brokers, and other server components target Java 21. Client CLI tools also remain Java 17 compatible. +`pulsarJavaVersion` and `pulsarClientJavaVersion` control these targets (defaults 21 and 17). +Test sources default to `pulsarJavaVersion` so client tests can use server fixtures; the dedicated +consumer tests use `pulsarClientJavaVersion` for compilation and their runtime toolchain. +The bytecode check and published JVM metadata follow the configured client target. + +When adding a client dependency, keep its full compile/runtime closure Java 17 compatible. JVM +variant attributes reject Java 21 project dependencies; `verifyClientJavaCompatibility` also checks +class-file versions of dependencies without Gradle metadata and the final shaded client jars. +Multi-release jars are checked using the entries selected by Java 17. This verifies bytecode, not +all possible reflective or JDK API usage in third-party libraries; the Java 17 consumer tests provide +runtime coverage. Do not mark a server implementation as Java 17 just to bypass a dependency error. + +The client fastutil minimizer reads CLI classes as build-only reachability roots. Its published jar has no +transitive project dependencies and is checked as Java 17. +`buildtools` and `testmocks` also stay Java 17 so the consumer compatibility tests can load them. + ### Changing the build When editing `build-logic/`, `settings.gradle.kts`, a module `build.gradle.kts`, `gradle.properties`, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b569a8ad04e6..85b14d111cc24 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,8 +29,39 @@ workflow (build, test, PR, CI). For the big-picture module map and the Gradle bu ## Building -**JDK 21, 25 or 26** is required to build `master` (bytecode targets Java 17; `-PskipJavaVersionCheck` -bypasses the check); `zip` is also needed. Use the bundled wrapper `./gradlew` (Linux/macOS) or +Standard Pulsar 5 server components and Functions implementations require Java 21 or later. +Client CLI tools remain Java 17 compatible. +Client libraries (including V5), their shared dependencies, and Functions/IO public interfaces +remain Java 17 compatible. Functions compiled on Java 17 can run in a Java 21+ Functions instance. +The build uses `--release` and publishes the corresponding JVM requirement in Gradle metadata. + +`assemble` checks Java 17 client/API bytecode and its compile/runtime dependencies, including shaded +jars. Run `./gradlew :tests:pulsar-client-java-compatibility:test -PtestRetryCount=0` to compile and run +consumer examples on an installed JDK 17. Ordinary tests target Java 21 because even client tests +can depend on broker/Functions fixtures. `-PtestJavaVersion=17` is only suitable for test modules +whose entire test dependency graph supports Java 17; it does not lower the server baseline. + +For custom builds, `-PpulsarJavaVersion=17` targets Java 17 for server code and ordinary test +sources. `-PpulsarClientJavaVersion=17` controls the client/public API target and consumer test +toolchain (17 is already the default). For example: + +```shell +./gradlew assemble -PpulsarJavaVersion=17 +PULSAR_MIN_JAVA_VERSION=17 bin/pulsar standalone +``` + +Gradle Docker builds bake `pulsarJavaVersion` into the image as `PULSAR_MIN_JAVA_VERSION`. +Direct Docker builds can set `--build-arg PULSAR_MIN_JAVA_VERSION=17`; both Alpine and Wolfi +Dockerfiles default to 21. The environment variable can also be overridden when running a container. + +`PULSAR_MIN_JAVA_VERSION` overrides the launcher check, which defaults to 21; it does not change +compiled bytecode or dependency requirements. A custom Java 17 build is only possible while the +sources and dependencies remain compatible; use of Java 21 features such as virtual threads will +prevent targeting Java 17. Java 17 server builds are not part of CI. These properties do not change +Gradle's build-JDK requirement. Use `-PtestJavaVersion=17` as well to run ordinary tests on an installed Java 17 JDK. + +**JDK 21, 25 or 26** is required to build `master` (`-PskipJavaVersionCheck` bypasses the check); +`zip` is also needed. Use the bundled wrapper `./gradlew` (Linux/macOS) or `gradlew.bat` (Windows) — no separate Gradle install. See the [build-tooling setup guide](https://pulsar.apache.org/contribute/setup-buildtools/) and the [IDE setup guide](https://pulsar.apache.org/contribute/setup-ide/). @@ -106,7 +137,7 @@ entire group. CI splits `pulsar-broker` tests into groups (see group are treated as `other` at runtime. `./gradlew verifyTestGroups` reports group assignments and flags tests not covered by any CI group. -Other test-related properties: `-PtestJavaVersion=17` (run tests on a different JDK toolchain), +Other test-related properties: `-PtestJavaVersion=21` (compile and run tests for a different JDK toolchain), `-PtestRetryCount=N`, `-PtestFailFast=true|false`, `-PprotobufVersion=4.31.1` (protobuf v4 compatibility tests). diff --git a/bin/pulsar b/bin/pulsar index d1afacee1a776..854bf3995a5e9 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -85,6 +85,7 @@ Environment variables: PULSAR_PROXY_CONF Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF) PULSAR_WORKER_CONF Configuration file for functions worker (default: $DEFAULT_WORKER_CONF) PULSAR_STANDALONE_CONF Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF) + PULSAR_MIN_JAVA_VERSION Minimum Java version accepted by the launcher (default: 21) PULSAR_EXTRA_OPTS Extra options to be passed to the jvm PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath PULSAR_PID_DIR Folder where the pulsar server PID file should be stored @@ -147,8 +148,9 @@ if [[ -z $JAVA_MAJOR_VERSION ]]; then done fi -if [[ $JAVA_MAJOR_VERSION -lt 17 ]]; then - echo "Error: Pulsar requires Java 17 or later." 1>&2 +PULSAR_MIN_JAVA_VERSION=${PULSAR_MIN_JAVA_VERSION:-21} +if [[ $JAVA_MAJOR_VERSION -lt $PULSAR_MIN_JAVA_VERSION ]]; then + echo "Error: Pulsar server commands require Java $PULSAR_MIN_JAVA_VERSION or later." 1>&2 exit 1 fi diff --git a/build-logic/conventions/src/main/kotlin/VerifyJavaCompatibility.kt b/build-logic/conventions/src/main/kotlin/VerifyJavaCompatibility.kt new file mode 100644 index 0000000000000..051125365cd26 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/VerifyJavaCompatibility.kt @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Classpath +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction +import org.gradle.work.DisableCachingByDefault +import java.io.DataInputStream +import java.io.InputStream +import java.util.jar.JarFile + +/** Checks bytecode as well as Gradle's JVM attributes (many Maven jars publish no JVM metadata). */ +@DisableCachingByDefault(because = "Verification has no outputs") +abstract class VerifyJavaCompatibility : DefaultTask() { + @get:Classpath + abstract val classpath: ConfigurableFileCollection + + @get:Input + abstract val javaVersion: Property + + @TaskAction + fun verify() { + val target = javaVersion.get() + val failures = mutableListOf() + fun inspect(name: String, stream: InputStream) { + DataInputStream(stream).use { input -> + if (input.readInt() != 0xCAFEBABE.toInt()) { + throw GradleException("Invalid class file: $name") + } + val minor = input.readUnsignedShort() + val major = input.readUnsignedShort() + if (major > target + 44 || minor == 65535) { + failures.add("$name requires Java ${major - 44}" + if (minor == 65535) " preview" else "") + } + } + } + for (file in classpath.files) { + if (file.isDirectory) { + file.walkTopDown().filter { it.isFile && it.extension == "class" }.forEach { + inspect(it.path, it.inputStream()) + } + } else if (file.extension == "jar") { + JarFile(file).use { jar -> + val multiRelease = jar.manifest?.mainAttributes?.getValue("Multi-Release") == "true" + // Select the same class entries as the target JVM, not classes for newer JVMs. + val selected = mutableMapOf>() + for (entry in jar.entries()) { + if (!entry.name.endsWith(".class")) continue + val match = Regex("META-INF/versions/([0-9]+)/(.*)").matchEntire(entry.name) + val version = match?.groupValues?.get(1)?.toInt() ?: 0 + if (match != null && (!multiRelease || version > target)) continue + val name = match?.groupValues?.get(2) ?: entry.name + if (version >= (selected[name]?.first ?: -1)) selected[name] = version to entry + } + selected.values.forEach { (_, entry) -> + inspect("${file.name}!/${entry.name}", jar.getInputStream(entry)) + } + } + } + } + if (failures.isNotEmpty()) { + throw GradleException("Java $target compatibility violated:\n" + failures.take(30).joinToString("\n") + + if (failures.size > 30) "\n... ${failures.size} incompatible classes in total" else "") + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar.client-shade-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.client-shade-conventions.gradle.kts index 03bd711eaa5b1..b68fa12761186 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.client-shade-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.client-shade-conventions.gradle.kts @@ -22,6 +22,7 @@ import org.gradle.api.attributes.Bundling import org.gradle.api.attributes.Category import org.gradle.api.attributes.LibraryElements import org.gradle.api.attributes.Usage +import org.gradle.api.attributes.java.TargetJvmVersion import org.gradle.api.component.AdhocComponentWithVariants import org.gradle.api.tasks.PathSensitivity import java.util.zip.ZipFile @@ -37,6 +38,7 @@ plugins { val shadePrefix = "org.apache.pulsar.shade" extra["shadePrefix"] = shadePrefix +val targetJavaVersion = extensions.getByType().targetCompatibility.majorVersion.toInt() // ---- Published dependency scopes for non-bundled dependencies ---- // The Shadow plugin publishes the `shadow` configuration's dependencies as the dependency-reduced @@ -66,6 +68,7 @@ val shadowApiElements = configurations.consumable("shadowApiElements") { attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category.LIBRARY)) attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, objects.named(LibraryElements.JAR)) attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling.SHADOWED)) + attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, targetJavaVersion) } // Carry the shaded jar so this variant is a complete API variant (like apiElements does for the // standard java-library component). diff --git a/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts index 885be271cf81d..2d64944edbe0e 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts @@ -18,6 +18,7 @@ */ import java.io.File +import org.gradle.api.attributes.java.TargetJvmVersion plugins { `java-library` @@ -65,12 +66,72 @@ configurations.matching { it.name in platformAlignedClasspaths }.configureEach { extendsFrom(internalPlatform) } +// Java 17 is a compatibility promise for client libraries and user-written Functions/IO APIs. +// Keep this list explicit: a new server module must not silently lower its baseline, and a new +// client dependency must be reviewed before joining the Java 17 dependency closure. Gradle's JVM +// attributes reject project dependencies from this group onto Java 21 modules. +val clientProjects = setOf( + ":pulsar-client-api", ":pulsar-client-api-v5", ":pulsar-client-admin-api", + ":pulsar-tls-factory-api", ":pulsar-http-client-api", ":pulsar-common", + ":pulsar-client-original", ":pulsar-client-v5", ":pulsar-client-admin-original", + ":pulsar-client-auth-athenz", ":pulsar-client-auth-sasl", ":pulsar-client-messagecrypto-bc", + ":pulsar-client-shaded", ":pulsar-client-all", ":pulsar-client-admin-shaded", + ":pulsar-client-v5-shaded", ":pulsar-client-v5-all", ":pulsar-client-fastutil-minimized", + ":pulsar-client-tools-api", ":pulsar-client-tools", ":pulsar-client-tools-test", + ":pulsar-client-tools-customcommand-example", ":pulsar-cli-utils", + ":pulsar-package-management:pulsar-package-core", + ":pulsar-functions:pulsar-functions-api", ":pulsar-io:pulsar-io-core", + // Test support must also load in the Java 17 consumer compatibility test JVM. + ":buildtools", ":testmocks", ":tests:pulsar-client-java-compatibility", +) +val pulsarJavaVersion = providers.gradleProperty("pulsarJavaVersion").map { it.toInt() }.orElse(21) +val pulsarClientJavaVersion = providers.gradleProperty("pulsarClientJavaVersion").map { it.toInt() }.orElse(17) +val mainJavaVersion = if (path in clientProjects) pulsarClientJavaVersion.get() else pulsarJavaVersion.get() +// Client tests can embed the broker and Functions implementation. Test bytecode and dependency +// resolution therefore have their own baseline, independent of the published main artifact. +val testJavaVersion = if (path == ":tests:pulsar-client-java-compatibility") { + pulsarClientJavaVersion +} else { + providers.gradleProperty("testJavaVersion").map { it.toInt() } +} +val testRelease = testJavaVersion.getOrElse(pulsarJavaVersion.get()) +if (path == ":tests:pulsar-client-java-compatibility") { + tasks.withType().configureEach { + systemProperty("pulsarClientJavaVersion", pulsarClientJavaVersion.get()) + } +} +java { + sourceCompatibility = JavaVersion.toVersion(mainJavaVersion) + targetCompatibility = JavaVersion.toVersion(mainJavaVersion) +} +configurations.matching { it.name in setOf("testCompileClasspath", "testRuntimeClasspath") }.configureEach { + // Follow explicit module overrides too (for example the Java 21 performance tools). + attributes.attributeProvider(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, + tasks.named("compileTestJava").flatMap { it.options.release }) +} + tasks.withType().configureEach { options.encoding = "UTF-8" - options.release.set(17) + options.release.set(mainJavaVersion) options.compilerArgs.addAll(listOf("-parameters", "-Xlint:deprecation", "-Xlint:unchecked")) } +tasks.named("compileTestJava") { + options.release.set(testRelease) +} + +if (path in clientProjects) { + val verifyClientJavaCompatibility = tasks.register("verifyClientJavaCompatibility") { + group = "verification" + description = "Check client/API classes and dependencies against pulsarClientJavaVersion." + javaVersion.set(pulsarClientJavaVersion) + classpath.from(sourceSets.main.get().output.classesDirs, + configurations.named("compileClasspath"), configurations.named("runtimeClasspath")) + } + tasks.named("check") { dependsOn(verifyClientJavaCompatibility) } + tasks.named("assemble") { dependsOn(verifyClientJavaCompatibility) } +} + configurations.all { // Force Jackson version to match the version catalog. Transitive dependencies // (e.g. from jackson-bom) can pull in newer versions that break API compatibility @@ -199,8 +260,7 @@ dependencies { "testRuntimeOnly"(catalog.findLibrary("log4j-jul").get()) } -// Allow overriding the JDK used for running tests via -PtestJavaVersion=17 -val testJavaVersion = providers.gradleProperty("testJavaVersion").map { it.toInt() } +// Allow overriding the JDK used for running tests via -PtestJavaVersion=17. val javaToolchains = extensions.getByType() // Effective Java major version used to run tests: the -PtestJavaVersion override when set, // otherwise the JVM running Gradle. diff --git a/build-logic/conventions/src/main/kotlin/pulsar.minimized-dependencies-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.minimized-dependencies-conventions.gradle.kts index 9a91e6181ea9b..d741cd3756bb7 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.minimized-dependencies-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.minimized-dependencies-conventions.gradle.kts @@ -91,3 +91,8 @@ val verifyMinimizedJar = tasks.register("verifyMinimizedJar") { tasks.named("check") { dependsOn(verifyMinimizedJar) } + +// Verify only the published classes: reachability roots are not bundled in this artifact. +tasks.withType().configureEach { + classpath.setFrom(tasks.named("shadowJar").flatMap { it.archiveFile }) +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar.shadow-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.shadow-conventions.gradle.kts index 9864c439d630b..0dc338ffab099 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.shadow-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.shadow-conventions.gradle.kts @@ -72,3 +72,9 @@ configurations { } } } + +// Check the actual shaded contents too, including relocated classes and multi-release entries. +tasks.withType().configureEach { + classpath.from(tasks.named("shadowJar") + .flatMap { it.archiveFile }) +} diff --git a/build-logic/conventions/src/test/kotlin/VerifyJavaCompatibilityTest.kt b/build-logic/conventions/src/test/kotlin/VerifyJavaCompatibilityTest.kt new file mode 100644 index 0000000000000..9a04af630e7da --- /dev/null +++ b/build-logic/conventions/src/test/kotlin/VerifyJavaCompatibilityTest.kt @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.GradleException +import org.gradle.testfixtures.ProjectBuilder +import org.testng.Assert.assertTrue +import org.testng.Assert.expectThrows +import org.testng.annotations.Test +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream +import java.nio.file.Files +import java.util.jar.Attributes +import java.util.jar.JarEntry +import java.util.jar.JarOutputStream +import java.util.jar.Manifest + +class VerifyJavaCompatibilityTest { + private fun verify(vararg entries: Pair, multiRelease: Boolean = false) { + val directory = Files.createTempDirectory("java-compatibility-test").toFile() + try { + val manifest = Manifest().apply { + mainAttributes[Attributes.Name.MANIFEST_VERSION] = "1.0" + if (multiRelease) mainAttributes.putValue("Multi-Release", "true") + } + val jar = directory.resolve("dependency.jar") + JarOutputStream(jar.outputStream(), manifest).use { output -> + for ((name, javaVersion) in entries) { + output.putNextEntry(JarEntry(name)) + val bytes = ByteArrayOutputStream() + DataOutputStream(bytes).use { + it.writeInt(0xCAFEBABE.toInt()) + it.writeShort(0) + it.writeShort(javaVersion + 44) + } + output.write(bytes.toByteArray()) + output.closeEntry() + } + } + val project = ProjectBuilder.builder().withProjectDir(directory).build() + val task = project.tasks.register("verify", VerifyJavaCompatibility::class.java).get() + task.javaVersion.set(17) + task.classpath.from(jar) + task.verify() + } finally { + directory.deleteRecursively() + } + } + + @Test + fun rejectsJava21DependencyWithoutGradleMetadata() { + val error = expectThrows(GradleException::class.java) { verify("Library.class" to 21) } + assertTrue(error.message!!.contains("Library.class requires Java 21")) + } + + @Test + fun acceptsJava17WithNewerMultiReleaseImplementations() { + verify("Library.class" to 17, "META-INF/versions/21/Library.class" to 21, multiRelease = true) + } + + @Test + fun rejectsIncompatibleSelectedMultiReleaseEntry() { + expectThrows(GradleException::class.java) { + verify("Library.class" to 8, "META-INF/versions/17/Library.class" to 21, multiRelease = true) + } + } + + @Test + fun ignoresVersionedEntriesWithoutMultiReleaseManifest() { + verify("Library.class" to 17, "META-INF/versions/17/Library.class" to 21) + } +} diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index d141cc9abf193..2b79ea9ea14f9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -557,7 +557,7 @@ BSD 3-clause "New" or "Revised" License * LevelDB -- (included in org.rocksdb.*.jar) -- ../licenses/LICENSE-LevelDB.txt * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt * JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt - * JLine3 -- org.jline-jline-4.3.1.jar -- ../licenses/LICENSE-JLine.txt + * JLine3 -- org.jline-jline-4.4.6-jdk11.jar -- ../licenses/LICENSE-JLine.txt * OW2 ASM - org.ow2.asm-asm-9.10.1.jar -- ../licenses/LICENSE-ASM.txt - org.ow2.asm-asm-commons-9.10.1.jar -- ../licenses/LICENSE-ASM.txt diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 7230daf30ae42..ab3601e55ed21 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -433,7 +433,7 @@ The Apache Software License, Version 2.0 * JetBrains Annotations -- annotations-26.1.0.jar BSD 3-clause "New" or "Revised" License - * JLine3 -- jline-4.3.1.jar -- ../licenses/LICENSE-JLine.txt + * JLine3 -- jline-4.4.6-jdk11.jar -- ../licenses/LICENSE-JLine.txt MIT License * SLF4J -- ../licenses/LICENSE-SLF4J.txt diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 66536cb8f6efe..72aaaaef69185 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -190,6 +190,10 @@ RUN pip3 install --break-system-packages --no-cache-dir \ COPY --from=jvm /opt/jvm /opt/jvm ENV JAVA_HOME=/opt/jvm +# Match the compiled server target; custom builds may override this build argument. +ARG PULSAR_MIN_JAVA_VERSION=21 +ENV PULSAR_MIN_JAVA_VERSION=${PULSAR_MIN_JAVA_VERSION} + COPY --from=snappy-java /tmp/libsnappyjava.so /usr/lib/libsnappyjava.so # The default is /pulsat/bin and cannot be written. diff --git a/docker/pulsar/Dockerfile.wolfi b/docker/pulsar/Dockerfile.wolfi index 52cc57ff5ed0b..02e1ee38d9058 100644 --- a/docker/pulsar/Dockerfile.wolfi +++ b/docker/pulsar/Dockerfile.wolfi @@ -115,6 +115,10 @@ RUN pip3 install --break-system-packages --no-cache-dir \ COPY --from=jvm /opt/jvm /opt/jvm ENV JAVA_HOME=/opt/jvm +# Match the compiled server target; custom builds may override this build argument. +ARG PULSAR_MIN_JAVA_VERSION=21 +ENV PULSAR_MIN_JAVA_VERSION=${PULSAR_MIN_JAVA_VERSION} + # The default is /pulsat/bin and cannot be written. ENV PULSAR_PID_DIR=/pulsar/logs diff --git a/docker/pulsar/build.gradle.kts b/docker/pulsar/build.gradle.kts index 1d9d348e322a7..bdb7eb47912cf 100644 --- a/docker/pulsar/build.gradle.kts +++ b/docker/pulsar/build.gradle.kts @@ -67,6 +67,7 @@ fun registerDockerBuild(taskName: String, dockerfile: String, imageTag: String) val pythonClientVersion = libs.versions.pulsar.client.python.get() val snappyVersion = libs.versions.snappy.get() val jdkMajorVersion = libs.versions.docker.jdk.get() + val minJavaVersion = providers.gradleProperty("pulsarJavaVersion").getOrElse("21") // Docker build context is the project directory workingDir = projectDir @@ -80,6 +81,7 @@ fun registerDockerBuild(taskName: String, dockerfile: String, imageTag: String) "--build-arg", "PULSAR_CLIENT_PYTHON_VERSION=${pythonClientVersion}", "--build-arg", "SNAPPY_VERSION=${snappyVersion}", "--build-arg", "IMAGE_JDK_MAJOR_VERSION=${jdkMajorVersion}", + "--build-arg", "PULSAR_MIN_JAVA_VERSION=${minJavaVersion}", "--build-arg", "PULSAR_OFFLOADER_TARBALL=build/target/${offloaderTarballName}", ) diff --git a/gradle.properties b/gradle.properties index 635f41f35aae2..b943db788c071 100644 --- a/gradle.properties +++ b/gradle.properties @@ -20,6 +20,11 @@ group=org.apache.pulsar version=5.0.0-SNAPSHOT +# Java release targets for server code and client/public API code, respectively. +# These do not change the JDK required to run Gradle itself. +pulsarJavaVersion=21 +pulsarClientJavaVersion=17 + org.gradle.configuration-cache=true org.gradle.configureondemand=true org.gradle.parallel=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index cd6569cc0b7e7..bfa8ab238e7e4 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -122,7 +122,7 @@ auth0-java-jwt = "4.6.0" auth0-jwks-rsa = "0.24.1" # CLI picocli = "4.7.7" -jline3 = "4.3.1" +jline3 = "4.4.6" javassist = "3.32.0-GA" rocksdb = "9.9.3" audience-annotations = "0.12.0" diff --git a/pulsar-build/run_unit_group_gradle.sh b/pulsar-build/run_unit_group_gradle.sh index 4ac3f74237918..59f503ebbb4d1 100755 --- a/pulsar-build/run_unit_group_gradle.sh +++ b/pulsar-build/run_unit_group_gradle.sh @@ -144,6 +144,7 @@ function test_group_other() { -x :tests:pulsar-client-v5-all-test:test \ -x :tests:pulsar-client-shade-test:test \ -x :tests:pulsar-client-native-image:test \ + -x :tests:pulsar-client-java-compatibility:test \ test # Run DnsResolverTest separately since it relies on static field values diff --git a/pulsar-client-tools-test/build.gradle.kts b/pulsar-client-tools-test/build.gradle.kts index 9823c9c6a3c9f..ed037cebf4a19 100644 --- a/pulsar-client-tools-test/build.gradle.kts +++ b/pulsar-client-tools-test/build.gradle.kts @@ -23,8 +23,6 @@ plugins { dependencies { implementation(libs.slog) - compileOnly(project(":pulsar-client-tools")) - compileOnly(project(":pulsar-broker")) testImplementation(project(":pulsar-client-tools")) testImplementation(project(":pulsar-broker")) diff --git a/pulsar-client-tools/build.gradle.kts b/pulsar-client-tools/build.gradle.kts index ad09db030bc7b..ce4d4a0d608db 100644 --- a/pulsar-client-tools/build.gradle.kts +++ b/pulsar-client-tools/build.gradle.kts @@ -32,12 +32,12 @@ dependencies { implementation(project(":pulsar-common")) implementation(project(":pulsar-client-messagecrypto-bc")) implementation(project(":pulsar-cli-utils")) - implementation(project(":pulsar-websocket")) { - exclude(group = "*", module = "*") - } api(libs.picocli) - implementation(libs.picocli.shell.jline3) - api(libs.jline) + implementation(libs.picocli.shell.jline3) { + exclude(group = "org.jline", module = "jline") + } + // The default bundle includes an optional FFM provider compiled for Java 22. + api(variantOf(libs.jline) { classifier("jdk11") }) implementation(libs.commons.io) implementation(libs.commons.lang3) // guava was previously leaked onto the compile classpath via compileOnly(swagger-core 1.x) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerCommand.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ConsumerCommand.java similarity index 100% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerCommand.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ConsumerCommand.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java similarity index 100% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ConsumerMessage.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/EndOfTopicResponse.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/EndOfTopicResponse.java similarity index 100% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/EndOfTopicResponse.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/EndOfTopicResponse.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerAck.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerAck.java similarity index 97% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerAck.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerAck.java index 1e62098ac2bc5..63a8deee95eb6 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerAck.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerAck.java @@ -24,7 +24,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.pulsar.websocket.WebSocketError; /** * Represent result of publishing a single message. diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerAcks.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerAcks.java similarity index 100% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerAcks.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerAcks.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java similarity index 100% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerMessage.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessages.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerMessages.java similarity index 100% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/ProducerMessages.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/ProducerMessages.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketError.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/WebSocketError.java similarity index 97% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketError.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/WebSocketError.java index 07d6959a3fe5e..b625bc60b1dd3 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketError.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/WebSocketError.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.websocket; +package org.apache.pulsar.websocket.data; /** * Enum for possible errors in the proxy diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/package-info.java b/pulsar-common/src/main/java/org/apache/pulsar/websocket/data/package-info.java similarity index 100% rename from pulsar-websocket/src/main/java/org/apache/pulsar/websocket/data/package-info.java rename to pulsar-common/src/main/java/org/apache/pulsar/websocket/data/package-info.java diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java index 930cb4f571a1d..ad95973357166 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/AbstractWebSocketHandler.java @@ -57,6 +57,7 @@ import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.websocket.data.ConsumerCommand; +import org.apache.pulsar.websocket.data.WebSocketError; import org.eclipse.jetty.ee10.websocket.server.JettyServerUpgradeResponse; import org.eclipse.jetty.websocket.api.Callback; import org.eclipse.jetty.websocket.api.Session; diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index b0ee29e48ec5e..c42522cd39fc2 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -55,6 +55,7 @@ import org.apache.pulsar.websocket.data.ConsumerCommand; import org.apache.pulsar.websocket.data.ConsumerMessage; import org.apache.pulsar.websocket.data.EndOfTopicResponse; +import org.apache.pulsar.websocket.data.WebSocketError; import org.eclipse.jetty.ee10.websocket.server.JettyServerUpgradeResponse; import org.eclipse.jetty.websocket.api.Callback; import org.eclipse.jetty.websocket.api.Session; diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java index 2d1f46cc47427..139f0ec41d661 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ProducerHandler.java @@ -22,9 +22,9 @@ import static java.lang.String.format; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.pulsar.common.api.EncryptionContext.EncryptionKey; -import static org.apache.pulsar.websocket.WebSocketError.FailedToDeserializeFromJSON; -import static org.apache.pulsar.websocket.WebSocketError.PayloadEncodingError; -import static org.apache.pulsar.websocket.WebSocketError.UnknownError; +import static org.apache.pulsar.websocket.data.WebSocketError.FailedToDeserializeFromJSON; +import static org.apache.pulsar.websocket.data.WebSocketError.PayloadEncodingError; +import static org.apache.pulsar.websocket.data.WebSocketError.UnknownError; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectReader; diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java index 5419b369c9223..2d03abad3b246 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java @@ -47,6 +47,7 @@ import org.apache.pulsar.websocket.data.ConsumerCommand; import org.apache.pulsar.websocket.data.ConsumerMessage; import org.apache.pulsar.websocket.data.EndOfTopicResponse; +import org.apache.pulsar.websocket.data.WebSocketError; import org.eclipse.jetty.ee10.websocket.server.JettyServerUpgradeResponse; import org.eclipse.jetty.websocket.api.Callback; import org.eclipse.jetty.websocket.api.Session; diff --git a/settings.gradle.kts b/settings.gradle.kts index a5e0390dda065..74aa52fe748e9 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -288,3 +288,6 @@ include("tests:pulsar-client-all-shade-test") project(":tests:pulsar-client-all-shade-test").projectDir = file("tests/pulsar-client-all-shade-test") include("tests:pulsar-client-native-image") project(":tests:pulsar-client-native-image").projectDir = file("tests/pulsar-client-native-image") + +// Compile and run consumer/API compatibility checks on the minimum supported client JDK. +include("tests:pulsar-client-java-compatibility") diff --git a/tests/pulsar-client-java-compatibility/build.gradle.kts b/tests/pulsar-client-java-compatibility/build.gradle.kts new file mode 100644 index 0000000000000..81e2e2eef2226 --- /dev/null +++ b/tests/pulsar-client-java-compatibility/build.gradle.kts @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("pulsar.java-conventions") +} + +// Use only the consumer dependency graph: broker fixtures and Functions implementations must +// never be needed to compile a Java 17 client or a user's Function/Source/Sink implementation. +dependencies { + implementation(project(":pulsar-client-tools")) + implementation(project(":pulsar-client-v5")) + implementation(project(":pulsar-client-admin-original")) + implementation(project(":pulsar-client-auth-athenz")) + implementation(project(":pulsar-client-auth-sasl")) + implementation(project(":pulsar-client-messagecrypto-bc")) + implementation(project(":pulsar-functions:pulsar-functions-api")) + implementation(project(":pulsar-io:pulsar-io-core")) +} + +// pulsar.java-conventions compiles/runs these tests with pulsarClientJavaVersion (default 17), +// independently of the build JVM or -PtestJavaVersion used for server tests. diff --git a/tests/pulsar-client-java-compatibility/src/test/java/org/apache/pulsar/compatibility/ClientJavaCompatibilityTest.java b/tests/pulsar-client-java-compatibility/src/test/java/org/apache/pulsar/compatibility/ClientJavaCompatibilityTest.java new file mode 100644 index 0000000000000..94a5ba6b49d36 --- /dev/null +++ b/tests/pulsar-client-java-compatibility/src/test/java/org/apache/pulsar/compatibility/ClientJavaCompatibilityTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.compatibility; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.util.Map; +import java.util.Properties; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.cli.PulsarClientTool; +import org.apache.pulsar.functions.api.Context; +import org.apache.pulsar.functions.api.Function; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; +import org.apache.pulsar.websocket.data.ProducerMessage; +import org.testng.annotations.Test; + +public class ClientJavaCompatibilityTest { + // Compile real user implementations against the complete API graph, not just the API jar. + private static class Echo implements Function { + @Override + public String process(String input, Context context) { + return input; + } + } + + private static class StringSink implements Sink { + private String value; + + @Override + public void open(Map config, SinkContext context) { + } + + @Override + public void write(Record record) { + value = record.getValue(); + } + + @Override + public void close() { + } + } + + @Test + public void loadClientImplementationsOnClientJavaVersion() { + assertEquals(Runtime.version().feature(), Integer.parseInt(System.getProperty("pulsarClientJavaVersion"))); + assertNotNull(PulsarClient.builder()); + assertNotNull(org.apache.pulsar.client.api.v5.PulsarClient.builder()); + assertNotNull(PulsarAdmin.builder()); + assertEquals(Schema.STRING.decode(Schema.STRING.encode("java17")), "java17"); + } + + @Test + public void loadClientToolsOnClientJavaVersion() { + assertNotNull(new PulsarClientTool(new Properties())); + ProducerMessage message = new ProducerMessage(); + message.setPayload("java17"); + assertEquals(message.getPayload(), "java17"); + } + + @Test + public void runUserFunctionAndSinkOnClientJavaVersion() { + assertEquals(new Echo().process("java17", null), "java17"); + StringSink sink = new StringSink(); + sink.write(() -> "java17"); + assertEquals(sink.value, "java17"); + } +}