diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index a79842c540764..9d16d3b0440c1 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -200,6 +200,12 @@ jobs: - name: Check that project's public libraries can be published to a Maven repository run: ./gradlew publishAllPublicationsToLocalDeployRepository + - name: Check API/SPI publication dependency closure with a custom group + # Validate generated POMs and Gradle metadata, including the BOM, without uploading artifacts. + run: >- + ./gradlew validateApiSpiPublication + -PpublishApiAndSpiOnly=true -Pgroup=myorg.pulsar + - name: Upload Gradle reports uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec3f0a2115266..b1a24db8d9221 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,11 @@ For the Gradle build infrastructure and how to change build files (convention pl catalog, configuration-cache rules), see [`ARCHITECTURE.md` → Build infrastructure](ARCHITECTURE.md#build-infrastructure). +### Publishing artifacts + +See [Publishing Maven artifacts](build-logic/PUBLISHING.md) for ordinary publishing, the +API/SPI subset, custom repositories, groups and credentials. + ## Running tests Most of these per-module "unit tests" are actually **integration-style** — they start a real in-JVM diff --git a/build-logic/PUBLISHING.md b/build-logic/PUBLISHING.md new file mode 100644 index 0000000000000..ae80240571e1e --- /dev/null +++ b/build-logic/PUBLISHING.md @@ -0,0 +1,285 @@ + + +# Publishing Maven artifacts + +Run the commands below from the repository root using `./gradlew`. The shared publication and +repository conventions live in [`conventions/src/main/kotlin`](conventions/src/main/kotlin). + +## Ordinary publishing + +By default (`publishApiAndSpiOnly` unset or `false`), Gradle publishes the modules that apply the +publishing conventions, including the root `pulsar` parent POM and the full `pulsar-bom`. The +repository's `gradle.properties` supplies the default group (`org.apache.pulsar`) and version. + +Choose the task for the intended destination: + +| Task | Destination | Version requirement | +| --- | --- | --- | +| `publishAllPublicationsToLocalDeployRepository` | `build/local-deploy-repo` | Any version | +| `publishToMavenLocal` | Local Maven repository, normally `~/.m2/repository` | Any version | +| `publishAllPublicationsToApacheSnapshotsRepository` | ASF Nexus snapshots | Ends in `-SNAPSHOT` | +| `publishAllPublicationsToApacheReleasesRepository` | ASF Nexus staging | Does not end in `-SNAPSHOT` | + +The remote publishing examples below assume a non-`SNAPSHOT` version. For snapshots, use the +alternative task noted in each example. + +```bash +# Inspect the complete publication set in a local directory. +./gradlew publishAllPublicationsToLocalDeployRepository + +# Publish a release using credentials supplied by the environment or properties file. +# For snapshots, use publishAllPublicationsToApacheSnapshotsRepository instead. +./gradlew publishAllPublicationsToApacheReleasesRepository +``` + +Use an unqualified task name from the root to publish across projects. A fully qualified task such +as `:pulsar-client-api:publishMavenPublicationToApacheReleasesRepository` publishes just that +project; it does not publish its dependency artifacts. Avoid the generic `publish` task when you +intend to target only one repository: it targets all configured publishing repositories. + +### ASF release staging and signing + +For an Apache Pulsar release, follow the canonical +[release process: stage artifacts in ASF Nexus](https://github.com/apache/pulsar-site/blob/main/contribute/release-process.md#stage-artifacts-in-the-asf-nexus-repository) +for release preparation, signing-key setup, staging, and closing the staging repository. From a +prepared release checkout with a non-snapshot version, the Gradle upload command is: + +```bash + # ASF_USERNAME, ASF_PASSWORD and APACHE_USER_GPGID are supplied by your environment. + # For snapshots, use publishAllPublicationsToApacheSnapshotsRepository + # and the apacheSnapshotsUsername/apacheSnapshotsPassword properties. + ORG_GRADLE_PROJECT_apacheReleasesUsername="$ASF_USERNAME" \ +ORG_GRADLE_PROJECT_apacheReleasesPassword="$ASF_PASSWORD" \ +./gradlew publishAllPublicationsToApacheReleasesRepository \ + -PuseGpgCmd=true -Psigning.gnupg.keyName="$APACHE_USER_GPGID" +``` + +`-PuseGpgCmd=true` enables command-line GPG signing. `signing.gnupg.keyName` selects a key; if +omitted, GPG uses its default key. Signing tasks are disabled when no signing configuration is +present, which supports local development. Configure signing for ASF release publication. +The build serializes Maven uploads within one Gradle invocation, so `--no-parallel` is unnecessary. +For ASF staging, finish and close one staging repository before starting another release upload. + +## Custom repositories, groups and credentials + +The existing repository definitions can target a custom Maven repository without editing build +scripts. Override these Gradle properties: + +| Repository | URL property | Credential properties | +| --- | --- | --- | +| `apacheSnapshots` | `apacheSnapshotsRepoUrl` | `apacheSnapshotsUsername`, `apacheSnapshotsPassword` | +| `apacheReleases` | `apacheReleasesRepoUrl` | `apacheReleasesUsername`, `apacheReleasesPassword` | + +The task names and snapshot/release version checks stay the same even when the URLs are overridden. +For example, a GitHub Packages destination still uses `publishAllPublicationsToApacheSnapshotsRepository` +for snapshots and `publishAllPublicationsToApacheReleasesRepository` for release versions. + +Set `group` to publish under a different Maven group. It applies to the parent POM, module +coordinates, BOM entries and publication-only Pulsar dependency replacements. Changing the group +does not change Java package names or imports, so switching to the custom group requires dependency +configuration changes only; no application source-code changes are required because of the group +change. The group override works with either ordinary publishing or the API/SPI subset. +Set a custom build version with +`-Pversion=5.0.0+mypatch.1`; it applies to all published artifacts, including the parent POM and BOM. +Use a version ending in `-SNAPSHOT` when publishing to the snapshots repository. + +### Properties file in ephemeral CI + +On an ephemeral CI worker, with `REPO_USERNAME` and `REPO_PASSWORD` supplied by CI secrets, the +following configures both repository definitions in the Gradle user properties file. The publishing +commands below set the custom group with `-Pgroup=myorg.pulsar`. +This example assumes the default Gradle user home, `~/.gradle`; when `GRADLE_USER_HOME` is set, use +`$GRADLE_USER_HOME/gradle.properties` instead. + +```bash +mkdir -p ~/.gradle +cat >> ~/.gradle/gradle.properties < **Note:** This custom-build use case is not intended for handling CVEs in transitive third-party +> client dependencies. With Pulsar 5, the recommended approach for dependencies bundled inside +> shaded JARs is to switch to **unshaded client dependencies** and update the affected dependency +> through the consuming build's dependency management. There is no need to create a custom Pulsar +> build just to replace a transitive dependency when using unshaded JARs. See the +> [`pulsar-client-v5-all` guide](../pulsar-client-v5-all/README.md) for the unshaded aggregate and +> migration instructions, including the exclusions needed to remove old shaded artifacts. This +> also applies to applications using the ordinary v4 client API: the aggregate includes v4, v5 +> and admin implementations, so switching dependencies does not require migrating source code to +> the v5 API. Verify compatibility with the replacement dependency. + +Use `-PpublishApiAndSpiOnly=true` to select this publication mode. The explicit project selection lives +in [`PulsarApiSpiPublication.projects`](conventions/src/main/kotlin/PulsarApiSpiPublication.kt). +Add a project's Gradle path there to include it. Without the property, publishing uses the normal +release publication set. + +Override the group with `-Pgroup=`. The parent POM and publication-only dependency +replacements use the same group. In API/SPI mode, `pulsar-bom` retains only constraints for selected +projects, in both its Maven POM and Gradle Module Metadata. + +```bash +# Validate the selected artifacts' generated POMs and Gradle Module Metadata, without uploading. +./gradlew validateApiSpiPublication -PpublishApiAndSpiOnly=true -Pgroup=com.example.pulsar + +# Publish the complete selection to build/local-deploy-repo for inspection. +./gradlew publishAllPublicationsToLocalDeployRepository \ + -PpublishApiAndSpiOnly=true -Pgroup=com.example.pulsar +``` + +Use the unqualified `publishAllPublicationsToRepository` task from the repository root +for any configured Maven repository. Gradle selects that task across projects; projects outside the +selection have no publications in this mode. A fully qualified per-project task publishes only that +project, so use the unqualified command when publishing the complete set to a fresh repository. + +To publish the subset to a custom releases repository configured as above: + +```bash +# For snapshots, use publishAllPublicationsToApacheSnapshotsRepository and a -SNAPSHOT version. +./gradlew publishAllPublicationsToApacheReleasesRepository \ + -PpublishApiAndSpiOnly=true -Pgroup=myorg.pulsar -Pversion=5.0.0+mypatch.1 +``` + +Every Maven upload (including `publishToMavenLocal`) in this mode depends on +`validateApiSpiPublication`. Validation checks the generated metadata of **every selected project**, +including parent references, platform constraints and publication-only dependency replacements. +Every Pulsar coordinate must identify another selected publication at the same published version; +checking all such edges establishes transitive closure. Missing projects fail validation before any +upload, with the referring artifact and missing coordinate. Test/build dependencies and dependencies +bundled inside shaded artifacts do not need separate publication unless the generated metadata +references them. Generating Gradle metadata can build JARs, so this is not a source-only check. + +Both configure-on-demand and configuration cache remain supported. A scoped build such as +`:pulsar-client-api:assemble` configures only the projects it needs even with API/SPI mode enabled. +Validation and publication intentionally configure the entire selected publication set and the +build dependencies needed to generate its metadata. diff --git a/build-logic/conventions/build.gradle.kts b/build-logic/conventions/build.gradle.kts index 46120a3102ed8..bd294d030d3e9 100644 --- a/build-logic/conventions/build.gradle.kts +++ b/build-logic/conventions/build.gradle.kts @@ -22,6 +22,7 @@ plugins { } dependencies { + testImplementation(libs.testng) // The Shadow plugin brings its own log4j-core onto the build classpath; align it with the // log4j version the rest of the build uses (`log4j2` in the version catalog). implementation(platform(libs.log4j.bom)) @@ -35,3 +36,7 @@ dependencies { "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" }) } + +tasks.test { + useTestNG() +} diff --git a/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt b/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt new file mode 100644 index 0000000000000..63f2420102a6e --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.Project + +/** API/SPI publication set for Java clients, Functions and plugins, including their published dependencies. */ +object PulsarApiSpiPublication { + val projects: Set = setOf( + ":", + ":buildtools", + ":managed-ledger", + ":pulsar-bom", + ":pulsar-broker", + ":pulsar-broker-auth-sasl", + ":pulsar-broker-common", + ":pulsar-cli-utils", + ":pulsar-client-admin-api", + ":pulsar-client-admin-original", + ":pulsar-client-admin-shaded", + ":pulsar-client-all", + ":pulsar-client-api", + ":pulsar-client-api-v5", + ":pulsar-client-auth-sasl", + ":pulsar-client-fastutil-minimized", + ":pulsar-client-messagecrypto-bc", + ":pulsar-client-original", + ":pulsar-client-shaded", + ":pulsar-client-v5", + ":pulsar-client-v5-all", + ":pulsar-client-v5-shaded", + ":pulsar-common", + ":pulsar-config-validation", + ":pulsar-dependencies", + ":pulsar-docs-tools", + ":pulsar-functions:pulsar-functions-api", + ":pulsar-functions:pulsar-functions-instance", + ":pulsar-functions:pulsar-functions-proto", + ":pulsar-functions:pulsar-functions-runtime", + ":pulsar-functions:pulsar-functions-secrets", + ":pulsar-functions:pulsar-functions-utils", + ":pulsar-functions:pulsar-functions-worker", + ":pulsar-http-client-api", + ":pulsar-io:pulsar-io-core", + ":pulsar-metadata", + ":pulsar-opentelemetry", + ":pulsar-package-management:pulsar-package-core", + ":pulsar-package-management:pulsar-package-filesystem-storage", + ":pulsar-proxy", + ":pulsar-tls-factory-api", + ":pulsar-transaction:pulsar-transaction-common", + ":pulsar-transaction:pulsar-transaction-coordinator", + ":pulsar-websocket", + ":testmocks", + ) + + fun isEnabled(project: Project): Boolean = + project.providers.gradleProperty("publishApiAndSpiOnly").getOrElse("false").toBoolean() + + fun includes(project: Project): Boolean = !isEnabled(project) || project.path in projects +} diff --git a/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt b/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt new file mode 100644 index 0000000000000..e9ae0714081e9 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction +import org.gradle.work.DisableCachingByDefault +import org.w3c.dom.Element +import javax.xml.parsers.DocumentBuilderFactory + +/** Checks the consumer graph, including publication-only rewrites and dependency-reduced shaded POMs. */ +@DisableCachingByDefault(because = "Verification has no outputs") +abstract class ValidateApiSpiPublication : DefaultTask() { + @get:InputFiles + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val poms: ConfigurableFileCollection + + @get:InputFiles + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val moduleMetadata: ConfigurableFileCollection + + @get:Input + abstract val publicationGroup: Property + + @TaskAction + fun validate() { + val factory = DocumentBuilderFactory.newInstance() + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + fun Element.childText(name: String): String = (0 until childNodes.length) + .map { childNodes.item(it) }.filterIsInstance() + .firstOrNull { it.tagName == name }?.textContent.orEmpty() + val documents = poms.files.sorted().associateWith { + factory.newDocumentBuilder().parse(it).documentElement + } + val coordinates = documents.values.map { + "${it.childText("groupId")}:${it.childText("artifactId")}:${it.childText("version")}" + }.toSet() + val failures = sortedSetOf() + fun check(source: String, group: String, artifact: String, version: String) { + if (group == publicationGroup.get() || group == "org.apache.pulsar") { + val coordinate = "$group:$artifact:$version" + if (coordinate !in coordinates) { + failures.add("$source -> $coordinate") + } + } + } + for ((file, root) in documents) { + val source = root.childText("artifactId") + " (" + file.name + ")" + if (root.childText("groupId") != publicationGroup.get()) { + failures.add("$source publishes under ${root.childText("groupId")} instead of ${publicationGroup.get()}") + } + for (tag in listOf("parent", "dependency")) { + val nodes = root.getElementsByTagName(tag) + for (i in 0 until nodes.length) { + val node = nodes.item(i) as Element + check(source, node.childText("groupId"), node.childText("artifactId"), node.childText("version")) + } + } + } + for (file in moduleMetadata.files.sorted()) { + val json = JsonSlurper().parse(file) as Map<*, *> + val component = json["component"] as Map<*, *> + val source = "${component["module"]} (Gradle metadata)" + for (variant in json["variants"] as List<*>) { + val data = variant as Map<*, *> + for (key in listOf("dependencies", "dependencyConstraints")) { + for (entry in data[key] as? List<*> ?: emptyList()) { + val dep = entry as Map<*, *> + val version = dep["version"] as? Map<*, *> + check(source, dep["group"].toString(), dep["module"].toString(), + (version?.get("strictly") ?: version?.get("requires") ?: version?.get("prefers")).toString()) + } + } + (data["available-at"] as? Map<*, *>)?.let { + check(source, it["group"].toString(), it["module"].toString(), it["version"].toString()) + } + } + } + if (failures.isNotEmpty()) { + throw GradleException("API/SPI publication contains unpublished Pulsar dependencies:\n" + + failures.joinToString("\n") + + "\nAdd the missing projects to PulsarApiSpiPublication.projects or correct their published coordinates.") + } + logger.lifecycle("Validated {} API/SPI publications and their published dependency closure.", coordinates.size) + } +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts index c0c9c1b7eb424..c71df08bc18bf 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts @@ -50,13 +50,14 @@ val pulsarPlatformModules = setOf( "pulsar-package-core", ) +val pulsarGroup = project.group.toString() configurations.named("runtimeClasspath") { exclude(group = "org.apache.bookkeeper") // Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle it. // Bundling a different version causes GeneratedMessage.getUnknownFields() conflicts. exclude(group = "com.google.protobuf") pulsarPlatformModules.forEach { module -> - exclude(group = "org.apache.pulsar", module = module) + exclude(group = pulsarGroup, module = module) } } @@ -97,7 +98,7 @@ if (parentProject != null && parentProject != rootProject && parentProject.paren // NAR modules bundle all dependencies, so the POM should have no section. publishing { publications { - named("maven") { + withType().configureEach { // Replace component-based artifacts with just the NAR file artifacts.clear() artifact(tasks.named("nar")) diff --git a/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts index 5841e7d70baa5..ac3cac573f969 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts @@ -30,7 +30,8 @@ plugins { // in scopes that end up in the published POM (api, implementation, runtimeOnly). // Test/compileOnly scoped dependencies are excluded since they don't appear in the POM. // NAR modules are not validated here — they bundle all dependencies and have empty POMs. -run { +// API/SPI mode validates the generated consumer metadata for the entire selection before uploading. +if (!PulsarApiSpiPublication.isEnabled(project)) { val publishedScopes = listOf("api", "implementation", "runtimeOnly", "shadow", "shadowApi") val configsToCheck = publishedScopes.mapNotNull { name -> configurations.findByName(name)?.let { name to it } diff --git a/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts index 58b11e0161278..253fd5b6ee338 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts @@ -28,6 +28,9 @@ plugins { // --- java-library projects: JAR + sources + javadoc --- pluginManager.withPlugin("java-library") { + if (!PulsarApiSpiPublication.includes(project)) { + return@withPlugin + } val sourceSets = the() // Match Maven's javadoc configuration: no doclint, don't fail on errors @@ -90,6 +93,9 @@ pluginManager.withPlugin("java-library") { // --- java-platform projects (BOM, dependencies): POM-only, no JAR --- pluginManager.withPlugin("java-platform") { + if (!PulsarApiSpiPublication.includes(project)) { + return@withPlugin + } publishing { publications { create("maven") { @@ -107,6 +113,7 @@ run { val isPlatformProject = plugins.hasPlugin("java-platform") val isRootProject = project == rootProject val pulsarVersion = version.toString() + val pulsarGroup = project.group.toString() // Per-module POM name and description. Read in afterEvaluate so that a description // assigned in a module's build script body is picked up, and captured as plain strings @@ -162,7 +169,7 @@ run { s = s.replace( "4.0.0", "4.0.0\n \n" + - " org.apache.pulsar\n" + + " $pulsarGroup\n" + " pulsar\n" + " $pulsarVersion\n" + " " diff --git a/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts index 4fe98e0e1ad0d..2ace22ea59a2e 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts @@ -167,3 +167,34 @@ tasks.withType().configureEach { providers.gradleProperty("signing.gnupg.keyName").isPresent || (providers.gradleProperty("useGpgCmd").orNull?.toBoolean() ?: false) } + +// Every upload in API/SPI mode waits for the entire selected consumer graph to pass validation. +// String task dependencies also configure the selected projects with configure-on-demand enabled. +if (PulsarApiSpiPublication.isEnabled(project)) { + if (project == rootProject) { + tasks.register("validateApiSpiPublication") { + group = "verification" + description = "Validate that every published Pulsar dependency belongs to the API/SPI publication set." + publicationGroup.set(project.group.toString()) + for (selectedPath in PulsarApiSpiPublication.projects) { + val selected = project(selectedPath) + // A qualified task invocation does not discover/configure container projects. + // Initialize their Kotlin script scopes before Gradle resolves nested task paths. + generateSequence(selected.parent) { it.parent } + .takeWhile { it != rootProject }.toList().asReversed().forEach { + project.evaluationDependsOn(it.path) + } + val prefix = if (selectedPath == ":") "" else selectedPath + poms.from(selected.layout.buildDirectory.file("publications/maven/pom-default.xml")) + dependsOn("$prefix:generatePomFileForMavenPublication") + if (selectedPath != ":") { + moduleMetadata.from(selected.layout.buildDirectory.file("publications/maven/module.json")) + dependsOn("$prefix:generateMetadataFileForMavenPublication") + } + } + } + } + tasks.withType().configureEach { + dependsOn(":validateApiSpiPublication") + } +} diff --git a/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt b/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt new file mode 100644 index 0000000000000..cda695d6f8212 --- /dev/null +++ b/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.GradleException +import org.gradle.testfixtures.ProjectBuilder +import org.testng.Assert.assertTrue +import org.testng.Assert.expectThrows +import org.testng.annotations.Test +import java.nio.file.Files + +class ValidateApiSpiPublicationTest { + private fun verify(pomDependency: String = "", metadataDependency: String = "", parent: String = "") { + val directory = Files.createTempDirectory("api-spi-publication-test").toFile() + try { + val project = ProjectBuilder.builder().withProjectDir(directory).build() + val task = project.tasks.register("validate", ValidateApiSpiPublication::class.java).get() + task.publicationGroup.set("example.pulsar") + val pom = directory.resolve("pom.xml") + pom.writeText(""" + 4.0.0 + $parent + example.pulsarclient1 + $pomDependency + """.trimIndent()) + val module = directory.resolve("module.json") + module.writeText(""" + {"component":{"group":"example.pulsar","module":"client","version":"1"}, + "variants":[{"name":"runtime","dependencies":[$metadataDependency]}]} + """.trimIndent()) + task.poms.from(pom) + task.moduleMetadata.from(module) + task.validate() + } finally { + directory.deleteRecursively() + } + } + + @Test + fun acceptsClosedGraphAndThirdPartyDependencies() { + verify( + "example.pulsarclient" + + "1", + """{"group":"third.party","module":"library","version":{"requires":"2"}}""" + ) + } + + @Test + fun rejectsMissingPomDependency() { + val error = expectThrows(GradleException::class.java) { + verify("example.pulsarforgotten" + + "1") + } + assertTrue(error.message!!.contains("client (pom.xml) -> example.pulsar:forgotten:1")) + } + + @Test + fun rejectsMissingMetadataDependency() { + val error = expectThrows(GradleException::class.java) { + verify(metadataDependency = + """{"group":"example.pulsar","module":"forgotten","version":{"requires":"1"}}""") + } + assertTrue(error.message!!.contains("client (Gradle metadata) -> example.pulsar:forgotten:1")) + } + + @Test + fun rejectsOriginalGroupAndMissingParent() { + val error = expectThrows(GradleException::class.java) { + verify(parent = "org.apache.pulsarpulsar" + + "1") + } + assertTrue(error.message!!.contains("org.apache.pulsar:pulsar:1")) + } + + @Test + fun rejectsWrongInternalVersion() { + val error = expectThrows(GradleException::class.java) { + verify(metadataDependency = + """{"group":"example.pulsar","module":"client","version":{"strictly":"2"}}""") + } + assertTrue(error.message!!.contains("example.pulsar:client:2")) + } +} diff --git a/buildtools/build.gradle.kts b/buildtools/build.gradle.kts index 8496a0bef970a..af4dc417f66f0 100644 --- a/buildtools/build.gradle.kts +++ b/buildtools/build.gradle.kts @@ -21,6 +21,11 @@ plugins { id("pulsar.java-conventions") } +// External consumers may use the test support classes without publishing them in normal releases. +if (PulsarApiSpiPublication.isEnabled(project)) { + apply(plugin = "pulsar.publish-conventions") +} + dependencies { implementation(libs.slog) implementation(libs.snakeyaml) diff --git a/pulsar-bom/build.gradle.kts b/pulsar-bom/build.gradle.kts index d8ec476f4ffc9..07c69fd698ab6 100644 --- a/pulsar-bom/build.gradle.kts +++ b/pulsar-bom/build.gradle.kts @@ -33,84 +33,91 @@ javaPlatform { dependencies { constraints { + fun selectedApi(dependency: ProjectDependency) { + if (!PulsarApiSpiPublication.isEnabled(project) || + dependency.path in PulsarApiSpiPublication.projects) { + add("api", dependency) + } + } + // Client API - api(project(":pulsar-client-api")) - api(project(":pulsar-client-admin-api")) - api(project(":pulsar-client-api-v5")) + selectedApi(project(":pulsar-client-api")) + selectedApi(project(":pulsar-client-admin-api")) + selectedApi(project(":pulsar-client-api-v5")) // Focused SPI modules (PIP-478): TLS factory SPI + HTTP client SPI - api(project(":pulsar-tls-factory-api")) - api(project(":pulsar-http-client-api")) + selectedApi(project(":pulsar-tls-factory-api")) + selectedApi(project(":pulsar-http-client-api")) // Shaded clients (the published artifacts users depend on) - api(project(":pulsar-client-shaded")) - api(project(":pulsar-client-admin-shaded")) - api(project(":pulsar-client-all")) - api(project(":pulsar-client-v5-shaded")) + selectedApi(project(":pulsar-client-shaded")) + selectedApi(project(":pulsar-client-admin-shaded")) + selectedApi(project(":pulsar-client-all")) + selectedApi(project(":pulsar-client-v5-shaded")) // Combined unshaded v4/v5 client and admin - api(project(":pulsar-client-v5-all")) + selectedApi(project(":pulsar-client-v5-all")) // Original (unshaded) clients - api(project(":pulsar-client-v5")) - api(project(":pulsar-client-original")) - api(project(":pulsar-client-admin-original")) + selectedApi(project(":pulsar-client-v5")) + selectedApi(project(":pulsar-client-original")) + selectedApi(project(":pulsar-client-admin-original")) // Client auth - api(project(":pulsar-client-auth-sasl")) - api(project(":pulsar-client-messagecrypto-bc")) + selectedApi(project(":pulsar-client-auth-sasl")) + selectedApi(project(":pulsar-client-messagecrypto-bc")) // Common - api(project(":pulsar-common")) - api(project(":pulsar-config-validation")) + selectedApi(project(":pulsar-common")) + selectedApi(project(":pulsar-config-validation")) // Functions API - api(project(":pulsar-functions:pulsar-functions-api")) + selectedApi(project(":pulsar-functions:pulsar-functions-api")) // IO core - api(project(":pulsar-io:pulsar-io-core")) - api(project(":pulsar-io:pulsar-io-common")) + selectedApi(project(":pulsar-io:pulsar-io-core")) + selectedApi(project(":pulsar-io:pulsar-io-common")) // Broker - api(project(":pulsar-broker")) - api(project(":pulsar-broker-common")) - api(project(":pulsar-broker-auth-oidc")) - api(project(":pulsar-broker-auth-sasl")) + selectedApi(project(":pulsar-broker")) + selectedApi(project(":pulsar-broker-common")) + selectedApi(project(":pulsar-broker-auth-oidc")) + selectedApi(project(":pulsar-broker-auth-sasl")) // Other core modules - api(project(":managed-ledger")) - api(project(":pulsar-metadata")) - api(project(":pulsar-proxy")) - api(project(":pulsar-websocket")) - api(project(":pulsar-testclient")) - api(project(":pulsar-cli-utils")) - api(project(":pulsar-client-tools")) - api(project(":pulsar-client-tools-api")) - api(project(":pulsar-opentelemetry")) - api(project(":testmocks")) + selectedApi(project(":managed-ledger")) + selectedApi(project(":pulsar-metadata")) + selectedApi(project(":pulsar-proxy")) + selectedApi(project(":pulsar-websocket")) + selectedApi(project(":pulsar-testclient")) + selectedApi(project(":pulsar-cli-utils")) + selectedApi(project(":pulsar-client-tools")) + selectedApi(project(":pulsar-client-tools-api")) + selectedApi(project(":pulsar-opentelemetry")) + selectedApi(project(":testmocks")) // Transaction - api(project(":pulsar-transaction:pulsar-transaction-common")) - api(project(":pulsar-transaction:pulsar-transaction-coordinator")) + selectedApi(project(":pulsar-transaction:pulsar-transaction-common")) + selectedApi(project(":pulsar-transaction:pulsar-transaction-coordinator")) // Functions - api(project(":pulsar-functions:pulsar-functions-instance")) - api(project(":pulsar-functions:pulsar-functions-runtime")) - api(project(":pulsar-functions:pulsar-functions-worker")) - api(project(":pulsar-functions:pulsar-functions-local-runner-original")) - api(project(":pulsar-functions:pulsar-functions-proto")) - api(project(":pulsar-functions:pulsar-functions-secrets")) - api(project(":pulsar-functions:pulsar-functions-utils")) + selectedApi(project(":pulsar-functions:pulsar-functions-instance")) + selectedApi(project(":pulsar-functions:pulsar-functions-runtime")) + selectedApi(project(":pulsar-functions:pulsar-functions-worker")) + selectedApi(project(":pulsar-functions:pulsar-functions-local-runner-original")) + selectedApi(project(":pulsar-functions:pulsar-functions-proto")) + selectedApi(project(":pulsar-functions:pulsar-functions-secrets")) + selectedApi(project(":pulsar-functions:pulsar-functions-utils")) // Athenz auth - api(project(":pulsar-client-auth-athenz")) - api(project(":pulsar-broker-auth-athenz")) + selectedApi(project(":pulsar-client-auth-athenz")) + selectedApi(project(":pulsar-broker-auth-athenz")) // Functions - api(project(":pulsar-functions:pulsar-functions-local-runner-shaded")) + selectedApi(project(":pulsar-functions:pulsar-functions-local-runner-shaded")) // Tiered storage - api(project(":tiered-storage:tiered-storage-jcloud")) - api(project(":tiered-storage:tiered-storage-file-system")) + selectedApi(project(":tiered-storage:tiered-storage-jcloud")) + selectedApi(project(":tiered-storage:tiered-storage-file-system")) } } diff --git a/pulsar-client/build.gradle.kts b/pulsar-client/build.gradle.kts index 6c802c39735b5..d5c4982b5176c 100644 --- a/pulsar-client/build.gradle.kts +++ b/pulsar-client/build.gradle.kts @@ -164,7 +164,7 @@ tasks.named("compileTestJava") { dependsOn(generateTestAvro) } run { val fromGroup = "it.unimi.dsi" val fromName = "fastutil" - val toGroup = "org.apache.pulsar" + val toGroup = project.group.toString() val toName = "pulsar-client-fastutil-minimized" val toVersion = version.toString() diff --git a/pulsar-functions/localrun-shaded/build.gradle.kts b/pulsar-functions/localrun-shaded/build.gradle.kts index 598ed425bc817..52a759f5ecbea 100644 --- a/pulsar-functions/localrun-shaded/build.gradle.kts +++ b/pulsar-functions/localrun-shaded/build.gradle.kts @@ -27,12 +27,13 @@ dependencies { } val shadePrefix = "org.apache.pulsar.functions.runtime.shaded" +val pulsarGroupPattern = Regex.escape(project.group.toString()) tasks.shadowJar { isZip64 = true dependencies { - include(dependency("org.apache.pulsar:.*")) + include(dependency("$pulsarGroupPattern:.*")) include(project(":pulsar-functions:pulsar-functions-local-runner-original")) include(project(":pulsar-client-original")) include(project(":pulsar-common"))