From 8bf71af60febc1df6aa1f8a3214da8a03b77ea0b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 14 Sep 2026 21:32:54 +0300 Subject: [PATCH] [improve][build] Add validated API/SPI publication subset Add publishApiAndSpiOnly to select Java client, Functions and plugin dependencies for publication under a custom group. Validate the generated POM and Gradle metadata dependency closure before uploads and filter the BOM to the selected projects. Document ordinary and subset publishing, custom repositories, signing and credential configuration in build-logic/PUBLISHING.md. Include validation tests and retain configuration-cache and configure-on-demand support. Assisted-by: Codex --- .github/workflows/pulsar-ci.yaml | 6 + CONTRIBUTING.md | 5 + build-logic/PUBLISHING.md | 285 ++++++++++++++++++ build-logic/conventions/build.gradle.kts | 5 + .../main/kotlin/PulsarApiSpiPublication.kt | 76 +++++ .../main/kotlin/ValidateApiSpiPublication.kt | 109 +++++++ .../kotlin/pulsar.nar-conventions.gradle.kts | 5 +- ...public-java-library-conventions.gradle.kts | 3 +- .../pulsar.publish-conventions.gradle.kts | 9 +- ...ublish-repositories-conventions.gradle.kts | 31 ++ .../kotlin/ValidateApiSpiPublicationTest.kt | 98 ++++++ buildtools/build.gradle.kts | 5 + pulsar-bom/build.gradle.kts | 103 ++++--- pulsar-client/build.gradle.kts | 2 +- .../localrun-shaded/build.gradle.kts | 3 +- 15 files changed, 691 insertions(+), 54 deletions(-) create mode 100644 build-logic/PUBLISHING.md create mode 100644 build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt create mode 100644 build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt create mode 100644 build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index a79842c540764..9d16d3b0440c1 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -200,6 +200,12 @@ jobs: - name: Check that project's public libraries can be published to a Maven repository run: ./gradlew publishAllPublicationsToLocalDeployRepository + - name: Check API/SPI publication dependency closure with a custom group + # Validate generated POMs and Gradle metadata, including the BOM, without uploading artifacts. + run: >- + ./gradlew validateApiSpiPublication + -PpublishApiAndSpiOnly=true -Pgroup=myorg.pulsar + - name: Upload Gradle reports uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec3f0a2115266..b1a24db8d9221 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,11 @@ For the Gradle build infrastructure and how to change build files (convention pl catalog, configuration-cache rules), see [`ARCHITECTURE.md` → Build infrastructure](ARCHITECTURE.md#build-infrastructure). +### Publishing artifacts + +See [Publishing Maven artifacts](build-logic/PUBLISHING.md) for ordinary publishing, the +API/SPI subset, custom repositories, groups and credentials. + ## Running tests Most of these per-module "unit tests" are actually **integration-style** — they start a real in-JVM diff --git a/build-logic/PUBLISHING.md b/build-logic/PUBLISHING.md new file mode 100644 index 0000000000000..ae80240571e1e --- /dev/null +++ b/build-logic/PUBLISHING.md @@ -0,0 +1,285 @@ + + +# Publishing Maven artifacts + +Run the commands below from the repository root using `./gradlew`. The shared publication and +repository conventions live in [`conventions/src/main/kotlin`](conventions/src/main/kotlin). + +## Ordinary publishing + +By default (`publishApiAndSpiOnly` unset or `false`), Gradle publishes the modules that apply the +publishing conventions, including the root `pulsar` parent POM and the full `pulsar-bom`. The +repository's `gradle.properties` supplies the default group (`org.apache.pulsar`) and version. + +Choose the task for the intended destination: + +| Task | Destination | Version requirement | +| --- | --- | --- | +| `publishAllPublicationsToLocalDeployRepository` | `build/local-deploy-repo` | Any version | +| `publishToMavenLocal` | Local Maven repository, normally `~/.m2/repository` | Any version | +| `publishAllPublicationsToApacheSnapshotsRepository` | ASF Nexus snapshots | Ends in `-SNAPSHOT` | +| `publishAllPublicationsToApacheReleasesRepository` | ASF Nexus staging | Does not end in `-SNAPSHOT` | + +The remote publishing examples below assume a non-`SNAPSHOT` version. For snapshots, use the +alternative task noted in each example. + +```bash +# Inspect the complete publication set in a local directory. +./gradlew publishAllPublicationsToLocalDeployRepository + +# Publish a release using credentials supplied by the environment or properties file. +# For snapshots, use publishAllPublicationsToApacheSnapshotsRepository instead. +./gradlew publishAllPublicationsToApacheReleasesRepository +``` + +Use an unqualified task name from the root to publish across projects. A fully qualified task such +as `:pulsar-client-api:publishMavenPublicationToApacheReleasesRepository` publishes just that +project; it does not publish its dependency artifacts. Avoid the generic `publish` task when you +intend to target only one repository: it targets all configured publishing repositories. + +### ASF release staging and signing + +For an Apache Pulsar release, follow the canonical +[release process: stage artifacts in ASF Nexus](https://github.com/apache/pulsar-site/blob/main/contribute/release-process.md#stage-artifacts-in-the-asf-nexus-repository) +for release preparation, signing-key setup, staging, and closing the staging repository. From a +prepared release checkout with a non-snapshot version, the Gradle upload command is: + +```bash + # ASF_USERNAME, ASF_PASSWORD and APACHE_USER_GPGID are supplied by your environment. + # For snapshots, use publishAllPublicationsToApacheSnapshotsRepository + # and the apacheSnapshotsUsername/apacheSnapshotsPassword properties. + ORG_GRADLE_PROJECT_apacheReleasesUsername="$ASF_USERNAME" \ +ORG_GRADLE_PROJECT_apacheReleasesPassword="$ASF_PASSWORD" \ +./gradlew publishAllPublicationsToApacheReleasesRepository \ + -PuseGpgCmd=true -Psigning.gnupg.keyName="$APACHE_USER_GPGID" +``` + +`-PuseGpgCmd=true` enables command-line GPG signing. `signing.gnupg.keyName` selects a key; if +omitted, GPG uses its default key. Signing tasks are disabled when no signing configuration is +present, which supports local development. Configure signing for ASF release publication. +The build serializes Maven uploads within one Gradle invocation, so `--no-parallel` is unnecessary. +For ASF staging, finish and close one staging repository before starting another release upload. + +## Custom repositories, groups and credentials + +The existing repository definitions can target a custom Maven repository without editing build +scripts. Override these Gradle properties: + +| Repository | URL property | Credential properties | +| --- | --- | --- | +| `apacheSnapshots` | `apacheSnapshotsRepoUrl` | `apacheSnapshotsUsername`, `apacheSnapshotsPassword` | +| `apacheReleases` | `apacheReleasesRepoUrl` | `apacheReleasesUsername`, `apacheReleasesPassword` | + +The task names and snapshot/release version checks stay the same even when the URLs are overridden. +For example, a GitHub Packages destination still uses `publishAllPublicationsToApacheSnapshotsRepository` +for snapshots and `publishAllPublicationsToApacheReleasesRepository` for release versions. + +Set `group` to publish under a different Maven group. It applies to the parent POM, module +coordinates, BOM entries and publication-only Pulsar dependency replacements. Changing the group +does not change Java package names or imports, so switching to the custom group requires dependency +configuration changes only; no application source-code changes are required because of the group +change. The group override works with either ordinary publishing or the API/SPI subset. +Set a custom build version with +`-Pversion=5.0.0+mypatch.1`; it applies to all published artifacts, including the parent POM and BOM. +Use a version ending in `-SNAPSHOT` when publishing to the snapshots repository. + +### Properties file in ephemeral CI + +On an ephemeral CI worker, with `REPO_USERNAME` and `REPO_PASSWORD` supplied by CI secrets, the +following configures both repository definitions in the Gradle user properties file. The publishing +commands below set the custom group with `-Pgroup=myorg.pulsar`. +This example assumes the default Gradle user home, `~/.gradle`; when `GRADLE_USER_HOME` is set, use +`$GRADLE_USER_HOME/gradle.properties` instead. + +```bash +mkdir -p ~/.gradle +cat >> ~/.gradle/gradle.properties < **Note:** This custom-build use case is not intended for handling CVEs in transitive third-party +> client dependencies. With Pulsar 5, the recommended approach for dependencies bundled inside +> shaded JARs is to switch to **unshaded client dependencies** and update the affected dependency +> through the consuming build's dependency management. There is no need to create a custom Pulsar +> build just to replace a transitive dependency when using unshaded JARs. See the +> [`pulsar-client-v5-all` guide](../pulsar-client-v5-all/README.md) for the unshaded aggregate and +> migration instructions, including the exclusions needed to remove old shaded artifacts. This +> also applies to applications using the ordinary v4 client API: the aggregate includes v4, v5 +> and admin implementations, so switching dependencies does not require migrating source code to +> the v5 API. Verify compatibility with the replacement dependency. + +Use `-PpublishApiAndSpiOnly=true` to select this publication mode. The explicit project selection lives +in [`PulsarApiSpiPublication.projects`](conventions/src/main/kotlin/PulsarApiSpiPublication.kt). +Add a project's Gradle path there to include it. Without the property, publishing uses the normal +release publication set. + +Override the group with `-Pgroup=`. The parent POM and publication-only dependency +replacements use the same group. In API/SPI mode, `pulsar-bom` retains only constraints for selected +projects, in both its Maven POM and Gradle Module Metadata. + +```bash +# Validate the selected artifacts' generated POMs and Gradle Module Metadata, without uploading. +./gradlew validateApiSpiPublication -PpublishApiAndSpiOnly=true -Pgroup=com.example.pulsar + +# Publish the complete selection to build/local-deploy-repo for inspection. +./gradlew publishAllPublicationsToLocalDeployRepository \ + -PpublishApiAndSpiOnly=true -Pgroup=com.example.pulsar +``` + +Use the unqualified `publishAllPublicationsToRepository` task from the repository root +for any configured Maven repository. Gradle selects that task across projects; projects outside the +selection have no publications in this mode. A fully qualified per-project task publishes only that +project, so use the unqualified command when publishing the complete set to a fresh repository. + +To publish the subset to a custom releases repository configured as above: + +```bash +# For snapshots, use publishAllPublicationsToApacheSnapshotsRepository and a -SNAPSHOT version. +./gradlew publishAllPublicationsToApacheReleasesRepository \ + -PpublishApiAndSpiOnly=true -Pgroup=myorg.pulsar -Pversion=5.0.0+mypatch.1 +``` + +Every Maven upload (including `publishToMavenLocal`) in this mode depends on +`validateApiSpiPublication`. Validation checks the generated metadata of **every selected project**, +including parent references, platform constraints and publication-only dependency replacements. +Every Pulsar coordinate must identify another selected publication at the same published version; +checking all such edges establishes transitive closure. Missing projects fail validation before any +upload, with the referring artifact and missing coordinate. Test/build dependencies and dependencies +bundled inside shaded artifacts do not need separate publication unless the generated metadata +references them. Generating Gradle metadata can build JARs, so this is not a source-only check. + +Both configure-on-demand and configuration cache remain supported. A scoped build such as +`:pulsar-client-api:assemble` configures only the projects it needs even with API/SPI mode enabled. +Validation and publication intentionally configure the entire selected publication set and the +build dependencies needed to generate its metadata. diff --git a/build-logic/conventions/build.gradle.kts b/build-logic/conventions/build.gradle.kts index 46120a3102ed8..bd294d030d3e9 100644 --- a/build-logic/conventions/build.gradle.kts +++ b/build-logic/conventions/build.gradle.kts @@ -22,6 +22,7 @@ plugins { } dependencies { + testImplementation(libs.testng) // The Shadow plugin brings its own log4j-core onto the build classpath; align it with the // log4j version the rest of the build uses (`log4j2` in the version catalog). implementation(platform(libs.log4j.bom)) @@ -35,3 +36,7 @@ dependencies { "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}" }) } + +tasks.test { + useTestNG() +} diff --git a/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt b/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt new file mode 100644 index 0000000000000..63f2420102a6e --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.Project + +/** API/SPI publication set for Java clients, Functions and plugins, including their published dependencies. */ +object PulsarApiSpiPublication { + val projects: Set = setOf( + ":", + ":buildtools", + ":managed-ledger", + ":pulsar-bom", + ":pulsar-broker", + ":pulsar-broker-auth-sasl", + ":pulsar-broker-common", + ":pulsar-cli-utils", + ":pulsar-client-admin-api", + ":pulsar-client-admin-original", + ":pulsar-client-admin-shaded", + ":pulsar-client-all", + ":pulsar-client-api", + ":pulsar-client-api-v5", + ":pulsar-client-auth-sasl", + ":pulsar-client-fastutil-minimized", + ":pulsar-client-messagecrypto-bc", + ":pulsar-client-original", + ":pulsar-client-shaded", + ":pulsar-client-v5", + ":pulsar-client-v5-all", + ":pulsar-client-v5-shaded", + ":pulsar-common", + ":pulsar-config-validation", + ":pulsar-dependencies", + ":pulsar-docs-tools", + ":pulsar-functions:pulsar-functions-api", + ":pulsar-functions:pulsar-functions-instance", + ":pulsar-functions:pulsar-functions-proto", + ":pulsar-functions:pulsar-functions-runtime", + ":pulsar-functions:pulsar-functions-secrets", + ":pulsar-functions:pulsar-functions-utils", + ":pulsar-functions:pulsar-functions-worker", + ":pulsar-http-client-api", + ":pulsar-io:pulsar-io-core", + ":pulsar-metadata", + ":pulsar-opentelemetry", + ":pulsar-package-management:pulsar-package-core", + ":pulsar-package-management:pulsar-package-filesystem-storage", + ":pulsar-proxy", + ":pulsar-tls-factory-api", + ":pulsar-transaction:pulsar-transaction-common", + ":pulsar-transaction:pulsar-transaction-coordinator", + ":pulsar-websocket", + ":testmocks", + ) + + fun isEnabled(project: Project): Boolean = + project.providers.gradleProperty("publishApiAndSpiOnly").getOrElse("false").toBoolean() + + fun includes(project: Project): Boolean = !isEnabled(project) || project.path in projects +} diff --git a/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt b/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt new file mode 100644 index 0000000000000..e9ae0714081e9 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction +import org.gradle.work.DisableCachingByDefault +import org.w3c.dom.Element +import javax.xml.parsers.DocumentBuilderFactory + +/** Checks the consumer graph, including publication-only rewrites and dependency-reduced shaded POMs. */ +@DisableCachingByDefault(because = "Verification has no outputs") +abstract class ValidateApiSpiPublication : DefaultTask() { + @get:InputFiles + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val poms: ConfigurableFileCollection + + @get:InputFiles + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val moduleMetadata: ConfigurableFileCollection + + @get:Input + abstract val publicationGroup: Property + + @TaskAction + fun validate() { + val factory = DocumentBuilderFactory.newInstance() + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + fun Element.childText(name: String): String = (0 until childNodes.length) + .map { childNodes.item(it) }.filterIsInstance() + .firstOrNull { it.tagName == name }?.textContent.orEmpty() + val documents = poms.files.sorted().associateWith { + factory.newDocumentBuilder().parse(it).documentElement + } + val coordinates = documents.values.map { + "${it.childText("groupId")}:${it.childText("artifactId")}:${it.childText("version")}" + }.toSet() + val failures = sortedSetOf() + fun check(source: String, group: String, artifact: String, version: String) { + if (group == publicationGroup.get() || group == "org.apache.pulsar") { + val coordinate = "$group:$artifact:$version" + if (coordinate !in coordinates) { + failures.add("$source -> $coordinate") + } + } + } + for ((file, root) in documents) { + val source = root.childText("artifactId") + " (" + file.name + ")" + if (root.childText("groupId") != publicationGroup.get()) { + failures.add("$source publishes under ${root.childText("groupId")} instead of ${publicationGroup.get()}") + } + for (tag in listOf("parent", "dependency")) { + val nodes = root.getElementsByTagName(tag) + for (i in 0 until nodes.length) { + val node = nodes.item(i) as Element + check(source, node.childText("groupId"), node.childText("artifactId"), node.childText("version")) + } + } + } + for (file in moduleMetadata.files.sorted()) { + val json = JsonSlurper().parse(file) as Map<*, *> + val component = json["component"] as Map<*, *> + val source = "${component["module"]} (Gradle metadata)" + for (variant in json["variants"] as List<*>) { + val data = variant as Map<*, *> + for (key in listOf("dependencies", "dependencyConstraints")) { + for (entry in data[key] as? List<*> ?: emptyList()) { + val dep = entry as Map<*, *> + val version = dep["version"] as? Map<*, *> + check(source, dep["group"].toString(), dep["module"].toString(), + (version?.get("strictly") ?: version?.get("requires") ?: version?.get("prefers")).toString()) + } + } + (data["available-at"] as? Map<*, *>)?.let { + check(source, it["group"].toString(), it["module"].toString(), it["version"].toString()) + } + } + } + if (failures.isNotEmpty()) { + throw GradleException("API/SPI publication contains unpublished Pulsar dependencies:\n" + + failures.joinToString("\n") + + "\nAdd the missing projects to PulsarApiSpiPublication.projects or correct their published coordinates.") + } + logger.lifecycle("Validated {} API/SPI publications and their published dependency closure.", coordinates.size) + } +} diff --git a/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts index c0c9c1b7eb424..c71df08bc18bf 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts @@ -50,13 +50,14 @@ val pulsarPlatformModules = setOf( "pulsar-package-core", ) +val pulsarGroup = project.group.toString() configurations.named("runtimeClasspath") { exclude(group = "org.apache.bookkeeper") // Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle it. // Bundling a different version causes GeneratedMessage.getUnknownFields() conflicts. exclude(group = "com.google.protobuf") pulsarPlatformModules.forEach { module -> - exclude(group = "org.apache.pulsar", module = module) + exclude(group = pulsarGroup, module = module) } } @@ -97,7 +98,7 @@ if (parentProject != null && parentProject != rootProject && parentProject.paren // NAR modules bundle all dependencies, so the POM should have no section. publishing { publications { - named("maven") { + withType().configureEach { // Replace component-based artifacts with just the NAR file artifacts.clear() artifact(tasks.named("nar")) diff --git a/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts index 5841e7d70baa5..ac3cac573f969 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts @@ -30,7 +30,8 @@ plugins { // in scopes that end up in the published POM (api, implementation, runtimeOnly). // Test/compileOnly scoped dependencies are excluded since they don't appear in the POM. // NAR modules are not validated here — they bundle all dependencies and have empty POMs. -run { +// API/SPI mode validates the generated consumer metadata for the entire selection before uploading. +if (!PulsarApiSpiPublication.isEnabled(project)) { val publishedScopes = listOf("api", "implementation", "runtimeOnly", "shadow", "shadowApi") val configsToCheck = publishedScopes.mapNotNull { name -> configurations.findByName(name)?.let { name to it } diff --git a/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts index 58b11e0161278..253fd5b6ee338 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts @@ -28,6 +28,9 @@ plugins { // --- java-library projects: JAR + sources + javadoc --- pluginManager.withPlugin("java-library") { + if (!PulsarApiSpiPublication.includes(project)) { + return@withPlugin + } val sourceSets = the() // Match Maven's javadoc configuration: no doclint, don't fail on errors @@ -90,6 +93,9 @@ pluginManager.withPlugin("java-library") { // --- java-platform projects (BOM, dependencies): POM-only, no JAR --- pluginManager.withPlugin("java-platform") { + if (!PulsarApiSpiPublication.includes(project)) { + return@withPlugin + } publishing { publications { create("maven") { @@ -107,6 +113,7 @@ run { val isPlatformProject = plugins.hasPlugin("java-platform") val isRootProject = project == rootProject val pulsarVersion = version.toString() + val pulsarGroup = project.group.toString() // Per-module POM name and description. Read in afterEvaluate so that a description // assigned in a module's build script body is picked up, and captured as plain strings @@ -162,7 +169,7 @@ run { s = s.replace( "4.0.0", "4.0.0\n \n" + - " org.apache.pulsar\n" + + " $pulsarGroup\n" + " pulsar\n" + " $pulsarVersion\n" + " " diff --git a/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts index 4fe98e0e1ad0d..2ace22ea59a2e 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts @@ -167,3 +167,34 @@ tasks.withType().configureEach { providers.gradleProperty("signing.gnupg.keyName").isPresent || (providers.gradleProperty("useGpgCmd").orNull?.toBoolean() ?: false) } + +// Every upload in API/SPI mode waits for the entire selected consumer graph to pass validation. +// String task dependencies also configure the selected projects with configure-on-demand enabled. +if (PulsarApiSpiPublication.isEnabled(project)) { + if (project == rootProject) { + tasks.register("validateApiSpiPublication") { + group = "verification" + description = "Validate that every published Pulsar dependency belongs to the API/SPI publication set." + publicationGroup.set(project.group.toString()) + for (selectedPath in PulsarApiSpiPublication.projects) { + val selected = project(selectedPath) + // A qualified task invocation does not discover/configure container projects. + // Initialize their Kotlin script scopes before Gradle resolves nested task paths. + generateSequence(selected.parent) { it.parent } + .takeWhile { it != rootProject }.toList().asReversed().forEach { + project.evaluationDependsOn(it.path) + } + val prefix = if (selectedPath == ":") "" else selectedPath + poms.from(selected.layout.buildDirectory.file("publications/maven/pom-default.xml")) + dependsOn("$prefix:generatePomFileForMavenPublication") + if (selectedPath != ":") { + moduleMetadata.from(selected.layout.buildDirectory.file("publications/maven/module.json")) + dependsOn("$prefix:generateMetadataFileForMavenPublication") + } + } + } + } + tasks.withType().configureEach { + dependsOn(":validateApiSpiPublication") + } +} diff --git a/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt b/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt new file mode 100644 index 0000000000000..cda695d6f8212 --- /dev/null +++ b/build-logic/conventions/src/test/kotlin/ValidateApiSpiPublicationTest.kt @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.GradleException +import org.gradle.testfixtures.ProjectBuilder +import org.testng.Assert.assertTrue +import org.testng.Assert.expectThrows +import org.testng.annotations.Test +import java.nio.file.Files + +class ValidateApiSpiPublicationTest { + private fun verify(pomDependency: String = "", metadataDependency: String = "", parent: String = "") { + val directory = Files.createTempDirectory("api-spi-publication-test").toFile() + try { + val project = ProjectBuilder.builder().withProjectDir(directory).build() + val task = project.tasks.register("validate", ValidateApiSpiPublication::class.java).get() + task.publicationGroup.set("example.pulsar") + val pom = directory.resolve("pom.xml") + pom.writeText(""" + 4.0.0 + $parent + example.pulsarclient1 + $pomDependency + """.trimIndent()) + val module = directory.resolve("module.json") + module.writeText(""" + {"component":{"group":"example.pulsar","module":"client","version":"1"}, + "variants":[{"name":"runtime","dependencies":[$metadataDependency]}]} + """.trimIndent()) + task.poms.from(pom) + task.moduleMetadata.from(module) + task.validate() + } finally { + directory.deleteRecursively() + } + } + + @Test + fun acceptsClosedGraphAndThirdPartyDependencies() { + verify( + "example.pulsarclient" + + "1", + """{"group":"third.party","module":"library","version":{"requires":"2"}}""" + ) + } + + @Test + fun rejectsMissingPomDependency() { + val error = expectThrows(GradleException::class.java) { + verify("example.pulsarforgotten" + + "1") + } + assertTrue(error.message!!.contains("client (pom.xml) -> example.pulsar:forgotten:1")) + } + + @Test + fun rejectsMissingMetadataDependency() { + val error = expectThrows(GradleException::class.java) { + verify(metadataDependency = + """{"group":"example.pulsar","module":"forgotten","version":{"requires":"1"}}""") + } + assertTrue(error.message!!.contains("client (Gradle metadata) -> example.pulsar:forgotten:1")) + } + + @Test + fun rejectsOriginalGroupAndMissingParent() { + val error = expectThrows(GradleException::class.java) { + verify(parent = "org.apache.pulsarpulsar" + + "1") + } + assertTrue(error.message!!.contains("org.apache.pulsar:pulsar:1")) + } + + @Test + fun rejectsWrongInternalVersion() { + val error = expectThrows(GradleException::class.java) { + verify(metadataDependency = + """{"group":"example.pulsar","module":"client","version":{"strictly":"2"}}""") + } + assertTrue(error.message!!.contains("example.pulsar:client:2")) + } +} diff --git a/buildtools/build.gradle.kts b/buildtools/build.gradle.kts index 8496a0bef970a..af4dc417f66f0 100644 --- a/buildtools/build.gradle.kts +++ b/buildtools/build.gradle.kts @@ -21,6 +21,11 @@ plugins { id("pulsar.java-conventions") } +// External consumers may use the test support classes without publishing them in normal releases. +if (PulsarApiSpiPublication.isEnabled(project)) { + apply(plugin = "pulsar.publish-conventions") +} + dependencies { implementation(libs.slog) implementation(libs.snakeyaml) diff --git a/pulsar-bom/build.gradle.kts b/pulsar-bom/build.gradle.kts index d8ec476f4ffc9..07c69fd698ab6 100644 --- a/pulsar-bom/build.gradle.kts +++ b/pulsar-bom/build.gradle.kts @@ -33,84 +33,91 @@ javaPlatform { dependencies { constraints { + fun selectedApi(dependency: ProjectDependency) { + if (!PulsarApiSpiPublication.isEnabled(project) || + dependency.path in PulsarApiSpiPublication.projects) { + add("api", dependency) + } + } + // Client API - api(project(":pulsar-client-api")) - api(project(":pulsar-client-admin-api")) - api(project(":pulsar-client-api-v5")) + selectedApi(project(":pulsar-client-api")) + selectedApi(project(":pulsar-client-admin-api")) + selectedApi(project(":pulsar-client-api-v5")) // Focused SPI modules (PIP-478): TLS factory SPI + HTTP client SPI - api(project(":pulsar-tls-factory-api")) - api(project(":pulsar-http-client-api")) + selectedApi(project(":pulsar-tls-factory-api")) + selectedApi(project(":pulsar-http-client-api")) // Shaded clients (the published artifacts users depend on) - api(project(":pulsar-client-shaded")) - api(project(":pulsar-client-admin-shaded")) - api(project(":pulsar-client-all")) - api(project(":pulsar-client-v5-shaded")) + selectedApi(project(":pulsar-client-shaded")) + selectedApi(project(":pulsar-client-admin-shaded")) + selectedApi(project(":pulsar-client-all")) + selectedApi(project(":pulsar-client-v5-shaded")) // Combined unshaded v4/v5 client and admin - api(project(":pulsar-client-v5-all")) + selectedApi(project(":pulsar-client-v5-all")) // Original (unshaded) clients - api(project(":pulsar-client-v5")) - api(project(":pulsar-client-original")) - api(project(":pulsar-client-admin-original")) + selectedApi(project(":pulsar-client-v5")) + selectedApi(project(":pulsar-client-original")) + selectedApi(project(":pulsar-client-admin-original")) // Client auth - api(project(":pulsar-client-auth-sasl")) - api(project(":pulsar-client-messagecrypto-bc")) + selectedApi(project(":pulsar-client-auth-sasl")) + selectedApi(project(":pulsar-client-messagecrypto-bc")) // Common - api(project(":pulsar-common")) - api(project(":pulsar-config-validation")) + selectedApi(project(":pulsar-common")) + selectedApi(project(":pulsar-config-validation")) // Functions API - api(project(":pulsar-functions:pulsar-functions-api")) + selectedApi(project(":pulsar-functions:pulsar-functions-api")) // IO core - api(project(":pulsar-io:pulsar-io-core")) - api(project(":pulsar-io:pulsar-io-common")) + selectedApi(project(":pulsar-io:pulsar-io-core")) + selectedApi(project(":pulsar-io:pulsar-io-common")) // Broker - api(project(":pulsar-broker")) - api(project(":pulsar-broker-common")) - api(project(":pulsar-broker-auth-oidc")) - api(project(":pulsar-broker-auth-sasl")) + selectedApi(project(":pulsar-broker")) + selectedApi(project(":pulsar-broker-common")) + selectedApi(project(":pulsar-broker-auth-oidc")) + selectedApi(project(":pulsar-broker-auth-sasl")) // Other core modules - api(project(":managed-ledger")) - api(project(":pulsar-metadata")) - api(project(":pulsar-proxy")) - api(project(":pulsar-websocket")) - api(project(":pulsar-testclient")) - api(project(":pulsar-cli-utils")) - api(project(":pulsar-client-tools")) - api(project(":pulsar-client-tools-api")) - api(project(":pulsar-opentelemetry")) - api(project(":testmocks")) + selectedApi(project(":managed-ledger")) + selectedApi(project(":pulsar-metadata")) + selectedApi(project(":pulsar-proxy")) + selectedApi(project(":pulsar-websocket")) + selectedApi(project(":pulsar-testclient")) + selectedApi(project(":pulsar-cli-utils")) + selectedApi(project(":pulsar-client-tools")) + selectedApi(project(":pulsar-client-tools-api")) + selectedApi(project(":pulsar-opentelemetry")) + selectedApi(project(":testmocks")) // Transaction - api(project(":pulsar-transaction:pulsar-transaction-common")) - api(project(":pulsar-transaction:pulsar-transaction-coordinator")) + selectedApi(project(":pulsar-transaction:pulsar-transaction-common")) + selectedApi(project(":pulsar-transaction:pulsar-transaction-coordinator")) // Functions - api(project(":pulsar-functions:pulsar-functions-instance")) - api(project(":pulsar-functions:pulsar-functions-runtime")) - api(project(":pulsar-functions:pulsar-functions-worker")) - api(project(":pulsar-functions:pulsar-functions-local-runner-original")) - api(project(":pulsar-functions:pulsar-functions-proto")) - api(project(":pulsar-functions:pulsar-functions-secrets")) - api(project(":pulsar-functions:pulsar-functions-utils")) + selectedApi(project(":pulsar-functions:pulsar-functions-instance")) + selectedApi(project(":pulsar-functions:pulsar-functions-runtime")) + selectedApi(project(":pulsar-functions:pulsar-functions-worker")) + selectedApi(project(":pulsar-functions:pulsar-functions-local-runner-original")) + selectedApi(project(":pulsar-functions:pulsar-functions-proto")) + selectedApi(project(":pulsar-functions:pulsar-functions-secrets")) + selectedApi(project(":pulsar-functions:pulsar-functions-utils")) // Athenz auth - api(project(":pulsar-client-auth-athenz")) - api(project(":pulsar-broker-auth-athenz")) + selectedApi(project(":pulsar-client-auth-athenz")) + selectedApi(project(":pulsar-broker-auth-athenz")) // Functions - api(project(":pulsar-functions:pulsar-functions-local-runner-shaded")) + selectedApi(project(":pulsar-functions:pulsar-functions-local-runner-shaded")) // Tiered storage - api(project(":tiered-storage:tiered-storage-jcloud")) - api(project(":tiered-storage:tiered-storage-file-system")) + selectedApi(project(":tiered-storage:tiered-storage-jcloud")) + selectedApi(project(":tiered-storage:tiered-storage-file-system")) } } diff --git a/pulsar-client/build.gradle.kts b/pulsar-client/build.gradle.kts index 6c802c39735b5..d5c4982b5176c 100644 --- a/pulsar-client/build.gradle.kts +++ b/pulsar-client/build.gradle.kts @@ -164,7 +164,7 @@ tasks.named("compileTestJava") { dependsOn(generateTestAvro) } run { val fromGroup = "it.unimi.dsi" val fromName = "fastutil" - val toGroup = "org.apache.pulsar" + val toGroup = project.group.toString() val toName = "pulsar-client-fastutil-minimized" val toVersion = version.toString() diff --git a/pulsar-functions/localrun-shaded/build.gradle.kts b/pulsar-functions/localrun-shaded/build.gradle.kts index 598ed425bc817..52a759f5ecbea 100644 --- a/pulsar-functions/localrun-shaded/build.gradle.kts +++ b/pulsar-functions/localrun-shaded/build.gradle.kts @@ -27,12 +27,13 @@ dependencies { } val shadePrefix = "org.apache.pulsar.functions.runtime.shaded" +val pulsarGroupPattern = Regex.escape(project.group.toString()) tasks.shadowJar { isZip64 = true dependencies { - include(dependency("org.apache.pulsar:.*")) + include(dependency("$pulsarGroupPattern:.*")) include(project(":pulsar-functions:pulsar-functions-local-runner-original")) include(project(":pulsar-client-original")) include(project(":pulsar-common"))