[improve][build] Add validated API/SPI publication subset

.github/workflows/pulsar-ci.yaml

@@ -200,6 +200,12 @@ jobs:
       - name: Check that project's public libraries can be published to a Maven repository
         run: ./gradlew publishAllPublicationsToLocalDeployRepository
 
+      - name: Check API/SPI publication dependency closure with a custom group
+        # Validate generated POMs and Gradle metadata, including the BOM, without uploading artifacts.
+        run: >-
+          ./gradlew validateApiSpiPublication
+          -PpublishApiAndSpiOnly=true -Pgroup=myorg.pulsar
+
       - name: Upload Gradle reports
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

CONTRIBUTING.md

@@ -63,6 +63,11 @@ For the Gradle build infrastructure and how to change build files (convention pl
 catalog, configuration-cache rules), see
 [`ARCHITECTURE.md` → Build infrastructure](ARCHITECTURE.md#build-infrastructure).
 
+### Publishing artifacts
+
+See [Publishing Maven artifacts](build-logic/PUBLISHING.md) for ordinary publishing, the
+API/SPI subset, custom repositories, groups and credentials.
+
 ## Running tests
 
 Most of these per-module "unit tests" are actually **integration-style** — they start a real in-JVM

build-logic/conventions/build.gradle.kts

@@ -22,6 +22,7 @@ plugins {
 }
 
 dependencies {
+    testImplementation(libs.testng)
     // The Shadow plugin brings its own log4j-core onto the build classpath; align it with the
     // log4j version the rest of the build uses (`log4j2` in the version catalog).
     implementation(platform(libs.log4j.bom))
@@ -35,3 +36,7 @@ dependencies {
         "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}"
     })
 }
+
+tasks.test {
+    useTestNG()
+}

build-logic/conventions/src/main/kotlin/PulsarApiSpiPublication.kt

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Project
+
+/** API/SPI publication set for Java clients, Functions and plugins, including their published dependencies. */
+object PulsarApiSpiPublication {
+    val projects: Set<String> = setOf(
+        ":",
+        ":buildtools",
+        ":managed-ledger",
+        ":pulsar-bom",
+        ":pulsar-broker",
+        ":pulsar-broker-auth-sasl",
+        ":pulsar-broker-common",
+        ":pulsar-cli-utils",
+        ":pulsar-client-admin-api",
+        ":pulsar-client-admin-original",
+        ":pulsar-client-admin-shaded",
+        ":pulsar-client-all",
+        ":pulsar-client-api",
+        ":pulsar-client-api-v5",
+        ":pulsar-client-auth-sasl",
+        ":pulsar-client-fastutil-minimized",
+        ":pulsar-client-messagecrypto-bc",
+        ":pulsar-client-original",
+        ":pulsar-client-shaded",
+        ":pulsar-client-v5",
+        ":pulsar-client-v5-all",
+        ":pulsar-client-v5-shaded",
+        ":pulsar-common",
+        ":pulsar-config-validation",
+        ":pulsar-dependencies",
+        ":pulsar-docs-tools",
+        ":pulsar-functions:pulsar-functions-api",
+        ":pulsar-functions:pulsar-functions-instance",
+        ":pulsar-functions:pulsar-functions-proto",
+        ":pulsar-functions:pulsar-functions-runtime",
+        ":pulsar-functions:pulsar-functions-secrets",
+        ":pulsar-functions:pulsar-functions-utils",
+        ":pulsar-functions:pulsar-functions-worker",
+        ":pulsar-http-client-api",
+        ":pulsar-io:pulsar-io-core",
+        ":pulsar-metadata",
+        ":pulsar-opentelemetry",
+        ":pulsar-package-management:pulsar-package-core",
+        ":pulsar-package-management:pulsar-package-filesystem-storage",
+        ":pulsar-proxy",
+        ":pulsar-tls-factory-api",
+        ":pulsar-transaction:pulsar-transaction-common",
+        ":pulsar-transaction:pulsar-transaction-coordinator",
+        ":pulsar-websocket",
+        ":testmocks",
+    )
+
+    fun isEnabled(project: Project): Boolean =
+        project.providers.gradleProperty("publishApiAndSpiOnly").getOrElse("false").toBoolean()
+
+    fun includes(project: Project): Boolean = !isEnabled(project) || project.path in projects
+}

build-logic/conventions/src/main/kotlin/ValidateApiSpiPublication.kt

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import groovy.json.JsonSlurper
+import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
+import org.gradle.api.file.ConfigurableFileCollection
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.api.tasks.TaskAction
+import org.gradle.work.DisableCachingByDefault
+import org.w3c.dom.Element
+import javax.xml.parsers.DocumentBuilderFactory
+
+/** Checks the consumer graph, including publication-only rewrites and dependency-reduced shaded POMs. */
+@DisableCachingByDefault(because = "Verification has no outputs")
+abstract class ValidateApiSpiPublication : DefaultTask() {
+    @get:InputFiles
+    @get:PathSensitive(PathSensitivity.RELATIVE)
+    abstract val poms: ConfigurableFileCollection
+
+    @get:InputFiles
+    @get:PathSensitive(PathSensitivity.RELATIVE)
+    abstract val moduleMetadata: ConfigurableFileCollection
+
+    @get:Input
+    abstract val publicationGroup: Property<String>
+
+    @TaskAction
+    fun validate() {
+        val factory = DocumentBuilderFactory.newInstance()
+        factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
+        fun Element.childText(name: String): String = (0 until childNodes.length)
+            .map { childNodes.item(it) }.filterIsInstance<Element>()
+            .firstOrNull { it.tagName == name }?.textContent.orEmpty()
+        val documents = poms.files.sorted().associateWith {
+            factory.newDocumentBuilder().parse(it).documentElement
+        }
+        val coordinates = documents.values.map {
+            "${it.childText("groupId")}:${it.childText("artifactId")}:${it.childText("version")}"
+        }.toSet()
+        val failures = sortedSetOf<String>()
+        fun check(source: String, group: String, artifact: String, version: String) {
+            if (group == publicationGroup.get() || group == "org.apache.pulsar") {
+                val coordinate = "$group:$artifact:$version"
+                if (coordinate !in coordinates) {
+                    failures.add("$source -> $coordinate")
+                }
+            }
+        }
+        for ((file, root) in documents) {
+            val source = root.childText("artifactId") + " (" + file.name + ")"
+            if (root.childText("groupId") != publicationGroup.get()) {
+                failures.add("$source publishes under ${root.childText("groupId")} instead of ${publicationGroup.get()}")
+            }
+            for (tag in listOf("parent", "dependency")) {
+                val nodes = root.getElementsByTagName(tag)
+                for (i in 0 until nodes.length) {
+                    val node = nodes.item(i) as Element
+                    check(source, node.childText("groupId"), node.childText("artifactId"), node.childText("version"))
+                }
+            }
+        }
+        for (file in moduleMetadata.files.sorted()) {
+            val json = JsonSlurper().parse(file) as Map<*, *>
+            val component = json["component"] as Map<*, *>
+            val source = "${component["module"]} (Gradle metadata)"
+            for (variant in json["variants"] as List<*>) {
+                val data = variant as Map<*, *>
+                for (key in listOf("dependencies", "dependencyConstraints")) {
+                    for (entry in data[key] as? List<*> ?: emptyList<Any>()) {
+                        val dep = entry as Map<*, *>
+                        val version = dep["version"] as? Map<*, *>
+                        check(source, dep["group"].toString(), dep["module"].toString(),
+                            (version?.get("strictly") ?: version?.get("requires") ?: version?.get("prefers")).toString())
+                    }
+                }
+                (data["available-at"] as? Map<*, *>)?.let {
+                    check(source, it["group"].toString(), it["module"].toString(), it["version"].toString())
+                }
+            }
+        }
+        if (failures.isNotEmpty()) {
+            throw GradleException("API/SPI publication contains unpublished Pulsar dependencies:\n" +
+                failures.joinToString("\n") +
+                "\nAdd the missing projects to PulsarApiSpiPublication.projects or correct their published coordinates.")
+        }
+        logger.lifecycle("Validated {} API/SPI publications and their published dependency closure.", coordinates.size)
+    }
+}

build-logic/conventions/src/main/kotlin/pulsar.nar-conventions.gradle.kts

@@ -50,13 +50,14 @@ val pulsarPlatformModules = setOf(
     "pulsar-package-core",
 )
 
+val pulsarGroup = project.group.toString()
 configurations.named("runtimeClasspath") {
     exclude(group = "org.apache.bookkeeper")
     // Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle it.
     // Bundling a different version causes GeneratedMessage.getUnknownFields() conflicts.
     exclude(group = "com.google.protobuf")
     pulsarPlatformModules.forEach { module ->
-        exclude(group = "org.apache.pulsar", module = module)
+        exclude(group = pulsarGroup, module = module)
     }
 }
 
@@ -97,7 +98,7 @@ if (parentProject != null && parentProject != rootProject && parentProject.paren
 // NAR modules bundle all dependencies, so the POM should have no <dependencies> section.
 publishing {
     publications {
-        named<MavenPublication>("maven") {
+        withType<MavenPublication>().configureEach {
             // Replace component-based artifacts with just the NAR file
             artifacts.clear()
             artifact(tasks.named("nar"))

build-logic/conventions/src/main/kotlin/pulsar.public-java-library-conventions.gradle.kts

@@ -30,7 +30,8 @@ plugins {
 // in scopes that end up in the published POM (api, implementation, runtimeOnly).
 // Test/compileOnly scoped dependencies are excluded since they don't appear in the POM.
 // NAR modules are not validated here — they bundle all dependencies and have empty POMs.
-run {
+// API/SPI mode validates the generated consumer metadata for the entire selection before uploading.
+if (!PulsarApiSpiPublication.isEnabled(project)) {
     val publishedScopes = listOf("api", "implementation", "runtimeOnly", "shadow", "shadowApi")
     val configsToCheck = publishedScopes.mapNotNull { name ->
         configurations.findByName(name)?.let { name to it }

build-logic/conventions/src/main/kotlin/pulsar.publish-conventions.gradle.kts

@@ -28,6 +28,9 @@ plugins {
 
 // --- java-library projects: JAR + sources + javadoc ---
 pluginManager.withPlugin("java-library") {
+    if (!PulsarApiSpiPublication.includes(project)) {
+        return@withPlugin
+    }
     val sourceSets = the<SourceSetContainer>()
 
     // Match Maven's javadoc configuration: no doclint, don't fail on errors
@@ -90,6 +93,9 @@ pluginManager.withPlugin("java-library") {
 
 // --- java-platform projects (BOM, dependencies): POM-only, no JAR ---
 pluginManager.withPlugin("java-platform") {
+    if (!PulsarApiSpiPublication.includes(project)) {
+        return@withPlugin
+    }
     publishing {
         publications {
             create<MavenPublication>("maven") {
@@ -107,6 +113,7 @@ run {
     val isPlatformProject = plugins.hasPlugin("java-platform")
     val isRootProject = project == rootProject
     val pulsarVersion = version.toString()
+    val pulsarGroup = project.group.toString()
 
     // Per-module POM name and description. Read in afterEvaluate so that a description
     // assigned in a module's build script body is picked up, and captured as plain strings
@@ -162,7 +169,7 @@ run {
                             s = s.replace(
                                 "<modelVersion>4.0.0</modelVersion>",
                                 "<modelVersion>4.0.0</modelVersion>\n  <parent>\n" +
-                                    "    <groupId>org.apache.pulsar</groupId>\n" +
+                                    "    <groupId>$pulsarGroup</groupId>\n" +
                                     "    <artifactId>pulsar</artifactId>\n" +
                                     "    <version>$pulsarVersion</version>\n" +
                                     "  </parent>"

build-logic/conventions/src/main/kotlin/pulsar.publish-repositories-conventions.gradle.kts

@@ -167,3 +167,34 @@ tasks.withType<Sign>().configureEach {
         providers.gradleProperty("signing.gnupg.keyName").isPresent ||
         (providers.gradleProperty("useGpgCmd").orNull?.toBoolean() ?: false)
 }
+
+// Every upload in API/SPI mode waits for the entire selected consumer graph to pass validation.
+// String task dependencies also configure the selected projects with configure-on-demand enabled.
+if (PulsarApiSpiPublication.isEnabled(project)) {
+    if (project == rootProject) {
+        tasks.register<ValidateApiSpiPublication>("validateApiSpiPublication") {
+            group = "verification"
+            description = "Validate that every published Pulsar dependency belongs to the API/SPI publication set."
+            publicationGroup.set(project.group.toString())
+            for (selectedPath in PulsarApiSpiPublication.projects) {
+                val selected = project(selectedPath)
+                // A qualified task invocation does not discover/configure container projects.
+                // Initialize their Kotlin script scopes before Gradle resolves nested task paths.
+                generateSequence(selected.parent) { it.parent }
+                    .takeWhile { it != rootProject }.toList().asReversed().forEach {
+                        project.evaluationDependsOn(it.path)
+                    }
+                val prefix = if (selectedPath == ":") "" else selectedPath
+                poms.from(selected.layout.buildDirectory.file("publications/maven/pom-default.xml"))
+                dependsOn("$prefix:generatePomFileForMavenPublication")
+                if (selectedPath != ":") {
+                    moduleMetadata.from(selected.layout.buildDirectory.file("publications/maven/module.json"))
+                    dependsOn("$prefix:generateMetadataFileForMavenPublication")
+                }
+            }
+        }
+    }
+    tasks.withType<AbstractPublishToMaven>().configureEach {
+        dependsOn(":validateApiSpiPublication")
+    }
+}