diff --git a/CHANGELOG.md b/CHANGELOG.md index 37752dbd8..bcdfc15c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file. - opa: Add 1.20.2 ([#1697]). - patchable: Use long hashes, and myers diff algorithm for portability ([#1705]). - spark: Add 4.1.3 and 4.2.0 ([#1710]). +- spark: Add HBase connector images for Spark 4.x. There is no connector release that supports Spark 4.x yet, so this is built from an upstream commit (`07f5791`) on the master branch ([#1723]) ### Changed @@ -184,6 +185,7 @@ All notable changes to this project will be documented in this file. [#1718]: https://github.com/stackabletech/docker-images/pull/1718 [#1720]: https://github.com/stackabletech/docker-images/pull/1720 [#1721]: https://github.com/stackabletech/docker-images/pull/1721 +[#1723]: https://github.com/stackabletech/docker-images/pull/1723 [#1725]: https://github.com/stackabletech/docker-images/pull/1725 ## [26.7.0] - 2026-07-21 diff --git a/spark-k8s/Dockerfile.4 b/spark-k8s/Dockerfile.4 index 5bde63646..5dd814b2a 100644 --- a/spark-k8s/Dockerfile.4 +++ b/spark-k8s/Dockerfile.4 @@ -4,6 +4,9 @@ # hadoop-builder: Provides Hadoop libraries FROM local-image/hadoop/hadoop AS hadoop-builder +# hbase-builder: Provides HBase libraries +FROM local-image/hbase AS hbase-builder + # spark-source-builder: Clone the Spark source code and apply patches FROM local-image/java-devel AS spark-source-builder @@ -28,6 +31,10 @@ tar -czf /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar chmod g=u /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz EOF +# hbase-connectors-builder: Build the Spark HBase connector and copy +# required JARs into /stackable/spark/jars +FROM local-image/spark-k8s/hbase-connectors AS hbase-connectors-builder + # spark-builder: Build Spark into /stackable/spark-${PRODUCT_VERSION}/dist, # download additional JARs and perform checks FROM local-image/java-devel AS spark-builder @@ -36,6 +43,7 @@ ARG PRODUCT_VERSION ARG HADOOP_HADOOP_VERSION # Reassign the arg to `HADOOP_VERSION` for better readability. ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} +ARG HBASE_VERSION ARG JACKSON_DATAFORMAT_XML_VERSION ARG STAX2_API_VERSION ARG WOODSTOX_CORE_VERSION @@ -136,6 +144,22 @@ WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/jar # Copy cloud libraries for s3a:// and abfs:// support COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-cloud-libraries/*.jar ./ +# Copy the HBase connector including required modules +COPY --from=hbase-connectors-builder --chown=${STACKABLE_USER_UID}:0 \ + /stackable/spark/jars/* \ + ./ + +# Copy modules required to access HBase +COPY --from=hbase-builder --chown=${STACKABLE_USER_UID}:0 \ + /stackable/hbase/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${HBASE_VERSION}-stackable${RELEASE_VERSION}.jar \ + /stackable/hbase/lib/shaded-clients/hbase-shaded-mapreduce-${HBASE_VERSION}-stackable${RELEASE_VERSION}.jar \ + ./ +COPY --from=hbase-builder --chown=${STACKABLE_USER_UID}:0 \ + /stackable/hbase/lib/client-facing-thirdparty/opentelemetry-api-*.jar \ + /stackable/hbase/lib/client-facing-thirdparty/opentelemetry-context-*.jar \ + /stackable/hbase/lib/client-facing-thirdparty/opentelemetry-semconv-*-alpha.jar \ + ./ + COPY spark-k8s/stackable/jmx /stackable/jmx WORKDIR /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist/extra-jars @@ -191,6 +215,7 @@ ARG PRODUCT_VERSION ARG PYTHON_VERSION ARG RELEASE_VERSION ARG JMX_EXPORTER_VERSION +ARG HBASE_CONNECTOR_VERSION ARG STACKABLE_USER_UID LABEL name="Apache Spark" \ @@ -213,6 +238,7 @@ ENV PYTHONPATH=$SPARK_HOME/python COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/dist /stackable/spark COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-connectors-builder /stackable/hbase-connector-${HBASE_CONNECTOR_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/assembly/target/bom.json /stackable/spark/spark-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.cdx.json COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-web-ui-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.cdx.json /stackable/spark/spark-web-ui-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}.cdx.json COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/jmx /stackable/jmx diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 8de6613b1..415269625 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -8,6 +8,8 @@ containerfile = "Dockerfile.4" "hadoop/hadoop" = "3.4.3" java-base = "21" java-devel = "21" +hbase = "2.6.7" +"spark-k8s/hbase-connectors" = "1.1.0-SNAPSHOT-07f5791_4.1.2" [versions."4.1.2".build-arguments] python-version = "3.12" @@ -16,6 +18,7 @@ stax2-api-version = "4.2.2" # Needs to match https://mvnrepository woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" +hbase-connector-version = "1.1.0-SNAPSHOT-07f5791_4.1.2" openlineage-spark-version = "1.54.0" # Scala 2.13 build for Spark 4.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13 # LTS since 26.11.0 @@ -26,6 +29,8 @@ containerfile = "Dockerfile.4" "hadoop/hadoop" = "3.4.3" java-base = "21" java-devel = "21" +hbase = "2.6.7" +"spark-k8s/hbase-connectors" = "1.1.0-SNAPSHOT-07f5791_4.1.3" [versions."4.1.3".build-arguments] python-version = "3.12" @@ -34,6 +39,7 @@ stax2-api-version = "4.2.2" # Needs to match https://mvnrepository woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" +hbase-connector-version = "1.1.0-SNAPSHOT-07f5791_4.1.3" openlineage-spark-version = "1.54.0" # Scala 2.13 build for Spark 4.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13 [versions."4.2.0"] @@ -43,6 +49,8 @@ containerfile = "Dockerfile.4" "hadoop/hadoop" = "3.5.0" java-base = "25" java-devel = "25" +hbase = "2.6.7" +"spark-k8s/hbase-connectors" = "1.1.0-SNAPSHOT-07f5791_4.2.0" [versions."4.2.0".build-arguments] python-version = "3.14" @@ -51,4 +59,5 @@ stax2-api-version = "4.2.2" # Needs to match https://mvnrepository woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.21.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" +hbase-connector-version = "1.1.0-SNAPSHOT-07f5791_4.2.0" openlineage-spark-version = "1.54.0" # Scala 2.13 build for Spark 4.x; see https://mvnrepository.com/artifact/io.openlineage/openlineage-spark_2.13 diff --git a/spark-k8s/hbase-connectors/Dockerfile b/spark-k8s/hbase-connectors/Dockerfile index 8d20f9159..3f78ae722 100644 --- a/spark-k8s/hbase-connectors/Dockerfile +++ b/spark-k8s/hbase-connectors/Dockerfile @@ -55,8 +55,7 @@ ARG RELEASE_VERSION ARG HADOOP_VERSION ARG HBASE_VERSION ARG HBASE_THIRDPARTY_VERSION -ARG PROTOBUF_VERSION -ARG PROTOBUF_PLUGIN_VERSION +ARG THIRDPARTY_PROTOBUF_VERSION ARG STACKABLE_USER_UID # Patch the hbase-connectors source code @@ -74,27 +73,21 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \ spark/env RUN <-_`. -[versions."1.0.1_3.5.8".build-arguments] -spark-version = "3.5.8" -# TODO: Bump to 3.4.3. Note that this causes compilation errors that need to be fixed. -hadoop-version = "3.4.2" -hbase-version = "2.6.6" -# The following three must match the hbase-thirdparty (and its bundled protobuf) +[versions."1.1.0-SNAPSHOT-07f5791_4.1.2".local-images] +java-devel = "21" + +[versions."1.1.0-SNAPSHOT-07f5791_4.1.2".build-arguments] +spark-version = "4.1.2" +hadoop-version = "3.4.3" +hbase-version = "2.6.7" +# The following two must match the hbase-thirdparty (and its bundled protobuf) # shipped by the hbase-shaded-client in the final Spark image. That client comes -# from the `hbase` local-image in spark-k8s/boil-config.toml (currently 2.6.6), -# whose hbase-thirdparty is 4.1.13 -> protobuf 4.34.0. Keep these in sync if the +# from the `hbase` local-image in spark-k8s/boil-config.toml (currently 2.6.7), +# whose hbase-thirdparty is 4.1.14 -> protobuf 4.34.0. Keep these in sync if the # Spark image's HBASE_VERSION changes. -hbase-thirdparty-version = "4.1.13" -protobuf-version = "4.34.0" -protobuf-plugin-version = "0.6.1" +hbase-thirdparty-version = "4.1.14" +thirdparty-protobuf-version = "4.34.0" + +[versions."1.1.0-SNAPSHOT-07f5791_4.1.3".local-images] +java-devel = "21" + +[versions."1.1.0-SNAPSHOT-07f5791_4.1.3".build-arguments] +spark-version = "4.1.3" +hadoop-version = "3.4.3" +hbase-version = "2.6.7" +# See the comment above. +hbase-thirdparty-version = "4.1.14" +thirdparty-protobuf-version = "4.34.0" + +[versions."1.1.0-SNAPSHOT-07f5791_4.2.0".local-images] +java-devel = "25" + +[versions."1.1.0-SNAPSHOT-07f5791_4.2.0".build-arguments] +spark-version = "4.2.0" +hadoop-version = "3.5.0" +hbase-version = "2.6.7" +# See the comment above. +hbase-thirdparty-version = "4.1.14" +thirdparty-protobuf-version = "4.34.0" diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch deleted file mode 100644 index 336444293..000000000 --- a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch +++ /dev/null @@ -1,59 +0,0 @@ -From 8211177df0ca6cae00bc3bebcd264364732691b3 Mon Sep 17 00:00:00 2001 -From: dervoeti -Date: Thu, 13 Mar 2025 22:10:56 +0100 -Subject: Fix protobuf on aarch64 - ---- - pom.xml | 19 ++++++++++++++++++- - spark/pom.xml | 2 +- - 2 files changed, 19 insertions(+), 2 deletions(-) - -diff --git a/pom.xml b/pom.xml -index e849cd167ccb42b7cd4f8c5c912a03e7cf15a9e7..f514e14d4d1d9c3e65ae3bed624494ab52eda31e 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -157,7 +157,10 @@ - 1.5.1 - 0.14.0 - -- 2.5.0 -+ -+ com.google.protobuf -+ 2.6.1 -+ ${external.protobuf.version} - 0.5.0 - 2.11.0 - 1.7.7 -@@ -933,5 +936,19 @@ - - - -+ -+ -+ aarch64 -+ -+ com.github.os72 -+ 2.6.1-build3 -+ -+ -+ -+ linux -+ aarch64 -+ -+ -+ - - -diff --git a/spark/pom.xml b/spark/pom.xml -index 3f1eb21c42d00cc39e6af8b1fa32b5fd45eaee30..fcdc73eda8b55716c19972c0a7b2c17b63ccdfa1 100644 ---- a/spark/pom.xml -+++ b/spark/pom.xml -@@ -84,7 +84,7 @@ - protobuf-maven-plugin - ${protobuf.plugin.version} - -- com.google.protobuf:protoc:${external.protobuf.version}:exe:${os.detected.classifier} -+ ${external.protobuf.groupId}:protoc:${external.protobuf.exe.version}:exe:${os.detected.classifier} - ${basedir}/src/main/protobuf/ - false - true diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.2/patchable.toml similarity index 56% rename from spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml rename to spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.2/patchable.toml index 6d8c2c8db..5ce87edde 100644 --- a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml +++ b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.2/patchable.toml @@ -1,2 +1,2 @@ -base = "e5217d13ed729703580ff2d1b02378ada2d94f4d" mirror = "https://github.com/stackabletech/hbase-connectors.git" +base = "07f5791c4921e471d328c8ce5f760ac72f0090cb" diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.3/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.3/patchable.toml new file mode 100644 index 000000000..5ce87edde --- /dev/null +++ b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.1.3/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/hbase-connectors.git" +base = "07f5791c4921e471d328c8ce5f760ac72f0090cb" diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.2.0/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.2.0/patchable.toml new file mode 100644 index 000000000..5ce87edde --- /dev/null +++ b/spark-k8s/hbase-connectors/stackable/patches/1.1.0-SNAPSHOT-07f5791_4.2.0/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/hbase-connectors.git" +base = "07f5791c4921e471d328c8ce5f760ac72f0090cb"