From dcf1f349046ebe7eb2914ad0f47e3622d6d8485a Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:45:13 +0100 Subject: [PATCH 1/3] fix(spark): restore version 3.5.7 --- spark-connect-client/boil-config.toml | 7 ++ spark-k8s/boil-config.toml | 26 +++++- spark-k8s/hbase-connectors/boil-config.toml | 14 +++- .../3.5.7/0001-Update-CycloneDX-plugin.patch | 40 +++++++++ ...1311-BUILD-Promote-bcprov-jdk18on-to.patch | 83 +++++++++++++++++++ .../stackable/patches/3.5.7/patchable.toml | 2 + 6 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch create mode 100644 spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch create mode 100644 spark-k8s/stackable/patches/3.5.7/patchable.toml diff --git a/spark-connect-client/boil-config.toml b/spark-connect-client/boil-config.toml index 9aa478ed6..f1ac8fbc9 100644 --- a/spark-connect-client/boil-config.toml +++ b/spark-connect-client/boil-config.toml @@ -1,3 +1,10 @@ +[versions."3.5.7".local-images] +spark-k8s = "3.5.7" +java-base = "17" + +[versions."3.5.7".build-arguments] +python-version = "3.11" + [versions."3.5.8".local-images] spark-k8s = "3.5.8" java-base = "17" diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 1c9cf7522..8ea324386 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -1,3 +1,25 @@ +[versions."3.5.7"] +containerfile = "Dockerfile.3" + +[versions."3.5.7".local-images] +"hadoop/hadoop" = "3.4.2" +java-base = "17" +java-devel = "17" +hbase = "2.6.3" +"spark-k8s/hbase-connectors" = "1.0.1_3.5.7" + +[versions."3.5.7".build-arguments] +python-version = "3.11" +aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop +azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop +azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop +jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 +stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +jmx-exporter-version = "1.3.0" +tini-version = "0.19.0" +hbase-connector-version = "1.0.1_3.5.7" + [versions."3.5.8"] containerfile = "Dockerfile.3" @@ -6,7 +28,7 @@ containerfile = "Dockerfile.3" java-base = "17" java-devel = "17" hbase = "2.6.3" -"spark-k8s/hbase-connectors" = "1.0.1_3" +"spark-k8s/hbase-connectors" = "1.0.1_3.5.8" [versions."3.5.8".build-arguments] python-version = "3.11" @@ -18,7 +40,7 @@ stax2-api-version = "4.2.1" # needs to match the jackson version h woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" -hbase-connector-version = "1.0.1_3" +hbase-connector-version = "1.0.1_3.5.8" [versions."4.0.1"] containerfile = "Dockerfile.4" diff --git a/spark-k8s/hbase-connectors/boil-config.toml b/spark-k8s/hbase-connectors/boil-config.toml index 555044b50..6cf4c0791 100644 --- a/spark-k8s/hbase-connectors/boil-config.toml +++ b/spark-k8s/hbase-connectors/boil-config.toml @@ -1,10 +1,16 @@ -[versions."1.0.1_3"] -containerfile = "Dockerfile" -[versions."1.0.1_3".local-images] +[versions."1.0.1_3.5.7".local-images] java-devel = "17" -[versions."1.0.1_3".build-arguments] +[versions."1.0.1_3.5.7".build-arguments] +spark-version = "3.5.7" +hadoop-version = "3.4.2" +hbase-version = "2.6.3" + +[versions."1.0.1_3.5.8".local-images] +java-devel = "17" + +[versions."1.0.1_3.5.8".build-arguments] spark-version = "3.5.8" hadoop-version = "3.4.2" hbase-version = "2.6.3" diff --git a/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch new file mode 100644 index 000000000..b3c2d89f7 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch @@ -0,0 +1,40 @@ +From 5e3c5568d4bf0fdd4feb19e559d606a55a0d2569 Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Tue, 11 Mar 2025 17:29:39 +0200 +Subject: Update CycloneDX plugin + +--- + dev/make-distribution.sh | 1 - + pom.xml | 7 ++++++- + 2 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh +index ef7c010e93..0f4c1c74e4 100755 +--- a/dev/make-distribution.sh ++++ b/dev/make-distribution.sh +@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \ + -Dmaven.javadoc.skip=true \ + -Dmaven.scaladoc.skip=true \ + -Dmaven.source.skip \ +- -Dcyclonedx.skip=true \ + $@) + + # Actually build the jar +diff --git a/pom.xml b/pom.xml +index a0e25ce4d8..b3755a04e7 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -3534,7 +3534,12 @@ + + org.cyclonedx + cyclonedx-maven-plugin +- 2.7.9 ++ 2.8.0 ++ ++ application ++ 1.5 ++ false ++ + + + package diff --git a/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch b/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch new file mode 100644 index 000000000..4f5dbc208 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch @@ -0,0 +1,83 @@ +From 0cfb1d6b733579ea91df842e685df9db5512ece8 Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Tue, 22 Jul 2025 17:34:03 +0200 +Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope + +--- + LICENSE-binary | 1 + + assembly/pom.xml | 14 ++++++++++++++ + dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 + + licenses-binary/LICENSE-bouncycastle.txt | 13 +++++++++++++ + 4 files changed, 29 insertions(+) + create mode 100644 licenses-binary/LICENSE-bouncycastle.txt + +diff --git a/LICENSE-binary b/LICENSE-binary +index 05645977a0..9834cf333f 100644 +--- a/LICENSE-binary ++++ b/LICENSE-binary +@@ -480,6 +480,7 @@ org.typelevel:algebra_2.12:jar + org.typelevel:cats-kernel_2.12 + org.typelevel:machinist_2.12 + net.razorvine:pickle ++org.bouncycastle:bcprov-jdk18on + org.slf4j:jcl-over-slf4j + org.slf4j:jul-to-slf4j + org.slf4j:slf4j-api +diff --git a/assembly/pom.xml b/assembly/pom.xml +index ffa6a0e2b0..19032d6054 100644 +--- a/assembly/pom.xml ++++ b/assembly/pom.xml +@@ -85,8 +85,22 @@ + guava + ${hadoop.deps.scope} + ++ ++ ++ ++ org.bouncycastle ++ bcprov-jdk18on ++ ${hadoop.deps.scope} ++ ++ + + ++ + + + +diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 +index f110a1988f..88324d3e27 100644 +--- a/dev/deps/spark-deps-hadoop-3-hive-2.3 ++++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 +@@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar + azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar + azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar + azure-storage/7.0.1//azure-storage-7.0.1.jar ++bcprov-jdk18on/1.77//bcprov-jdk18on-1.77.jar + blas/3.0.3//blas-3.0.3.jar + bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar + breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar +diff --git a/licenses-binary/LICENSE-bouncycastle.txt b/licenses-binary/LICENSE-bouncycastle.txt +new file mode 100644 +index 0000000000..277dcd1ebb +--- /dev/null ++++ b/licenses-binary/LICENSE-bouncycastle.txt +@@ -0,0 +1,13 @@ ++Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). ++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and ++associated documentation files (the "Software"), to deal in the Software without restriction, ++including without limitation the rights to use, copy, modify, merge, publish, distribute, ++sub license, and/or sell copies of the Software, and to permit persons to whom the Software is ++furnished to do so, subject to the following conditions: The above copyright notice and this ++permission notice shall be included in all copies or substantial portions of the Software. ++ ++**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT ++NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, ++DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT ++OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** diff --git a/spark-k8s/stackable/patches/3.5.7/patchable.toml b/spark-k8s/stackable/patches/3.5.7/patchable.toml new file mode 100644 index 000000000..49202d309 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.7/patchable.toml @@ -0,0 +1,2 @@ +base = "ed00d046951a7ecda6429accd3b9c5b2dc792b65" +mirror = "https://github.com/stackabletech/spark.git" From da21e175e3ff5e1ab0edf93612d5a2b65687eb3c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:48:09 +0100 Subject: [PATCH 2/3] update changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b40a2e1a7..4751ead5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,8 +56,8 @@ All notable changes to this project will be documented in this file. - trino: Remove `451` and `476` ([#1403]). - hbase: Remove `2.6.2` ([#1408]). - spark: remove the JRE before installing the JDK to prevent version conflicts ([#1410]). -- spark: Remove `3.5.7` ([#1414]). -- spark-connect-client: Remove `3.5.7` ([#1414]). +- spark: Remove `3.5.7` in ([#1414]) but restored in ([#1423]). +- spark-connect-client: Remove `3.5.7` in ([#1414]) but restored in ([#1423]). ### Fixed @@ -94,6 +94,7 @@ All notable changes to this project will be documented in this file. [#1409]: https://github.com/stackabletech/docker-images/pull/1409 [#1410]: https://github.com/stackabletech/docker-images/pull/1410 [#1414]: https://github.com/stackabletech/docker-images/pull/1414 +[#1423]: https://github.com/stackabletech/docker-images/pull/1414 ## [25.11.0] - 2025-11-07 From 77217c48ff8e37675c52e64c683bdf96a64c3adf Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 17:03:49 +0100 Subject: [PATCH 3/3] fix hbase-connectors patchable config --- .../0001-Fix-protobuf-on-aarch64.patch | 0 .../{1.0.1_3 => 1.0.1_3.5.7}/patchable.toml | 0 .../0001-Fix-protobuf-on-aarch64.patch | 59 +++++++++++++++++++ .../patches/1.0.1_3.5.8/patchable.toml | 2 + 4 files changed, 61 insertions(+) rename spark-k8s/hbase-connectors/stackable/patches/{1.0.1_3 => 1.0.1_3.5.7}/0001-Fix-protobuf-on-aarch64.patch (100%) rename spark-k8s/hbase-connectors/stackable/patches/{1.0.1_3 => 1.0.1_3.5.7}/patchable.toml (100%) create mode 100644 spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch create mode 100644 spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3/0001-Fix-protobuf-on-aarch64.patch b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/0001-Fix-protobuf-on-aarch64.patch similarity index 100% rename from spark-k8s/hbase-connectors/stackable/patches/1.0.1_3/0001-Fix-protobuf-on-aarch64.patch rename to spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/0001-Fix-protobuf-on-aarch64.patch diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/patchable.toml similarity index 100% rename from spark-k8s/hbase-connectors/stackable/patches/1.0.1_3/patchable.toml rename to spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.7/patchable.toml diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch new file mode 100644 index 000000000..37d8663c9 --- /dev/null +++ b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch @@ -0,0 +1,59 @@ +From 8211177df0ca6cae00bc3bebcd264364732691b3 Mon Sep 17 00:00:00 2001 +From: dervoeti +Date: Thu, 13 Mar 2025 22:10:56 +0100 +Subject: Fix protobuf on aarch64 + +--- + pom.xml | 19 ++++++++++++++++++- + spark/pom.xml | 2 +- + 2 files changed, 19 insertions(+), 2 deletions(-) + +diff --git a/pom.xml b/pom.xml +index e849cd1..f514e14 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -157,7 +157,10 @@ + 1.5.1 + 0.14.0 + +- 2.5.0 ++ ++ com.google.protobuf ++ 2.6.1 ++ ${external.protobuf.version} + 0.5.0 + 2.11.0 + 1.7.7 +@@ -933,5 +936,19 @@ + + + ++ ++ ++ aarch64 ++ ++ com.github.os72 ++ 2.6.1-build3 ++ ++ ++ ++ linux ++ aarch64 ++ ++ ++ + + +diff --git a/spark/pom.xml b/spark/pom.xml +index 3f1eb21..fcdc73e 100644 +--- a/spark/pom.xml ++++ b/spark/pom.xml +@@ -84,7 +84,7 @@ + protobuf-maven-plugin + ${protobuf.plugin.version} + +- com.google.protobuf:protoc:${external.protobuf.version}:exe:${os.detected.classifier} ++ ${external.protobuf.groupId}:protoc:${external.protobuf.exe.version}:exe:${os.detected.classifier} + ${basedir}/src/main/protobuf/ + false + true diff --git a/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml new file mode 100644 index 000000000..6d8c2c8db --- /dev/null +++ b/spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml @@ -0,0 +1,2 @@ +base = "e5217d13ed729703580ff2d1b02378ada2d94f4d" +mirror = "https://github.com/stackabletech/hbase-connectors.git"