From 9627b7cc69f67276968cb2877902ee558f3afe5e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 11:39:55 +0100 Subject: [PATCH 1/5] add spark 3.5.8 --- spark-k8s/boil-config.toml | 22 +++++ .../3.5.8/0001-Update-CycloneDX-plugin.patch | 40 +++++++++ ...1311-BUILD-Promote-bcprov-jdk18on-to.patch | 83 +++++++++++++++++++ .../stackable/patches/3.5.8/patchable.toml | 2 + 4 files changed, 147 insertions(+) create mode 100644 spark-k8s/stackable/patches/3.5.8/0001-Update-CycloneDX-plugin.patch create mode 100644 spark-k8s/stackable/patches/3.5.8/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch create mode 100644 spark-k8s/stackable/patches/3.5.8/patchable.toml diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 9dd88b8b0..01e1e941f 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -20,6 +20,28 @@ jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1_3" +[versions."3.5.8"] +containerfile = "Dockerfile.3" + +[versions."3.5.8".local-images] +"hadoop/hadoop" = "3.4.2" +java-base = "17" +java-devel = "17" +hbase = "2.6.3" +"spark-k8s/hbase-connectors" = "1.0.1_3" + +[versions."3.5.8".build-arguments] +python-version = "3.11" +aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop +azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop +azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop +jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 +stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +jmx-exporter-version = "1.3.0" +tini-version = "0.19.0" +hbase-connector-version = "1.0.1_3" + [versions."4.0.1"] containerfile = "Dockerfile.4" diff --git a/spark-k8s/stackable/patches/3.5.8/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/3.5.8/0001-Update-CycloneDX-plugin.patch new file mode 100644 index 000000000..288033225 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.8/0001-Update-CycloneDX-plugin.patch @@ -0,0 +1,40 @@ +From 4a1e0b6cb70add10e83c071b6402b76bab1ac380 Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Tue, 11 Mar 2025 17:29:39 +0200 +Subject: Update CycloneDX plugin + +--- + dev/make-distribution.sh | 1 - + pom.xml | 7 ++++++- + 2 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh +index ef7c010e930..0f4c1c74e40 100755 +--- a/dev/make-distribution.sh ++++ b/dev/make-distribution.sh +@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \ + -Dmaven.javadoc.skip=true \ + -Dmaven.scaladoc.skip=true \ + -Dmaven.source.skip \ +- -Dcyclonedx.skip=true \ + $@) + + # Actually build the jar +diff --git a/pom.xml b/pom.xml +index edd2ad57880..667f6f8cbfb 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -3534,7 +3534,12 @@ + + org.cyclonedx + cyclonedx-maven-plugin +- 2.7.9 ++ 2.8.0 ++ ++ application ++ 1.5 ++ false ++ + + + package diff --git a/spark-k8s/stackable/patches/3.5.8/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch b/spark-k8s/stackable/patches/3.5.8/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch new file mode 100644 index 000000000..ed85f3e59 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.8/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch @@ -0,0 +1,83 @@ +From 49abdb63abc57d1b905dbd333d4b0ac487d42ae7 Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Tue, 22 Jul 2025 17:34:03 +0200 +Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope + +--- + LICENSE-binary | 1 + + assembly/pom.xml | 14 ++++++++++++++ + dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 + + licenses-binary/LICENSE-bouncycastle.txt | 13 +++++++++++++ + 4 files changed, 29 insertions(+) + create mode 100644 licenses-binary/LICENSE-bouncycastle.txt + +diff --git a/LICENSE-binary b/LICENSE-binary +index 05645977a0b..9834cf333f7 100644 +--- a/LICENSE-binary ++++ b/LICENSE-binary +@@ -480,6 +480,7 @@ org.typelevel:algebra_2.12:jar + org.typelevel:cats-kernel_2.12 + org.typelevel:machinist_2.12 + net.razorvine:pickle ++org.bouncycastle:bcprov-jdk18on + org.slf4j:jcl-over-slf4j + org.slf4j:jul-to-slf4j + org.slf4j:slf4j-api +diff --git a/assembly/pom.xml b/assembly/pom.xml +index 1533c2f0c2d..d75faf07900 100644 +--- a/assembly/pom.xml ++++ b/assembly/pom.xml +@@ -85,8 +85,22 @@ + guava + ${hadoop.deps.scope} + ++ ++ ++ ++ org.bouncycastle ++ bcprov-jdk18on ++ ${hadoop.deps.scope} ++ ++ + + ++ + + + +diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 +index b4d91a1b161..b7409dd53fa 100644 +--- a/dev/deps/spark-deps-hadoop-3-hive-2.3 ++++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 +@@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar + azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar + azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar + azure-storage/7.0.1//azure-storage-7.0.1.jar ++bcprov-jdk18on/1.77//bcprov-jdk18on-1.77.jar + blas/3.0.3//blas-3.0.3.jar + bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar + breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar +diff --git a/licenses-binary/LICENSE-bouncycastle.txt b/licenses-binary/LICENSE-bouncycastle.txt +new file mode 100644 +index 00000000000..277dcd1ebbb +--- /dev/null ++++ b/licenses-binary/LICENSE-bouncycastle.txt +@@ -0,0 +1,13 @@ ++Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). ++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and ++associated documentation files (the "Software"), to deal in the Software without restriction, ++including without limitation the rights to use, copy, modify, merge, publish, distribute, ++sub license, and/or sell copies of the Software, and to permit persons to whom the Software is ++furnished to do so, subject to the following conditions: The above copyright notice and this ++permission notice shall be included in all copies or substantial portions of the Software. ++ ++**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT ++NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, ++DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT ++OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** diff --git a/spark-k8s/stackable/patches/3.5.8/patchable.toml b/spark-k8s/stackable/patches/3.5.8/patchable.toml new file mode 100644 index 000000000..0bf043048 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.8/patchable.toml @@ -0,0 +1,2 @@ +base = "5a48a37b2dbd7b51e3640cd1d947438459556cc6" +mirror = "https://github.com/stackabletech/spark.git" From df403218f64f9aad7860d62123af751f78cf4597 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 11:41:39 +0100 Subject: [PATCH 2/5] delete spark 3.5.7 --- spark-k8s/boil-config.toml | 22 ----- .../3.5.7/0001-Update-CycloneDX-plugin.patch | 40 --------- ...1311-BUILD-Promote-bcprov-jdk18on-to.patch | 83 ------------------- .../stackable/patches/3.5.7/patchable.toml | 2 - 4 files changed, 147 deletions(-) delete mode 100644 spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch delete mode 100644 spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch delete mode 100644 spark-k8s/stackable/patches/3.5.7/patchable.toml diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 01e1e941f..1c9cf7522 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -1,25 +1,3 @@ -[versions."3.5.7"] -containerfile = "Dockerfile.3" - -[versions."3.5.7".local-images] -"hadoop/hadoop" = "3.4.2" -java-base = "17" -java-devel = "17" -hbase = "2.6.3" -"spark-k8s/hbase-connectors" = "1.0.1_3" - -[versions."3.5.7".build-arguments] -python-version = "3.11" -aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop -azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop -azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 -stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -jmx-exporter-version = "1.3.0" -tini-version = "0.19.0" -hbase-connector-version = "1.0.1_3" - [versions."3.5.8"] containerfile = "Dockerfile.3" diff --git a/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch deleted file mode 100644 index b3c2d89f7..000000000 --- a/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch +++ /dev/null @@ -1,40 +0,0 @@ -From 5e3c5568d4bf0fdd4feb19e559d606a55a0d2569 Mon Sep 17 00:00:00 2001 -From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Tue, 11 Mar 2025 17:29:39 +0200 -Subject: Update CycloneDX plugin - ---- - dev/make-distribution.sh | 1 - - pom.xml | 7 ++++++- - 2 files changed, 6 insertions(+), 2 deletions(-) - -diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh -index ef7c010e93..0f4c1c74e4 100755 ---- a/dev/make-distribution.sh -+++ b/dev/make-distribution.sh -@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \ - -Dmaven.javadoc.skip=true \ - -Dmaven.scaladoc.skip=true \ - -Dmaven.source.skip \ -- -Dcyclonedx.skip=true \ - $@) - - # Actually build the jar -diff --git a/pom.xml b/pom.xml -index a0e25ce4d8..b3755a04e7 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -3534,7 +3534,12 @@ - - org.cyclonedx - cyclonedx-maven-plugin -- 2.7.9 -+ 2.8.0 -+ -+ application -+ 1.5 -+ false -+ - - - package diff --git a/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch b/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch deleted file mode 100644 index 4f5dbc208..000000000 --- a/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch +++ /dev/null @@ -1,83 +0,0 @@ -From 0cfb1d6b733579ea91df842e685df9db5512ece8 Mon Sep 17 00:00:00 2001 -From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Tue, 22 Jul 2025 17:34:03 +0200 -Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope - ---- - LICENSE-binary | 1 + - assembly/pom.xml | 14 ++++++++++++++ - dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 + - licenses-binary/LICENSE-bouncycastle.txt | 13 +++++++++++++ - 4 files changed, 29 insertions(+) - create mode 100644 licenses-binary/LICENSE-bouncycastle.txt - -diff --git a/LICENSE-binary b/LICENSE-binary -index 05645977a0..9834cf333f 100644 ---- a/LICENSE-binary -+++ b/LICENSE-binary -@@ -480,6 +480,7 @@ org.typelevel:algebra_2.12:jar - org.typelevel:cats-kernel_2.12 - org.typelevel:machinist_2.12 - net.razorvine:pickle -+org.bouncycastle:bcprov-jdk18on - org.slf4j:jcl-over-slf4j - org.slf4j:jul-to-slf4j - org.slf4j:slf4j-api -diff --git a/assembly/pom.xml b/assembly/pom.xml -index ffa6a0e2b0..19032d6054 100644 ---- a/assembly/pom.xml -+++ b/assembly/pom.xml -@@ -85,8 +85,22 @@ - guava - ${hadoop.deps.scope} - -+ -+ -+ -+ org.bouncycastle -+ bcprov-jdk18on -+ ${hadoop.deps.scope} -+ -+ - - -+ - - - -diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 -index f110a1988f..88324d3e27 100644 ---- a/dev/deps/spark-deps-hadoop-3-hive-2.3 -+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 -@@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar - azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar - azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar - azure-storage/7.0.1//azure-storage-7.0.1.jar -+bcprov-jdk18on/1.77//bcprov-jdk18on-1.77.jar - blas/3.0.3//blas-3.0.3.jar - bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar - breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar -diff --git a/licenses-binary/LICENSE-bouncycastle.txt b/licenses-binary/LICENSE-bouncycastle.txt -new file mode 100644 -index 0000000000..277dcd1ebb ---- /dev/null -+++ b/licenses-binary/LICENSE-bouncycastle.txt -@@ -0,0 +1,13 @@ -+Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org). -+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -+associated documentation files (the "Software"), to deal in the Software without restriction, -+including without limitation the rights to use, copy, modify, merge, publish, distribute, -+sub license, and/or sell copies of the Software, and to permit persons to whom the Software is -+furnished to do so, subject to the following conditions: The above copyright notice and this -+permission notice shall be included in all copies or substantial portions of the Software. -+ -+**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT -+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT -+OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** diff --git a/spark-k8s/stackable/patches/3.5.7/patchable.toml b/spark-k8s/stackable/patches/3.5.7/patchable.toml deleted file mode 100644 index 49202d309..000000000 --- a/spark-k8s/stackable/patches/3.5.7/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -base = "ed00d046951a7ecda6429accd3b9c5b2dc792b65" -mirror = "https://github.com/stackabletech/spark.git" From 34652f79a8f319305633cf337bd5e5f735a0830d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 11:43:20 +0100 Subject: [PATCH 3/5] bump spark connect from 3.5.7 to 3.5.8 --- spark-connect-client/boil-config.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spark-connect-client/boil-config.toml b/spark-connect-client/boil-config.toml index 304faa090..9aa478ed6 100644 --- a/spark-connect-client/boil-config.toml +++ b/spark-connect-client/boil-config.toml @@ -1,8 +1,8 @@ -[versions."3.5.7".local-images] -spark-k8s = "3.5.7" +[versions."3.5.8".local-images] +spark-k8s = "3.5.8" java-base = "17" -[versions."3.5.7".build-arguments] +[versions."3.5.8".build-arguments] python-version = "3.11" [versions."4.0.1".local-images] From 3ed50e052078b9937f6216e456142a3654e9c77a Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 11:46:22 +0100 Subject: [PATCH 4/5] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ab94bfc..b40a2e1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ All notable changes to this project will be documented in this file. - spark-k8s/hbase-connectors: new image extracted from spark dockerfile ([#1402]). - trino: Add `479` ([#1403]). - hbase: Add `2.6.4` (and phoenix `5.3.0`) ([#1408]). +- spark: Add `3.5.8` ([#1414]). +- spark-connect-client: Add `3.5.8` ([#1414]). ### Changed @@ -54,6 +56,8 @@ All notable changes to this project will be documented in this file. - trino: Remove `451` and `476` ([#1403]). - hbase: Remove `2.6.2` ([#1408]). - spark: remove the JRE before installing the JDK to prevent version conflicts ([#1410]). +- spark: Remove `3.5.7` ([#1414]). +- spark-connect-client: Remove `3.5.7` ([#1414]). ### Fixed @@ -89,6 +93,7 @@ All notable changes to this project will be documented in this file. [#1408]: https://github.com/stackabletech/docker-images/pull/1408 [#1409]: https://github.com/stackabletech/docker-images/pull/1409 [#1410]: https://github.com/stackabletech/docker-images/pull/1410 +[#1414]: https://github.com/stackabletech/docker-images/pull/1414 ## [25.11.0] - 2025-11-07 From 4b6249a6ecb1cc0f5f688fb6adb59f2784056cdf Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Feb 2026 12:35:09 +0100 Subject: [PATCH 5/5] update hbase-connectors deps --- spark-k8s/hbase-connectors/boil-config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-k8s/hbase-connectors/boil-config.toml b/spark-k8s/hbase-connectors/boil-config.toml index beb9e4493..555044b50 100644 --- a/spark-k8s/hbase-connectors/boil-config.toml +++ b/spark-k8s/hbase-connectors/boil-config.toml @@ -5,6 +5,6 @@ containerfile = "Dockerfile" java-devel = "17" [versions."1.0.1_3".build-arguments] -spark-version = "3.5.7" +spark-version = "3.5.8" hadoop-version = "3.4.2" hbase-version = "2.6.3"