fix(spark): restore version 3.5.7

CHANGELOG.md

@@ -56,8 +56,8 @@ All notable changes to this project will be documented in this file.
 - trino: Remove `451` and `476` ([#1403]).
 - hbase: Remove `2.6.2` ([#1408]).
 - spark: remove the JRE before installing the JDK to prevent version conflicts ([#1410]).
-- spark: Remove `3.5.7` ([#1414]).
-- spark-connect-client: Remove `3.5.7` ([#1414]).
+- spark: Remove `3.5.7` in ([#1414]) but restored in ([#1423]).
+- spark-connect-client: Remove `3.5.7` in ([#1414]) but restored in ([#1423]).
 
 ### Fixed
 
@@ -94,6 +94,7 @@ All notable changes to this project will be documented in this file.
 [#1409]: https://github.com/stackabletech/docker-images/pull/1409
 [#1410]: https://github.com/stackabletech/docker-images/pull/1410
 [#1414]: https://github.com/stackabletech/docker-images/pull/1414
+[#1423]: https://github.com/stackabletech/docker-images/pull/1414
 [#1422]: https://github.com/stackabletech/docker-images/pull/1422
 
 ## [25.11.0] - 2025-11-07

spark-connect-client/boil-config.toml

@@ -1,3 +1,10 @@
+[versions."3.5.7".local-images]
+spark-k8s = "3.5.7"
+java-base = "17"
+
+[versions."3.5.7".build-arguments]
+python-version = "3.11"
+
 [versions."3.5.8".local-images]
 spark-k8s = "3.5.8"
 java-base = "17"

spark-k8s/boil-config.toml

@@ -1,3 +1,25 @@
+[versions."3.5.7"]
+containerfile = "Dockerfile.3"
+
+[versions."3.5.7".local-images]
+"hadoop/hadoop" = "3.4.2"
+java-base = "17"
+java-devel = "17"
+hbase = "2.6.3"
+"spark-k8s/hbase-connectors" = "1.0.1_3.5.7"
+
+[versions."3.5.7".build-arguments]
+python-version = "3.11"
+aws-java-sdk-bundle-version = "2.29.52"   # needs to match the version shipped by Hadoop
+azure-storage-version = "7.0.1"           # needs to match the version shipped by Hadoop
+azure-keyvault-core-version = "1.0.0"     # needs to match the version shipped by Hadoop
+jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7
+stax2-api-version = "4.2.1"               # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+woodstox-core-version = "6.5.1"           # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+jmx-exporter-version = "1.3.0"
+tini-version = "0.19.0"
+hbase-connector-version = "1.0.1_3.5.7"
+
 [versions."3.5.8"]
 containerfile = "Dockerfile.3"
 
@@ -6,7 +28,7 @@ containerfile = "Dockerfile.3"
 java-base = "17"
 java-devel = "17"
 hbase = "2.6.3"
-"spark-k8s/hbase-connectors" = "1.0.1_3"
+"spark-k8s/hbase-connectors" = "1.0.1_3.5.8"
 
 [versions."3.5.8".build-arguments]
 python-version = "3.11"
@@ -18,7 +40,7 @@ stax2-api-version = "4.2.1"               # needs to match the jackson version h
 woodstox-core-version = "6.5.1"           # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
 jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
-hbase-connector-version = "1.0.1_3"
+hbase-connector-version = "1.0.1_3.5.8"
 
 [versions."4.0.1"]
 containerfile = "Dockerfile.4"

spark-k8s/hbase-connectors/boil-config.toml

@@ -1,10 +1,16 @@
-[versions."1.0.1_3"]
-containerfile = "Dockerfile"
 
-[versions."1.0.1_3".local-images]
+[versions."1.0.1_3.5.7".local-images]
 java-devel = "17"
 
-[versions."1.0.1_3".build-arguments]
+[versions."1.0.1_3.5.7".build-arguments]
+spark-version = "3.5.7"
+hadoop-version = "3.4.2"
+hbase-version = "2.6.3"
+
+[versions."1.0.1_3.5.8".local-images]
+java-devel = "17"
+
+[versions."1.0.1_3.5.8".build-arguments]
 spark-version = "3.5.8"
 hadoop-version = "3.4.2"
 hbase-version = "2.6.3"

spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/0001-Fix-protobuf-on-aarch64.patch

@@ -0,0 +1,59 @@
+From 8211177df0ca6cae00bc3bebcd264364732691b3 Mon Sep 17 00:00:00 2001
+From: dervoeti <lukas.krug@stackable.tech>
+Date: Thu, 13 Mar 2025 22:10:56 +0100
+Subject: Fix protobuf on aarch64
+
+---
+ pom.xml       | 19 ++++++++++++++++++-
+ spark/pom.xml |  2 +-
+ 2 files changed, 19 insertions(+), 2 deletions(-)
+
+diff --git a/pom.xml b/pom.xml
+index e849cd1..f514e14 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -157,7 +157,10 @@
+     <extra.enforcer.version>1.5.1</extra.enforcer.version>
+     <restrict-imports.enforcer.version>0.14.0</restrict-imports.enforcer.version>
+     <!--Internally we use a different version of protobuf. See hbase-protocol-shaded-->
+-    <external.protobuf.version>2.5.0</external.protobuf.version>
++    <!-- com.google repo will be used except on Aarch64 platform. -->
++    <external.protobuf.groupId>com.google.protobuf</external.protobuf.groupId>
++    <external.protobuf.version>2.6.1</external.protobuf.version>
++    <external.protobuf.exe.version>${external.protobuf.version}</external.protobuf.exe.version>
+     <protobuf.plugin.version>0.5.0</protobuf.plugin.version>
+     <commons-io.version>2.11.0</commons-io.version>
+     <avro.version>1.7.7</avro.version>
+@@ -933,5 +936,19 @@
+         </plugins>
+       </build>
+     </profile>
++    <!-- use com.github.os72 on aarch64 platform -->
++    <profile>
++      <id>aarch64</id>
++      <properties>
++        <external.protobuf.groupId>com.github.os72</external.protobuf.groupId>
++        <external.protobuf.exe.version>2.6.1-build3</external.protobuf.exe.version>
++      </properties>
++      <activation>
++        <os>
++          <family>linux</family>
++          <arch>aarch64</arch>
++        </os>
++      </activation>
++    </profile>
+   </profiles>
+ </project>
+diff --git a/spark/pom.xml b/spark/pom.xml
+index 3f1eb21..fcdc73e 100644
+--- a/spark/pom.xml
++++ b/spark/pom.xml
+@@ -84,7 +84,7 @@
+           <artifactId>protobuf-maven-plugin</artifactId>
+           <version>${protobuf.plugin.version}</version>
+           <configuration>
+-            <protocArtifact>com.google.protobuf:protoc:${external.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
++            <protocArtifact>${external.protobuf.groupId}:protoc:${external.protobuf.exe.version}:exe:${os.detected.classifier}</protocArtifact>
+             <protoSourceRoot>${basedir}/src/main/protobuf/</protoSourceRoot>
+             <clearOutputDirectory>false</clearOutputDirectory>
+             <checkStaleness>true</checkStaleness>

spark-k8s/hbase-connectors/stackable/patches/1.0.1_3.5.8/patchable.toml

@@ -0,0 +1,2 @@
+base = "e5217d13ed729703580ff2d1b02378ada2d94f4d"
+mirror = "https://github.com/stackabletech/hbase-connectors.git"

spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch

@@ -0,0 +1,40 @@
+From 5e3c5568d4bf0fdd4feb19e559d606a55a0d2569 Mon Sep 17 00:00:00 2001
+From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
+Date: Tue, 11 Mar 2025 17:29:39 +0200
+Subject: Update CycloneDX plugin
+
+---
+ dev/make-distribution.sh | 1 -
+ pom.xml                  | 7 ++++++-
+ 2 files changed, 6 insertions(+), 2 deletions(-)
+
+diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
+index ef7c010e93..0f4c1c74e4 100755
+--- a/dev/make-distribution.sh
++++ b/dev/make-distribution.sh
+@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \
+     -Dmaven.javadoc.skip=true \
+     -Dmaven.scaladoc.skip=true \
+     -Dmaven.source.skip \
+-    -Dcyclonedx.skip=true \
+     $@)
+
+ # Actually build the jar
+diff --git a/pom.xml b/pom.xml
+index a0e25ce4d8..b3755a04e7 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -3534,7 +3534,12 @@
+       <plugin>
+         <groupId>org.cyclonedx</groupId>
+         <artifactId>cyclonedx-maven-plugin</artifactId>
+-        <version>2.7.9</version>
++        <version>2.8.0</version>
++        <configuration>
++          <projectType>application</projectType>
++          <schemaVersion>1.5</schemaVersion>
++          <skipNotDeployed>false</skipNotDeployed>
++        </configuration>
+         <executions>
+           <execution>
+             <phase>package</phase>

spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch

@@ -0,0 +1,83 @@
+From 0cfb1d6b733579ea91df842e685df9db5512ece8 Mon Sep 17 00:00:00 2001
+From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
+Date: Tue, 22 Jul 2025 17:34:03 +0200
+Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope
+
+---
+ LICENSE-binary                           |  1 +
+ assembly/pom.xml                         | 14 ++++++++++++++
+ dev/deps/spark-deps-hadoop-3-hive-2.3    |  1 +
+ licenses-binary/LICENSE-bouncycastle.txt | 13 +++++++++++++
+ 4 files changed, 29 insertions(+)
+ create mode 100644 licenses-binary/LICENSE-bouncycastle.txt
+
+diff --git a/LICENSE-binary b/LICENSE-binary
+index 05645977a0..9834cf333f 100644
+--- a/LICENSE-binary
++++ b/LICENSE-binary
+@@ -480,6 +480,7 @@ org.typelevel:algebra_2.12:jar
+ org.typelevel:cats-kernel_2.12
+ org.typelevel:machinist_2.12
+ net.razorvine:pickle
++org.bouncycastle:bcprov-jdk18on
+ org.slf4j:jcl-over-slf4j
+ org.slf4j:jul-to-slf4j
+ org.slf4j:slf4j-api
+diff --git a/assembly/pom.xml b/assembly/pom.xml
+index ffa6a0e2b0..19032d6054 100644
+--- a/assembly/pom.xml
++++ b/assembly/pom.xml
+@@ -85,8 +85,22 @@
+       <artifactId>guava</artifactId>
+       <scope>${hadoop.deps.scope}</scope>
+     </dependency>
++
++    <!--
++      Adapted from https://github.com/apache/spark/pull/50077
++
++      SPARK-51311: HDFS-15098 (3.4.0) adds hard dependency on bcprov-jdk18on, Spark fails to submit
++      to Kerberized cluster without this dependency, until HADOOP-19152 (3.5.0, unreleased)
++      -->
++    <dependency>
++      <groupId>org.bouncycastle</groupId>
++      <artifactId>bcprov-jdk18on</artifactId>
++      <scope>${hadoop.deps.scope}</scope>
++    </dependency>
++
+   </dependencies>
+
++
+   <build>
+     <plugins>
+       <plugin>
+diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
+index f110a1988f..88324d3e27 100644
+--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
++++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
+@@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar
+ azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
+ azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
+ azure-storage/7.0.1//azure-storage-7.0.1.jar
++bcprov-jdk18on/1.77//bcprov-jdk18on-1.77.jar
+ blas/3.0.3//blas-3.0.3.jar
+ bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
+ breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar
+diff --git a/licenses-binary/LICENSE-bouncycastle.txt b/licenses-binary/LICENSE-bouncycastle.txt
+new file mode 100644
+index 0000000000..277dcd1ebb
+--- /dev/null
++++ b/licenses-binary/LICENSE-bouncycastle.txt
+@@ -0,0 +1,13 @@
++Copyright (c) 2000-2024 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org).
++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
++associated documentation files (the "Software"), to deal in the Software without restriction,
++including without limitation the rights to use, copy, modify, merge, publish, distribute,
++sub license, and/or sell copies of the Software, and to permit persons to whom the Software is
++furnished to do so, subject to the following conditions: The above copyright notice and this
++permission notice shall be included in all copies or substantial portions of the Software.
++
++**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
++NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
++NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
++DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
++OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.**

spark-k8s/stackable/patches/3.5.7/patchable.toml

@@ -0,0 +1,2 @@
+base = "ed00d046951a7ecda6429accd3b9c5b2dc792b65"
+mirror = "https://github.com/stackabletech/spark.git"