From 7a38ce4352d0295e5022a6d58f0afb81f0e3ca21 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 18 Feb 2026 12:40:01 +0800 Subject: [PATCH 1/2] [AURON #2017] [BUILD] Add Spark 4.x support to dev/reformat script. Signed-off-by: slfan1989 --- .github/workflows/style.yml | 6 ++++++ dev/reformat | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 6f5ef9a9f..bfd44e51f 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -46,5 +46,11 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'adopt-hotspot' + java-version: 17 + check-latest: false - run: | ./dev/reformat --check diff --git a/dev/reformat b/dev/reformat index 5a463a639..6eacb78ef 100755 --- a/dev/reformat +++ b/dev/reformat @@ -52,12 +52,19 @@ fi sparkver=spark-3.5 for celebornver in celeborn-0.5 celeborn-0.6 do - run_maven -P"${sparkver}" -Pceleborn,"${celebornver}" -Puniffle,uniffle-0.10 -Ppaimon,paimon-1.2 -Pflink,flink-1.18 -Piceberg,iceberg-1.9 + run_maven -P"${sparkver}" -Pceleborn,"${celebornver}" -Puniffle,uniffle-0.10 -Ppaimon,paimon-1.2 -Pflink-1.18 -Piceberg,iceberg-1.9 done -sparkvers=(spark-3.0 spark-3.1 spark-3.2 spark-3.3 spark-3.4) +sparkvers=(spark-3.0 spark-3.1 spark-3.2 spark-3.3 spark-3.4 spark-4.0 spark-4.1) for sparkver in "${sparkvers[@]}" do + if [[ $sparkver == spark-4.* ]]; then + SCALA_PROFILE=scala-2.13 + export JAVA_HOME=$(/usr/libexec/java_home -v 17) + else + SCALA_PROFILE=scala-2.12 + export JAVA_HOME=$(/usr/libexec/java_home -v 1.8) + fi run_maven -P"${sparkver}" done From a7219e428cbbb430c3ca6fb72ca58b8c7040b45f Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 18 Feb 2026 13:44:24 +0800 Subject: [PATCH 2/2] [AURON #2017] [BUILD] Add Spark 4.x support to dev/reformat script. Signed-off-by: slfan1989 --- dev/reformat | 2 +- .../scala/org/apache/spark/sql/auron/AuronConverters.scala | 6 +++--- .../spark/sql/auron/AuronSparkSessionExtension.scala | 2 +- .../scala/org/apache/spark/sql/auron/NativeHelper.scala | 2 +- .../apache/spark/sql/auron/util/TaskContextHelper.scala | 2 +- .../auron/plan/NativeParquetInsertIntoHiveTableBase.scala | 7 ++++--- .../auron/shuffle/uniffle/AuronUniffleShuffleReader.scala | 2 +- 7 files changed, 12 insertions(+), 11 deletions(-) diff --git a/dev/reformat b/dev/reformat index 6eacb78ef..6e6108cf5 100755 --- a/dev/reformat +++ b/dev/reformat @@ -52,7 +52,7 @@ fi sparkver=spark-3.5 for celebornver in celeborn-0.5 celeborn-0.6 do - run_maven -P"${sparkver}" -Pceleborn,"${celebornver}" -Puniffle,uniffle-0.10 -Ppaimon,paimon-1.2 -Pflink-1.18 -Piceberg,iceberg-1.9 + run_maven -P"${sparkver}" -Pceleborn,"${celebornver}" -Puniffle,uniffle-0.10 -Ppaimon,paimon-1.2 -Pflink-1.18 -Piceberg-1.9 done diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala index def645d51..3129d1074 100644 --- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala +++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala @@ -424,7 +424,7 @@ object AuronConverters extends Logging { assert( !exec.requiredSchema.exists(e => existTimestampType(e.dataType)), s"Parquet scan with timestamp type is not supported for table: ${tableIdentifier - .getOrElse("unknown")}. " + + .getOrElse("unknown")}. " + "Set spark.auron.enable.scan.parquet.timestamp=true to enable timestamp support " + "or remove timestamp columns from the query.") } @@ -435,7 +435,7 @@ object AuronConverters extends Logging { assert( !exec.requiredSchema.exists(e => existTimestampType(e.dataType)), s"ORC scan with timestamp type is not supported for tableIdentifier: ${tableIdentifier - .getOrElse("unknown")}. " + + .getOrElse("unknown")}. " + "Set spark.auron.enable.scan.orc.timestamp=true to enable timestamp support " + "or remove timestamp columns from the query.") } @@ -443,7 +443,7 @@ object AuronConverters extends Logging { case p => throw new NotImplementedError( s"Cannot convert FileSourceScanExec tableIdentifier: ${tableIdentifier.getOrElse( - "unknown")}, class: ${p.getClass.getName}") + "unknown")}, class: ${p.getClass.getName}") } } diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronSparkSessionExtension.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronSparkSessionExtension.scala index b68b04954..47492aa3d 100644 --- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronSparkSessionExtension.scala +++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronSparkSessionExtension.scala @@ -91,7 +91,7 @@ case class AuronColumnarOverrides(sparkSession: SparkSession) extends ColumnarRu dumpSimpleSparkPlanTreeNode(sparkPlanTransformed) logInfo(s"Transformed spark plan after preColumnarTransitions:\n${sparkPlanTransformed - .treeString(verbose = true, addSuffix = true)}") + .treeString(verbose = true, addSuffix = true)}") // post-transform Shims.get.postTransform(sparkPlanTransformed, sparkSession.sparkContext) diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala index e16656471..7a1e34724 100644 --- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala +++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala @@ -74,7 +74,7 @@ object NativeHelper extends Logging { val heapMemory = Runtime.getRuntime.maxMemory() val offheapMemory = totalMemory - heapMemory logWarning(s"memory total: ${Utils.bytesToString(totalMemory)}, onheap: ${Utils.bytesToString( - heapMemory)}, offheap: ${Utils.bytesToString(offheapMemory)}") + heapMemory)}, offheap: ${Utils.bytesToString(offheapMemory)}") offheapMemory } diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/util/TaskContextHelper.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/util/TaskContextHelper.scala index 8e5d7353f..eecec5ef4 100644 --- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/util/TaskContextHelper.scala +++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/util/TaskContextHelper.scala @@ -47,7 +47,7 @@ object TaskContextHelper extends Logging { val thread = Thread.currentThread() val threadName = if (context != null) { s"auron native task ${context.partitionId()}.${context.attemptNumber()} in stage ${context - .stageId()}.${context.stageAttemptNumber()} (TID ${context.taskAttemptId()})" + .stageId()}.${context.stageAttemptNumber()} (TID ${context.taskAttemptId()})" } else { "auron native task " + thread.getName } diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetInsertIntoHiveTableBase.scala b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetInsertIntoHiveTableBase.scala index d43f7d17d..b9fb7f99b 100644 --- a/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetInsertIntoHiveTableBase.scala +++ b/spark-extension/src/main/scala/org/apache/spark/sql/execution/auron/plan/NativeParquetInsertIntoHiveTableBase.scala @@ -69,9 +69,10 @@ abstract class NativeParquetInsertIntoHiveTableBase( .filterKeys(Set("stage_id", "output_rows", "elapsed_compute")) .toSeq :+ ("io_time", SQLMetrics.createNanoTimingMetric(sparkContext, "Native.io_time")) - :+ ("bytes_written", - SQLMetrics - .createSizeMetric(sparkContext, "Native.bytes_written")): _*) + :+ ( + "bytes_written", + SQLMetrics + .createSizeMetric(sparkContext, "Native.bytes_written")): _*) def check(): Unit = { val hadoopConf = sparkContext.hadoopConfiguration diff --git a/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala b/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala index 9a152eca9..9cb665a8c 100644 --- a/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala +++ b/thirdparty/auron-uniffle/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/uniffle/AuronUniffleShuffleReader.scala @@ -166,7 +166,7 @@ class AuronUniffleShuffleReader[K, C]( } if (!emptyPartitionIds.isEmpty) { logDebug(s"Found ${emptyPartitionIds - .size()} empty shuffle partitions: ${emptyPartitionIds.asScala.mkString(",")}") + .size()} empty shuffle partitions: ${emptyPartitionIds.asScala.mkString(",")}") } iterators = shuffleDataIterList.iterator() if (iterators.hasNext) {