pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  -->

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.apache</groupId>
    <artifactId>apache</artifactId>
    <version>18</version>
  </parent>
  <groupId>org.apache.spark</groupId>
  <artifactId>spark-parent_2.12</artifactId>
  <version>3.4.0-SNAPSHOT</version>
  <packaging>pom</packaging>
  <name>Spark Project Parent POM</name>
  <url>https://spark.apache.org/</url>
  <licenses>
    <license>
      <name>Apache 2.0 License</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
      <distribution>repo</distribution>
    </license>
  </licenses>
  <scm>
    <connection>scm:git:git@github.com:apache/spark.git</connection>
    <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/spark.git</developerConnection>
    <url>scm:git:git@github.com:apache/spark.git</url>
    <tag>HEAD</tag>
  </scm>
  <organization>
    <name>Apache Software Foundation</name>
    <url>https://www.apache.org</url>
  </organization>
  <issueManagement>
    <system>JIRA</system>
    <url>https://issues.apache.org/jira/browse/SPARK</url>
  </issueManagement>

  <mailingLists>
    <mailingList>
      <name>Dev Mailing List</name>
      <post>dev@spark.apache.org</post>
      <subscribe>dev-subscribe@spark.apache.org</subscribe>
      <unsubscribe>dev-unsubscribe@spark.apache.org</unsubscribe>
    </mailingList>

    <mailingList>
      <name>User Mailing List</name>
      <post>user@spark.apache.org</post>
      <subscribe>user-subscribe@spark.apache.org</subscribe>
      <unsubscribe>user-unsubscribe@spark.apache.org</unsubscribe>
    </mailingList>

    <mailingList>
      <name>Commits Mailing List</name>
      <post>commits@spark.apache.org</post>
      <subscribe>commits-subscribe@spark.apache.org</subscribe>
      <unsubscribe>commits-unsubscribe@spark.apache.org</unsubscribe>
    </mailingList>
  </mailingLists>

  <modules>
    <module>common/sketch</module>
    <module>common/kvstore</module>
    <module>common/network-common</module>
    <module>common/network-shuffle</module>
    <module>common/unsafe</module>
    <module>common/tags</module>
    <module>core</module>
    <module>graphx</module>
    <module>mllib</module>
    <module>mllib-local</module>
    <module>tools</module>
    <module>streaming</module>
    <module>sql/catalyst</module>
    <module>sql/core</module>
    <module>sql/hive</module>
    <module>assembly</module>
    <module>examples</module>
    <module>repl</module>
    <module>launcher</module>
    <module>connector/kafka-0-10-token-provider</module>
    <module>connector/kafka-0-10</module>
    <module>connector/kafka-0-10-assembly</module>
    <module>connector/kafka-0-10-sql</module>
    <module>connector/avro</module>
    <module>connector/connect</module>
    <module>connector/protobuf</module>
    <!-- See additional modules enabled by profiles below -->
  </modules>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <java.version>1.8</java.version>
    <maven.compiler.source>${java.version}</maven.compiler.source>
    <maven.compiler.target>${java.version}</maven.compiler.target>
    <maven.version>3.8.6</maven.version>
    <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
    <sbt.project.name>spark</sbt.project.name>
    <slf4j.version>2.0.3</slf4j.version>
    <log4j.version>2.19.0</log4j.version>
    <!-- make sure to update IsolatedClientLoader whenever this version is changed -->
    <hadoop.version>3.3.4</hadoop.version>
    <protobuf.version>2.5.0</protobuf.version>
    <yarn.version>${hadoop.version}</yarn.version>
    <zookeeper.version>3.6.2</zookeeper.version>
    <curator.version>2.13.0</curator.version>
    <hive.group>org.apache.hive</hive.group>
    <hive.classifier>core</hive.classifier>
    <!-- Version used in Maven Hive dependency -->
    <hive.version>2.3.9</hive.version>
    <hive23.version>2.3.9</hive23.version>
    <!-- Version used for internal directory structure -->
    <hive.version.short>2.3</hive.version.short>
    <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
    <kafka.version>3.2.3</kafka.version>
    <!-- After 10.15.1.3, the minimum required version is JDK9 -->
    <derby.version>10.14.2.0</derby.version>
    <parquet.version>1.12.3</parquet.version>
    <orc.version>1.8.0</orc.version>
    <orc.classifier>shaded-protobuf</orc.classifier>
    <jetty.version>9.4.49.v20220914</jetty.version>
    <jakartaservlet.version>4.0.3</jakartaservlet.version>
    <chill.version>0.10.0</chill.version>
    <ivy.version>2.5.0</ivy.version>
    <oro.version>2.0.8</oro.version>
    <!--
    If you changes codahale.metrics.version, you also need to change
    the link to metrics.dropwizard.io in docs/monitoring.md.
    -->
    <codahale.metrics.version>4.2.12</codahale.metrics.version>
    <!-- Should be consistent with SparkBuild.scala and docs -->
    <avro.version>1.11.1</avro.version>
    <aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
    <!-- Should be consistent with Kinesis client dependency -->
    <aws.java.sdk.version>1.11.655</aws.java.sdk.version>
    <!-- the producer is used in tests -->
    <aws.kinesis.producer.version>0.12.8</aws.kinesis.producer.version>
    <gcs-connector.version>hadoop3-2.2.7</gcs-connector.version>
    <!--  org.apache.httpcomponents/httpclient-->
    <commons.httpclient.version>4.5.13</commons.httpclient.version>
    <commons.httpcore.version>4.4.14</commons.httpcore.version>
    <commons.math3.version>3.6.1</commons.math3.version>
    <!-- managed up from 3.2.1 for SPARK-11652 -->
    <commons.collections.version>3.2.2</commons.collections.version>
    <commons.collections4.version>4.4</commons.collections4.version>
    <scala.version>2.12.17</scala.version>
    <scala.binary.version>2.12</scala.binary.version>
    <scalatest-maven-plugin.version>2.1.0</scalatest-maven-plugin.version>
    <!--
      This needs to be managed in different profiles to avoid
      errors building different Hadoop versions.
      See: SPARK-36547, SPARK-38394.
       -->
    <scala-maven-plugin.version>4.7.1</scala-maven-plugin.version>
    <!-- for now, not running scalafmt as part of default verify pipeline -->
    <scalafmt.skip>true</scalafmt.skip>
    <scalafmt.validateOnly>true</scalafmt.validateOnly>
    <scalafmt.changedOnly>true</scalafmt.changedOnly>
    <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
    <fasterxml.jackson.version>2.13.4</fasterxml.jackson.version>
    <fasterxml.jackson.databind.version>2.13.4.2</fasterxml.jackson.databind.version>
    <snappy.version>1.1.8.4</snappy.version>
    <netlib.ludovic.dev.version>3.0.2</netlib.ludovic.dev.version>
    <commons-codec.version>1.15</commons-codec.version>
    <commons-compress.version>1.21</commons-compress.version>
    <commons-io.version>2.11.0</commons-io.version>
    <!-- org.apache.commons/commons-lang/-->
    <commons-lang2.version>2.6</commons-lang2.version>
    <!-- org.apache.commons/commons-lang3/-->
    <commons-lang3.version>3.12.0</commons-lang3.version>
    <!-- org.apache.commons/commons-pool2/-->
    <commons-pool2.version>2.11.1</commons-pool2.version>
    <datanucleus-core.version>4.1.17</datanucleus-core.version>
    <guava.version>14.0.1</guava.version>
    <janino.version>3.1.7</janino.version>
    <jersey.version>2.36</jersey.version>
    <joda.version>2.12.0</joda.version>
    <jodd.version>3.5.2</jodd.version>
    <jsr305.version>3.0.0</jsr305.version>
    <libthrift.version>0.12.0</libthrift.version>
    <!-- Please don't upgrade the version to 4.10+, it depends on JDK 11 -->
    <antlr4.version>4.9.3</antlr4.version>
    <jpam.version>1.1</jpam.version>
    <selenium.version>4.4.0</selenium.version>
    <htmlunit-driver.version>3.64.0</htmlunit-driver.version>
    <htmlunit.version>2.64.0</htmlunit.version>
    <maven-antrun.version>1.8</maven-antrun.version>
    <commons-crypto.version>1.1.0</commons-crypto.version>
    <commons-cli.version>1.5.0</commons-cli.version>
    <bouncycastle.version>1.60</bouncycastle.version>
    <tink.version>1.7.0</tink.version>
    <!-- When upgrading `netty.version`, need to check whether
         the version of `netty-tcnative-classes.version` also needs to be upgraded
    -->
    <netty.version>4.1.84.Final</netty.version>
    <netty-tcnative-classes.version>2.0.54.Final</netty-tcnative-classes.version>
    <!--
    If you are changing Arrow version specification, please check
    ./python/pyspark/sql/pandas/utils.py, and ./python/setup.py too.
    -->
    <arrow.version>9.0.0</arrow.version>
    <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
    <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
    <kubernetes-client.version>6.2.0</kubernetes-client.version>

    <test.java.home>${java.home}</test.java.home>

    <!-- Some UI tests require Chrome and Chrome driver installed so those tests are disabled by default. -->
    <test.default.exclude.tags>org.apache.spark.tags.ChromeUITest</test.default.exclude.tags>
    <test.exclude.tags></test.exclude.tags>
    <test.include.tags></test.include.tags>

    <test.jdwp.address>localhost:0</test.jdwp.address>
    <test.jdwp.suspend>y</test.jdwp.suspend>
    <test.jdwp.server>y</test.jdwp.server>
    <test.debug.suite>false</test.debug.suite>

    <!-- Package to use when relocating shaded classes. -->
    <spark.shade.packageName>org.sparkproject</spark.shade.packageName>

    <!-- Modules that copy jars to the build directory should do so under this location. -->
    <jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>

    <!-- Allow modules to enable / disable certain build plugins easily. -->
    <build.testJarPhase>prepare-package</build.testJarPhase>
    <build.copyDependenciesPhase>none</build.copyDependenciesPhase>

    <!--
      Dependency scopes that can be overridden by enabling certain profiles. These profiles are
      declared in the projects that build assemblies.

      For other projects the scope should remain as "compile", otherwise they are not available
      during compilation if the dependency is transitive (e.g. "graphx/" depending on "core/" and
      needing Hadoop classes in the classpath to compile).
    -->
    <hadoop.deps.scope>compile</hadoop.deps.scope>
    <hive.deps.scope>compile</hive.deps.scope>
    <hive.storage.version>2.7.3</hive.storage.version>
    <hive.storage.scope>compile</hive.storage.scope>
    <hive.common.scope>compile</hive.common.scope>
    <hive.llap.scope>compile</hive.llap.scope>
    <hive.serde.scope>compile</hive.serde.scope>
    <hive.shims.scope>compile</hive.shims.scope>
    <orc.deps.scope>compile</orc.deps.scope>
    <parquet.deps.scope>compile</parquet.deps.scope>
    <parquet.test.deps.scope>test</parquet.test.deps.scope>

    <!--
      These default to Hadoop 3.x shaded client/minicluster jars, but since the shaded jars are
      only available in 3.x, we have to switch to non-shaded Hadoop client jars when the active
      profile is hadoop-2.

      To make the above work, in the hadoop-2 profile section we bind hadoop-client-api to
      hadoop-client and hadoop-client-runtime to hadoop-yarn-api, which is a dependency of the
      former. This effectively moves the hadoop-yarn-api to the same level as hadoop-client, but
      should be fine since both dependencies are required. We cannot use hadoop-client for both
      of these because the maven enforcer plugin bans duplicate dependencies.

      We still leave hadoop-client-minicluster to use hadoop-client because it is of test scope,
      and is only used by spark-yarn module. To avoid the duplicate dependency issue we only
      declare the dependency when we're using hadoop-3 profile, in
      resource-managers/yarn/pom.xml.

      Please check SPARK-36835 for more details.
    -->
    <hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
    <hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
    <hadoop-client-minicluster.artifact>hadoop-client-minicluster</hadoop-client-minicluster.artifact>
    <spark.yarn.isHadoopProvided>false</spark.yarn.isHadoopProvided>

    <!--
      Overridable test home. So that you can call individual pom files directly without
      things breaking.
    -->
    <spark.test.home>${session.executionRootDirectory}</spark.test.home>
    <spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
    <spark.test.docker.keepContainer>false</spark.test.docker.keepContainer>
    <spark.test.docker.removePulledImage>true</spark.test.docker.removePulledImage>

    <CodeCacheSize>128m</CodeCacheSize>
    <!-- Needed for consistent times -->
    <maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>

    <!-- SPARK-36796 for JDK-17 test-->
    <extraJavaTestArgs>
      -XX:+IgnoreUnrecognizedVMOptions
      --add-opens=java.base/java.lang=ALL-UNNAMED
      --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
      --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
      --add-opens=java.base/java.io=ALL-UNNAMED
      --add-opens=java.base/java.net=ALL-UNNAMED
      --add-opens=java.base/java.nio=ALL-UNNAMED
      --add-opens=java.base/java.util=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
      --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
      --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
      --add-opens=java.base/sun.security.action=ALL-UNNAMED
      --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
      -Djdk.reflect.useDirectMethodHandle=false
    </extraJavaTestArgs>
  </properties>
  <repositories>
    <repository>
      <id>gcs-maven-central-mirror</id>
      <!--
        Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
        See https://storage-download.googleapis.com/maven-central/index.html
      -->
      <name>GCS Maven Central mirror</name>
      <url>https://maven-central.storage-download.googleapis.com/maven2/</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </repository>
    <repository>
      <!--
        This is used as a fallback when the first try fails.
      -->
      <id>central</id>
      <name>Maven Repository</name>
      <url>https://repo.maven.apache.org/maven2</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </repository>
  </repositories>
  <pluginRepositories>
    <pluginRepository>
      <id>gcs-maven-central-mirror</id>
      <!--
        Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
        See https://storage-download.googleapis.com/maven-central/index.html
      -->
      <name>GCS Maven Central mirror</name>
      <url>https://maven-central.storage-download.googleapis.com/maven2/</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </pluginRepository>
    <pluginRepository>
      <id>central</id>
      <url>https://repo.maven.apache.org/maven2</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </pluginRepository>
  </pluginRepositories>
  <dependencies>
    <!--
      This is a dummy dependency that is used to trigger the maven-shade plugin so that Spark's
      published POMs are flattened and do not contain variables. Without this dependency, some
      subprojects' published POMs would contain variables like ${scala.binary.version} that will
      be substituted according to the default properties instead of the ones determined by the
      profiles that were active during publishing, causing the Scala 2.10 build's POMs to have 2.11
      dependencies due to the incorrect substitutions. By ensuring that maven-shade runs for all
      subprojects, we eliminate this problem because the substitutions are baked into the final POM.

      For more details, see SPARK-3812 and MNG-2971.
    -->
    <dependency>
      <groupId>org.spark-project.spark</groupId>
      <artifactId>unused</artifactId>
      <version>1.0.0</version>
    </dependency>
    <!--
         This is needed by the scalatest plugin, and so is declared here to be available in
         all child modules, just as scalatest is run in all children
    -->
    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>scalacheck-1-17_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>mockito-4-6_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>selenium-4-4_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>com.github.sbt</groupId>
      <artifactId>junit-interface</artifactId>
      <scope>test</scope>
    </dependency>
  </dependencies>
  <dependencyManagement>
    <dependencies>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-tags_${scala.binary.version}</artifactId>
        <version>${project.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-tags_${scala.binary.version}</artifactId>
        <version>${project.version}</version>
        <type>test-jar</type>
      </dependency>
      <!-- #if scala-2.13 --><!--
      <dependency>
        <groupId>org.scala-lang.modules</groupId>
        <artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
        <version>1.0.3</version>
      </dependency>
      --><!-- #endif scala-2.13 -->
      <dependency>
        <groupId>com.twitter</groupId>
        <artifactId>chill_${scala.binary.version}</artifactId>
        <version>${chill.version}</version>
        <exclusions>
          <exclusion>
            <groupId>org.apache.xbean</groupId>
            <artifactId>xbean-asm7-shaded</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>com.twitter</groupId>
        <artifactId>chill-java</artifactId>
        <version>${chill.version}</version>
      </dependency>
      <dependency>
        <groupId>com.github.jnr</groupId>
        <artifactId>jnr-posix</artifactId>
        <version>3.1.15</version>
        <scope>test</scope>
      </dependency>
      <!-- This artifact is a shaded version of ASM 9.x. The POM that was used to produce this
           is at https://github.com/apache/geronimo-xbean/tree/trunk/xbean-asm9-shaded
           For context on why we shade ASM, see SPARK-782 and SPARK-6152. -->
      <dependency>
        <groupId>org.apache.xbean</groupId>
        <artifactId>xbean-asm9-shaded</artifactId>
        <version>4.22</version>
      </dependency>

      <!-- Shaded deps marked as provided. These are promoted to compile scope
           in the modules where we want the shaded classes to appear in the
           associated jar. -->
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-http</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-continuation</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-servlet</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-servlets</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-proxy</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-client</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-util</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-security</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-plus</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-server</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.eclipse.jetty</groupId>
        <artifactId>jetty-webapp</artifactId>
        <version>${jetty.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>com.google.guava</groupId>
        <artifactId>guava</artifactId>
        <version>${guava.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.jpmml</groupId>
        <artifactId>pmml-model</artifactId>
        <version>1.4.8</version>
        <scope>provided</scope>
        <exclusions>
          <exclusion>
            <groupId>org.jpmml</groupId>
            <artifactId>pmml-agent</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <!-- End of shaded deps -->

      <!-- Provide a JAXB impl; no longer auto available in Java 9+ in the JDK -->
      <dependency>
        <groupId>org.glassfish.jaxb</groupId>
        <artifactId>jaxb-runtime</artifactId>
        <version>2.3.2</version>
        <scope>compile</scope>
        <exclusions>
          <!-- for now, we only write XML in PMML export, and these can be excluded -->
          <exclusion>
            <groupId>com.sun.xml.fastinfoset</groupId>
            <artifactId>FastInfoset</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.glassfish.jaxb</groupId>
            <artifactId>txw2</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jvnet.staxex</groupId>
            <artifactId>stax-ex</artifactId>
          </exclusion>
          <!--
            SPARK-27611: Exclude redundant javax.activation implementation, which
            conflicts with the existing javax.activation:activation:1.1.1 dependency.
            -->
          <exclusion>
            <groupId>jakarta.activation</groupId>
            <artifactId>jakarta.activation-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-lang3</artifactId>
        <version>${commons-lang3.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-text</artifactId>
        <version>1.10.0</version>
      </dependency>
      <dependency>
        <groupId>commons-lang</groupId>
        <artifactId>commons-lang</artifactId>
        <version>${commons-lang2.version}</version>
      </dependency>
      <dependency>
        <groupId>commons-io</groupId>
        <artifactId>commons-io</artifactId>
        <version>${commons-io.version}</version>
      </dependency>
      <dependency>
        <groupId>commons-codec</groupId>
        <artifactId>commons-codec</artifactId>
        <version>${commons-codec.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-compress</artifactId>
        <version>${commons-compress.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-math3</artifactId>
        <version>${commons.math3.version}</version>
      </dependency>
      <dependency>
        <groupId>commons-collections</groupId>
        <artifactId>commons-collections</artifactId>
        <version>${commons.collections.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-collections4</artifactId>
        <version>${commons.collections4.version}</version>
      </dependency>
      <dependency>
        <groupId>commons-beanutils</groupId>
        <artifactId>commons-beanutils</artifactId>
        <version>1.9.4</version>
      </dependency>
      <dependency>
        <groupId>commons-logging</groupId>
        <artifactId>commons-logging</artifactId>
        <!-- Hive uses commons-logging 1.1.3 from 0.13 to 1.2 -->
        <version>1.1.3</version>
      </dependency>
      <dependency>
        <groupId>org.apache.ivy</groupId>
        <artifactId>ivy</artifactId>
        <version>${ivy.version}</version>
      </dependency>
      <dependency>
        <groupId>com.google.code.findbugs</groupId>
        <artifactId>jsr305</artifactId>
        <version>${jsr305.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>${commons.httpclient.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpmime</artifactId>
        <version>${commons.httpclient.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpcore</artifactId>
        <version>${commons.httpcore.version}</version>
      </dependency>
      <dependency>
        <groupId>org.rocksdb</groupId>
        <artifactId>rocksdbjni</artifactId>
        <version>7.7.3</version>
      </dependency>
      <dependency>
        <groupId>${leveldbjni.group}</groupId>
        <artifactId>leveldbjni-all</artifactId>
        <version>1.8</version>
      </dependency>
      <dependency>
        <groupId>org.seleniumhq.selenium</groupId>
        <artifactId>selenium-java</artifactId>
        <version>${selenium.version}</version>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.auto.service</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>net.bytebuddy</groupId>
            <artifactId>byte-buddy</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.seleniumhq.selenium</groupId>
        <artifactId>htmlunit-driver</artifactId>
        <version>${htmlunit-driver.version}</version>
        <scope>test</scope>
      </dependency>
      <!-- Update htmlunit dependency that selenium uses for better JS support -->
      <dependency>
        <groupId>net.sourceforge.htmlunit</groupId>
        <artifactId>htmlunit</artifactId>
        <version>${htmlunit.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>net.sourceforge.htmlunit</groupId>
        <artifactId>htmlunit-core-js</artifactId>
        <version>${htmlunit.version}</version>
        <scope>test</scope>
      </dependency>
      <!-- Added for selenium only, and should match its dependent version: -->
      <dependency>
        <groupId>xml-apis</groupId>
        <artifactId>xml-apis</artifactId>
        <version>1.4.01</version>
      </dependency>

      <!-- log4j -->
      <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-api</artifactId>
        <version>${slf4j.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>jul-to-slf4j</artifactId>
        <version>${slf4j.version}</version>
      </dependency>
      <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>jcl-over-slf4j</artifactId>
        <version>${slf4j.version}</version>
        <!-- runtime scope is appropriate, but causes SBT build problems -->
      </dependency>

      <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-slf4j2-impl</artifactId>
        <version>${log4j.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-api</artifactId>
        <version>${log4j.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-core</artifactId>
        <version>${log4j.version}</version>
      </dependency>
      <dependency>
        <!-- API bridge between log4j 1 and 2 -->
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-1.2-api</artifactId>
        <version>${log4j.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>

      <!-- end -->

      <dependency>
        <groupId>com.ning</groupId>
        <artifactId>compress-lzf</artifactId>
        <version>1.1</version>
      </dependency>
      <dependency>
        <groupId>org.xerial.snappy</groupId>
        <artifactId>snappy-java</artifactId>
        <version>${snappy.version}</version>
      </dependency>
      <dependency>
        <groupId>org.lz4</groupId>
        <artifactId>lz4-java</artifactId>
        <version>1.8.0</version>
      </dependency>
      <dependency>
        <groupId>com.github.luben</groupId>
        <artifactId>zstd-jni</artifactId>
        <version>1.5.2-5</version>
      </dependency>
      <dependency>
        <groupId>com.clearspring.analytics</groupId>
        <artifactId>stream</artifactId>
        <version>2.9.6</version>
        <exclusions>
          <!-- Only HyperLogLogPlus is used, which doesn't depend on fastutil -->
          <exclusion>
            <groupId>it.unimi.dsi</groupId>
            <artifactId>fastutil</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <!-- In theory we need not directly depend on protobuf since Spark does not directly
           use it. However, when building with Hadoop/YARN 2.2 Maven doesn't correctly bump
           the protobuf version up from the one Mesos gives. For now we include this variable
           to explicitly bump the version when building with YARN. It would be nice to figure
           out why Maven can't resolve this correctly (like SBT does). -->
      <dependency>
        <groupId>com.google.protobuf</groupId>
        <artifactId>protobuf-java</artifactId>
        <version>${protobuf.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.roaringbitmap</groupId>
        <artifactId>RoaringBitmap</artifactId>
        <version>0.9.32</version>
      </dependency>

      <!-- Netty Begin -->
      <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-all</artifactId>
        <version>${netty.version}</version>
        <exclusions>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-dns</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-haproxy</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-memcache</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-mqtt</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-redis</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-smtp</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-stomp</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec-xml</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-resolver-dns</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-resolver-dns-classes-macos</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-resolver-dns-native-macos</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-transport-rxtx</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-transport-sctp</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-transport-udt</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jctools</groupId>
            <artifactId>jctools-core</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <!-- SPARK-38885: After Netty 4.1.76, add the following `Netty` dependencies explicitly
           to ensure `./dev/test-dependencies.sh` produce the same results on Linux and MacOS.
      -->
      <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-transport-native-epoll</artifactId>
        <version>${netty.version}</version>
        <classifier>linux-x86_64</classifier>
      </dependency>
      <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-transport-native-epoll</artifactId>
        <version>${netty.version}</version>
        <classifier>linux-aarch_64</classifier>
      </dependency>
      <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-transport-native-kqueue</artifactId>
        <version>${netty.version}</version>
        <classifier>osx-aarch_64</classifier>
      </dependency>
      <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-transport-native-kqueue</artifactId>
        <version>${netty.version}</version>
        <classifier>osx-x86_64</classifier>
      </dependency>
      <!-- SPARK-38541: Add the explicit dependency on `netty-tcnative-classes` because the dependency on
           `netty-tcnative-classes` is changed to `Optional` in [netty](https://github.com/netty/netty/pull/12146).
           `netty-tcnative-classes` module is mainly used by SSL package in `netty-handler`.
           We need to ensure that `netty-tcnative-classes` uses the corresponding version when Netty upgrade.
      -->
      <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-tcnative-classes</artifactId>
        <version>${netty-tcnative-classes.version}</version>
      </dependency>
      <!-- Netty End -->

      <dependency>
        <groupId>org.apache.derby</groupId>
        <artifactId>derby</artifactId>
        <version>${derby.version}</version>
      </dependency>
      <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-core</artifactId>
        <version>${codahale.metrics.version}</version>
      </dependency>
      <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-jvm</artifactId>
        <version>${codahale.metrics.version}</version>
      </dependency>
      <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-json</artifactId>
        <version>${codahale.metrics.version}</version>
      </dependency>
      <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-graphite</artifactId>
        <version>${codahale.metrics.version}</version>
      </dependency>
      <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-jmx</artifactId>
        <version>${codahale.metrics.version}</version>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-core</artifactId>
        <version>${fasterxml.jackson.version}</version>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>${fasterxml.jackson.databind.version}</version>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-annotations</artifactId>
        <version>${fasterxml.jackson.version}</version>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.datatype</groupId>
        <artifactId>jackson-datatype-jsr310</artifactId>
        <version>${fasterxml.jackson.version}</version>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.module</groupId>
        <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
        <version>${fasterxml.jackson.version}</version>
      </dependency>
      <dependency>
        <groupId>com.fasterxml.jackson.module</groupId>
        <artifactId>jackson-module-jaxb-annotations</artifactId>
        <version>${fasterxml.jackson.version}</version>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey.core</groupId>
        <artifactId>jersey-server</artifactId>
        <version>${jersey.version}</version>
        <!-- SPARK-28765 Unused JDK11-specific dependency -->
        <exclusions>
          <exclusion>
            <groupId>jakarta.xml.bind</groupId>
            <artifactId>jakarta.xml.bind-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey.core</groupId>
        <artifactId>jersey-common</artifactId>
        <version>${jersey.version}</version>
        <!-- SPARK-28765 Unused JDK11-specific dependency -->
        <exclusions>
          <exclusion>
            <groupId>com.sun.activation</groupId>
            <artifactId>jakarta.activation</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey.core</groupId>
        <artifactId>jersey-client</artifactId>
        <version>${jersey.version}</version>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey.containers</groupId>
        <artifactId>jersey-container-servlet</artifactId>
        <version>${jersey.version}</version>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey.containers</groupId>
        <artifactId>jersey-container-servlet-core</artifactId>
        <version>${jersey.version}</version>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey.inject</groupId>
        <artifactId>jersey-hk2</artifactId>
        <version>${jersey.version}</version>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey.test-framework.providers</groupId>
        <artifactId>jersey-test-framework-provider-simple</artifactId>
        <version>${jersey.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.glassfish.jersey</groupId>
        <artifactId>jersey-client</artifactId>
        <version>${jersey.version}</version>
      </dependency>
      <dependency>
        <groupId>javax.ws.rs</groupId>
        <artifactId>javax.ws.rs-api</artifactId>
        <version>2.0.1</version>
      </dependency>
      <dependency>
        <groupId>javax.xml.bind</groupId>
        <artifactId>jaxb-api</artifactId>
        <version>2.2.11</version>
      </dependency>
      <dependency>
        <groupId>org.scalanlp</groupId>
        <artifactId>breeze_${scala.binary.version}</artifactId>
        <version>2.1.0</version>
        <exclusions>
          <exclusion>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-math3</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>com.chuusai</groupId>
        <artifactId>shapeless_${scala.binary.version}</artifactId>
        <version>2.3.9</version>
      </dependency>
      <dependency>
        <groupId>org.json4s</groupId>
        <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
        <version>3.7.0-M11</version>
        <exclusions>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.scala-lang.modules</groupId>
        <artifactId>scala-xml_${scala.binary.version}</artifactId>
        <version>2.1.0</version>
      </dependency>
      <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-compiler</artifactId>
        <version>${scala.version}</version>
        <exclusions>
          <exclusion>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-xml_2.12</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-reflect</artifactId>
        <version>${scala.version}</version>
      </dependency>
      <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-library</artifactId>
        <version>${scala.version}</version>
      </dependency>
      <dependency>
        <groupId>org.scala-lang.modules</groupId>
        <artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
        <version>1.1.2</version>
      </dependency>
      <dependency>
        <groupId>jline</groupId>
        <artifactId>jline</artifactId>
        <version>2.14.6</version>
      </dependency>
      <dependency>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest_${scala.binary.version}</artifactId>
        <version>3.2.14</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.scalatestplus</groupId>
        <artifactId>scalacheck-1-17_${scala.binary.version}</artifactId>
        <version>3.2.14.0</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.scalatestplus</groupId>
        <artifactId>mockito-4-6_${scala.binary.version}</artifactId>
        <version>3.2.14.0</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.scalatestplus</groupId>
        <artifactId>selenium-4-4_${scala.binary.version}</artifactId>
        <version>3.2.14.0</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.mockito</groupId>
        <artifactId>mockito-core</artifactId>
        <version>4.6.1</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.mockito</groupId>
        <artifactId>mockito-inline</artifactId>
        <version>4.6.1</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.jmock</groupId>
        <artifactId>jmock-junit4</artifactId>
        <scope>test</scope>
        <version>2.12.0</version>
      </dependency>
      <dependency>
        <groupId>org.scalacheck</groupId>
        <artifactId>scalacheck_${scala.binary.version}</artifactId>
        <version>1.17.0</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.13.2</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.github.sbt</groupId>
        <artifactId>junit-interface</artifactId>
        <version>0.13.3</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.spotify</groupId>
        <artifactId>docker-client</artifactId>
        <version>8.14.1</version>
        <scope>test</scope>
        <classifier>shaded</classifier>
        <exclusions>
          <exclusion>
            <artifactId>guava</artifactId>
            <groupId>com.google.guava</groupId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>mysql</groupId>
        <artifactId>mysql-connector-java</artifactId>
        <version>8.0.30</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.mariadb.jdbc</groupId>
        <artifactId>mariadb-java-client</artifactId>
        <version>2.7.4</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.postgresql</groupId>
        <artifactId>postgresql</artifactId>
        <version>42.5.0</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.ibm.db2</groupId>
        <artifactId>jcc</artifactId>
        <version>11.5.6.0</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.microsoft.sqlserver</groupId>
        <artifactId>mssql-jdbc</artifactId>
        <version>9.4.0.jre8</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.oracle.database.jdbc</groupId>
        <artifactId>ojdbc8</artifactId>
        <version>21.3.0.0</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-recipes</artifactId>
        <version>${curator.version}</version>
        <scope>${hadoop.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>jline</groupId>
            <artifactId>jline</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-client</artifactId>
        <version>${curator.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-framework</artifactId>
        <version>${curator.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-test</artifactId>
        <version>${curator.version}</version>
        <scope>test</scope>
      </dependency>
      <!-- Hadoop 3.x dependencies -->
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>${hadoop-client-api.artifact}</artifactId>
        <version>${hadoop.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>${hadoop-client-runtime.artifact}</artifactId>
        <version>${hadoop.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>${hadoop-client-minicluster.artifact}</artifactId>
        <version>${yarn.version}</version>
        <scope>test</scope>
      </dependency>
      <!-- End of Hadoop 3.x dependencies -->
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>${hadoop.version}</version>
        <scope>${hadoop.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.fusesource.leveldbjni</groupId>
            <artifactId>leveldbjni-all</artifactId>
          </exclusion>
          <exclusion>
            <groupId>asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.codehaus.jackson</groupId>
            <artifactId>jackson-mapper-asl</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.ow2.asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <!-- BeanUtils >= 1.9.0 no longer splits out -core; exclude it -->
            <groupId>commons-beanutils</groupId>
            <artifactId>commons-beanutils-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.mockito</groupId>
            <artifactId>mockito-all</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.mortbay.jetty</groupId>
            <artifactId>servlet-api-2.5</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.jersey-test-framework</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.contribs</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>net.java.dev.jets3t</groupId>
            <artifactId>jets3t</artifactId>
          </exclusion>
          <!-- Hadoop-3.x -->
          <exclusion>
            <groupId>javax.ws.rs</groupId>
            <artifactId>jsr311-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.eclipse.jetty</groupId>
            <artifactId>jetty-webapp</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-minikdc</artifactId>
        <version>${hadoop.version}</version>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.directory.api</groupId>
            <artifactId>api-ldap-schema-data</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-reload4j</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.bouncycastle</groupId>
        <artifactId>bcprov-jdk15on</artifactId>
        <version>${bouncycastle.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.bouncycastle</groupId>
        <artifactId>bcpkix-jdk15on</artifactId>
        <version>${bouncycastle.version}</version>
        <scope>test</scope>
      </dependency>
      <!-- Managed up to match Hadoop in HADOOP-16530 -->
      <dependency>
        <groupId>xerces</groupId>
        <artifactId>xercesImpl</artifactId>
        <version>2.12.2</version>
      </dependency>
      <dependency>
        <groupId>org.apache.avro</groupId>
        <artifactId>avro</artifactId>
        <version>${avro.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.avro</groupId>
        <artifactId>avro-mapred</artifactId>
        <version>${avro.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro-ipc-jetty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.mortbay.jetty</groupId>
            <artifactId>jetty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.mortbay.jetty</groupId>
            <artifactId>jetty-util</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.mortbay.jetty</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.velocity</groupId>
            <artifactId>velocity-engine-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.annotation</groupId>
            <artifactId>javax.annotation-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.github.luben</groupId>
            <artifactId>zstd-jni</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.tukaani</groupId>
        <artifactId>xz</artifactId>
        <version>1.8</version>
      </dependency>
      <!-- See SPARK-23654 for info on this dependency;
      It is used to keep javax.activation at v1.1.1 after dropping
      jets3t as a dependency.
       -->
      <dependency>
        <groupId>javax.activation</groupId>
        <artifactId>activation</artifactId>
        <version>1.1.1</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-yarn-api</artifactId>
        <version>${yarn.version}</version>
        <scope>${hadoop.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.ow2.asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.jersey-test-framework</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.contribs</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>jdk.tools</groupId>
            <artifactId>jdk.tools</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-yarn-common</artifactId>
        <version>${yarn.version}</version>
        <scope>${hadoop.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.ow2.asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.jersey-test-framework</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.contribs</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-yarn-server-tests</artifactId>
        <version>${yarn.version}</version>
        <classifier>tests</classifier>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>org.fusesource.leveldbjni</groupId>
            <artifactId>leveldbjni-all</artifactId>
          </exclusion>
          <exclusion>
            <groupId>asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.ow2.asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.jersey-test-framework</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.contribs</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <!--
        Hack to exclude org.apache.hadoop:hadoop-yarn-server-resourcemanager:jar:tests.
        For some reasons, SBT starts to pull the dependencies of 'hadoop-yarn-server-tests' above
        with 'tests' classifier after upgrading SBT 1.3 (SPARK-21708). Otherwise, some tests might
        fail, see also SPARK-33104.
      -->
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
        <version>${yarn.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-yarn-server-web-proxy</artifactId>
        <version>${yarn.version}</version>
        <scope>${hadoop.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-server-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.bouncycastle</groupId>
            <artifactId>bcprov-jdk15on</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.bouncycastle</groupId>
            <artifactId>bcpkix-jdk15on</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.fusesource.leveldbjni</groupId>
            <artifactId>leveldbjni-all</artifactId>
          </exclusion>
          <exclusion>
            <groupId>asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.ow2.asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>javax.servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.jersey-test-framework</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.contribs</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <!-- Hadoop-3.x -->
          <exclusion>
            <groupId>com.zaxxer</groupId>
            <artifactId>HikariCP-java7</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.microsoft.sqlserver</groupId>
            <artifactId>mssql-jdbc</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-yarn-client</artifactId>
        <version>${yarn.version}</version>
        <scope>${hadoop.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.ow2.asm</groupId>
            <artifactId>asm</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.jersey-test-framework</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.sun.jersey.contribs</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.zookeeper</groupId>
        <artifactId>zookeeper</artifactId>
        <version>${zookeeper.version}</version>
        <scope>${hadoop.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.jboss.netty</groupId>
            <artifactId>netty</artifactId>
          </exclusion>
          <exclusion>
            <groupId>jline</groupId>
            <artifactId>jline</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-handler</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-transport-native-epoll</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.github.spotbugs</groupId>
            <artifactId>spotbugs-annotations</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.codehaus.jackson</groupId>
        <artifactId>jackson-core-asl</artifactId>
        <version>${codehaus.jackson.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.codehaus.jackson</groupId>
        <artifactId>jackson-mapper-asl</artifactId>
        <version>${codehaus.jackson.version}</version>
        <scope>${hadoop.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.codehaus.jackson</groupId>
        <artifactId>jackson-xc</artifactId>
        <version>${codehaus.jackson.version}</version>
      </dependency>
      <dependency>
        <groupId>org.codehaus.jackson</groupId>
        <artifactId>jackson-jaxrs</artifactId>
        <version>${codehaus.jackson.version}</version>
      </dependency>
      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-beeline</artifactId>
        <version>${hive.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-exec</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-jdbc</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-metastore</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-service</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-service-rpc</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-shims</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-cli</artifactId>
        <version>${hive.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-exec</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-jdbc</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-metastore</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-serde</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-service</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-shims</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-common</artifactId>
        <version>${hive.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-shims</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.ant</groupId>
            <artifactId>ant</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-auth</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.zookeeper</groupId>
            <artifactId>zookeeper</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <!-- Begin of Hive 2.3 exclusion -->
          <!--
            ORC is needed, but the version should be consistent with the `sql/core` ORC data source.
            Looks like this is safe, please see the major changes from ORC 1.3.3 to 1.5.4:
            HIVE-17631 and HIVE-19465
          -->
          <exclusion>
            <groupId>org.apache.orc</groupId>
            <artifactId>orc-core</artifactId>
          </exclusion>
          <!-- jetty-all conflict with jetty 9.4.12.v20180830 -->
          <exclusion>
            <groupId>org.eclipse.jetty.aggregate</groupId>
            <artifactId>jetty-all</artifactId>
          </exclusion>
          <!-- org.apache.logging.log4j:* conflict with log4j 1.2.17 -->
          <exclusion>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <!-- Hive includes javax.servlet to fix the Hive on Spark test failure; see HIVE-12783 -->
          <exclusion>
            <groupId>org.eclipse.jetty.orbit</groupId>
            <artifactId>javax.servlet</artifactId>
          </exclusion>
          <!-- hive-storage-api is needed and must be explicitly included later -->
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-storage-api</artifactId>
          </exclusion>
          <!-- End of Hive 2.3 exclusion -->
        </exclusions>
      </dependency>

      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-exec</artifactId>
        <classifier>${hive.classifier}</classifier>
        <version>${hive.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>

          <!-- pull this in when needed; the explicit definition culls the surplus-->
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-metastore</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-shims</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-ant</artifactId>
          </exclusion>
          <!-- break the loop -->
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>spark-client</artifactId>
          </exclusion>

          <!-- excluded dependencies & transitive.
           Some may be needed to be explicitly included-->
          <exclusion>
            <groupId>ant</groupId>
            <artifactId>ant</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.ant</groupId>
            <artifactId>ant</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.esotericsoftware.kryo</groupId>
            <artifactId>kryo</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-codec</groupId>
            <artifactId>commons-codec</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro-mapred</artifactId>
          </exclusion>
          <!--  Do not need Calcite because we disabled hive.cbo.enable -->
          <exclusion>
            <groupId>org.apache.calcite</groupId>
            <artifactId>calcite-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.calcite</groupId>
            <artifactId>calcite-avatica</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>apache-curator</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-client</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-framework</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libfb303</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.zookeeper</groupId>
            <artifactId>zookeeper</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.codehaus.groovy</groupId>
            <artifactId>groovy-all</artifactId>
          </exclusion>
          <exclusion>
            <groupId>jline</groupId>
            <artifactId>jline</artifactId>
          </exclusion>
          <!-- Cat X license now; see SPARK-18262 -->
          <exclusion>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>
          </exclusion>
          <!-- Begin of Hive 2.3 exclusion -->
          <!-- Do not need Tez -->
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-llap-tez</artifactId>
          </exclusion>
          <!-- Do not need Calcite, see SPARK-27054 -->
          <exclusion>
            <groupId>org.apache.calcite</groupId>
            <artifactId>calcite-druid</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.calcite.avatica</groupId>
            <artifactId>avatica</artifactId>
          </exclusion>
          <!-- org.apache.logging.log4j:* conflict with log4j 1.2.17 -->
          <exclusion>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>net.hydromatic</groupId>
            <artifactId>eigenbase-properties</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.codehaus.janino</groupId>
            <artifactId>commons-compiler</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.codehaus.janino</groupId>
            <artifactId>janino</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.pentaho</groupId>
            <artifactId>pentaho-aggdesigner-algorithm</artifactId>
          </exclusion>
          <!-- End of Hive 2.3 exclusion -->
        </exclusions>
      </dependency>
      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-jdbc</artifactId>
        <version>${hive.version}</version>
        <exclusions>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-metastore</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-serde</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-service</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-service-rpc</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-shims</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-framework</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libfb303</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.zookeeper</groupId>
            <artifactId>zookeeper</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.codehaus.groovy</groupId>
            <artifactId>groovy-all</artifactId>
          </exclusion>
        </exclusions>
      </dependency>

      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-metastore</artifactId>
        <version>${hive.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-serde</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-shims</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libfb303</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.mortbay.jetty</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <!-- Begin of Hive 2.3 exclusion -->
          <!-- Hive removes the HBase Metastore; see HIVE-17234 -->
          <exclusion>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
          </exclusion>
          <exclusion>
            <groupId>co.cask.tephra</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <!-- End of Hive 2.3 exclusion -->
        </exclusions>
      </dependency>

      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-serde</artifactId>
        <version>${hive.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-shims</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-codec</groupId>
            <artifactId>commons-codec</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.code.findbugs</groupId>
            <artifactId>jsr305</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libfb303</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.codehaus.groovy</groupId>
            <artifactId>groovy-all</artifactId>
          </exclusion>
          <!-- Begin of Hive 2.3 exclusion -->
          <exclusion>
            <groupId>${hive.group}</groupId>
            <artifactId>hive-service-rpc</artifactId>
          </exclusion>
          <!-- parquet-hadoop-bundle:1.8.1 conflict with 1.12.3 -->
          <exclusion>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-hadoop-bundle</artifactId>
          </exclusion>
          <!-- Do not need Jasper, see HIVE-19799 -->
          <exclusion>
            <groupId>tomcat</groupId>
            <artifactId>jasper-compiler</artifactId>
          </exclusion>
          <exclusion>
            <groupId>tomcat</groupId>
            <artifactId>jasper-runtime</artifactId>
          </exclusion>
          <!-- End of Hive 2.3 exclusion -->
        </exclusions>
      </dependency>

      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-service-rpc</artifactId>
        <version>3.1.3</version>
        <exclusions>
          <exclusion>
            <groupId>*</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>net.sf.jpam</groupId>
        <artifactId>jpam</artifactId>
        <scope>${hive.deps.scope}</scope>
        <version>${jpam.version}</version>
        <exclusions>
          <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>

      <!-- hive shims pulls in hive 0.23 and a transitive dependency of the Hadoop version
        Hive was built against. This dependency cuts out the YARN/hadoop dependency, which
        is needed by Hive to submit work to a YARN cluster.-->
      <dependency>
        <groupId>${hive.group}</groupId>
        <artifactId>hive-shims</artifactId>
        <version>${hive.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-framework</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.zookeeper</groupId>
            <artifactId>zookeeper</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
          </exclusion>
          <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.codehaus.groovy</groupId>
            <artifactId>groovy-all</artifactId>
          </exclusion>
          <!-- Begin of Hive 2.3 exclusion -->
          <!-- Exclude log4j-slf4j-impl, otherwise throw NCDFE when starting spark-shell -->
          <exclusion>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-slf4j-impl</artifactId>
          </exclusion>
          <!-- End of Hive 2.3 exclusion -->
        </exclusions>
      </dependency>

      <!-- hive-llap-common is needed when registering UDFs in Hive 2.3.
         We add it here, otherwise -Phive-provided won't work. -->
      <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-llap-common</artifactId>
        <version>${hive23.version}</version>
        <scope>${hive.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-serde</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <!-- hive-llap-client is needed when run MapReduce test in Hive 2.3. -->
      <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-llap-client</artifactId>
        <version>${hive23.version}</version>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-serde</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-llap-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>curator-framework</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.curator</groupId>
            <artifactId>apache-curator</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.zookeeper</groupId>
            <artifactId>zookeeper</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>

      <dependency>
        <groupId>org.apache.orc</groupId>
        <artifactId>orc-core</artifactId>
        <version>${orc.version}</version>
        <classifier>${orc.classifier}</classifier>
        <scope>${orc.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-storage-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.orc</groupId>
        <artifactId>orc-mapreduce</artifactId>
        <version>${orc.version}</version>
        <classifier>${orc.classifier}</classifier>
        <scope>${orc.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.orc</groupId>
            <artifactId>orc-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-storage-api</artifactId>
          </exclusion>
          <exclusion>
            <groupId> com.esotericsoftware</groupId>
            <artifactId>kryo-shaded</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-column</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-encoding</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.test.deps.scope}</scope>
        <classifier>tests</classifier>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-common</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.test.deps.scope}</scope>
        <classifier>tests</classifier>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-column</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.test.deps.scope}</scope>
        <classifier>tests</classifier>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-hadoop</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.deps.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>commons-pool</groupId>
            <artifactId>commons-pool</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.annotation</groupId>
            <artifactId>javax.annotation-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-avro</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.test.deps.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.codehaus.janino</groupId>
        <artifactId>janino</artifactId>
        <version>${janino.version}</version>
      </dependency>
      <dependency>
        <groupId>org.codehaus.janino</groupId>
        <artifactId>commons-compiler</artifactId>
        <version>${janino.version}</version>
      </dependency>
      <dependency>
        <groupId>joda-time</groupId>
        <artifactId>joda-time</artifactId>
        <version>${joda.version}</version>
      </dependency>
      <dependency>
        <groupId>org.jodd</groupId>
        <artifactId>jodd-core</artifactId>
        <version>${jodd.version}</version>
      </dependency>
      <dependency>
        <groupId>org.datanucleus</groupId>
        <artifactId>datanucleus-core</artifactId>
        <version>${datanucleus-core.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.thrift</groupId>
        <artifactId>libthrift</artifactId>
        <version>${libthrift.version}</version>
        <exclusions>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.thrift</groupId>
        <artifactId>libfb303</artifactId>
        <version>0.9.3</version>
        <exclusions>
          <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.antlr</groupId>
        <artifactId>antlr4-runtime</artifactId>
        <version>${antlr4.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-crypto</artifactId>
        <version>${commons-crypto.version}</version>
        <exclusions>
          <exclusion>
            <groupId>net.java.dev.jna</groupId>
            <artifactId>jna</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>com.google.crypto.tink</groupId>
        <artifactId>tink</artifactId>
        <version>${tink.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.arrow</groupId>
        <artifactId>arrow-vector</artifactId>
        <version>${arrow.version}</version>
        <exclusions>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-common</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.arrow</groupId>
        <artifactId>arrow-memory-netty</artifactId>
        <version>${arrow.version}</version>
        <exclusions>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-buffer</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-common</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>com.univocity</groupId>
        <artifactId>univocity-parsers</artifactId>
        <version>2.9.1</version>
      </dependency>
      <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-storage-api</artifactId>
        <version>${hive.storage.version}</version>
        <scope>${hive.storage.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>commons-lang</groupId>
            <artifactId>commons-lang</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>commons-cli</groupId>
        <artifactId>commons-cli</artifactId>
        <version>${commons-cli.version}</version>
      </dependency>
      <dependency>
        <groupId>dev.ludovic.netlib</groupId>
        <artifactId>blas</artifactId>
        <version>${netlib.ludovic.dev.version}</version>
      </dependency>
      <dependency>
        <groupId>dev.ludovic.netlib</groupId>
        <artifactId>lapack</artifactId>
        <version>${netlib.ludovic.dev.version}</version>
      </dependency>
      <dependency>
        <groupId>dev.ludovic.netlib</groupId>
        <artifactId>arpack</artifactId>
        <version>${netlib.ludovic.dev.version}</version>
      </dependency>
    </dependencies>
  </dependencyManagement>

  <build>
    <pluginManagement>
      <plugins>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-enforcer-plugin</artifactId>
          <version>3.0.0-M2</version>
          <executions>
            <execution>
              <id>enforce-versions</id>
              <goals>
                <goal>enforce</goal>
              </goals>
              <configuration>
                <rules>
                  <requireMavenVersion>
                    <version>${maven.version}</version>
                  </requireMavenVersion>
                  <requireJavaVersion>
                    <version>${java.version}</version>
                  </requireJavaVersion>
                  <bannedDependencies>
                    <excludes>
                      <exclude>org.jboss.netty</exclude>
                      <exclude>org.codehaus.groovy</exclude>
                      <exclude>*:*_2.11</exclude>
                      <exclude>*:*_2.10</exclude>
                    </excludes>
                    <searchTransitive>true</searchTransitive>
                  </bannedDependencies>
                </rules>
              </configuration>
            </execution>
            <execution>
              <id>enforce-no-duplicate-dependencies</id>
              <goals>
                <goal>enforce</goal>
              </goals>
              <configuration>
                <rules>
                  <banDuplicatePomDependencyVersions/>
                </rules>
              </configuration>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>build-helper-maven-plugin</artifactId>
          <version>3.2.0</version>
          <executions>
            <execution>
              <id>module-timestamp-property</id>
              <phase>validate</phase>
              <goals>
                <goal>timestamp-property</goal>
              </goals>
              <configuration>
                <name>module.build.timestamp</name>
                <pattern>${maven.build.timestamp.format}</pattern>
                <timeSource>current</timeSource>
                <timeZone>America/Los_Angeles</timeZone>
              </configuration>
            </execution>
            <execution>
              <id>local-timestamp-property</id>
              <phase>validate</phase>
              <goals>
                <goal>timestamp-property</goal>
              </goals>
              <configuration>
                <name>local.build.timestamp</name>
                <pattern>${maven.build.timestamp.format}</pattern>
                <timeSource>build</timeSource>
                <timeZone>America/Los_Angeles</timeZone>
              </configuration>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>net.alchim31.maven</groupId>
          <artifactId>scala-maven-plugin</artifactId>
          <version>${scala-maven-plugin.version}</version>
          <executions>
            <execution>
              <id>eclipse-add-source</id>
              <goals>
                <goal>add-source</goal>
              </goals>
            </execution>
            <execution>
              <id>scala-compile-first</id>
              <goals>
                <goal>compile</goal>
              </goals>
            </execution>
            <execution>
              <id>scala-test-compile-first</id>
              <goals>
                <goal>testCompile</goal>
              </goals>
            </execution>
            <execution>
              <id>attach-scaladocs</id>
              <phase>verify</phase>
              <goals>
                <goal>doc-jar</goal>
              </goals>
            </execution>
          </executions>
          <configuration>
            <scalaVersion>${scala.version}</scalaVersion>
            <checkMultipleScalaVersions>true</checkMultipleScalaVersions>
            <failOnMultipleScalaVersions>true</failOnMultipleScalaVersions>
            <recompileMode>incremental</recompileMode>
            <args>
              <arg>-unchecked</arg>
              <arg>-deprecation</arg>
              <arg>-feature</arg>
              <arg>-explaintypes</arg>
              <arg>-target:jvm-1.8</arg>
              <arg>-Xfatal-warnings</arg>
              <arg>-Ywarn-unused:imports</arg>
              <arg>-P:silencer:globalFilters=.*deprecated.*</arg>
            </args>
            <jvmArgs>
              <jvmArg>-Xss128m</jvmArg>
              <jvmArg>-Xms4g</jvmArg>
              <jvmArg>-Xmx4g</jvmArg>
              <jvmArg>-XX:MaxMetaspaceSize=2g</jvmArg>
              <jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
            </jvmArgs>
            <javacArgs>
              <javacArg>-source</javacArg>
              <javacArg>${java.version}</javacArg>
              <javacArg>-target</javacArg>
              <javacArg>${java.version}</javacArg>
              <javacArg>-Xlint:all,-serial,-path,-try</javacArg>
            </javacArgs>
            <compilerPlugins>
              <compilerPlugin>
                <groupId>com.github.ghik</groupId>
                <artifactId>silencer-plugin_${scala.version}</artifactId>
                <version>1.7.10</version>
              </compilerPlugin>
            </compilerPlugins>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-compiler-plugin</artifactId>
          <version>3.10.1</version>
          <configuration>
            <source>${java.version}</source>
            <target>${java.version}</target>
            <skipMain>true</skipMain> <!-- skip compile -->
            <skip>true</skip> <!-- skip testCompile -->
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.antlr</groupId>
          <artifactId>antlr4-maven-plugin</artifactId>
          <version>${antlr4.version}</version>
        </plugin>
        <!-- Surefire runs all Java tests -->
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-surefire-plugin</artifactId>
          <version>3.0.0-M7</version>
          <!-- Note config is repeated in scalatest config -->
          <configuration>
            <includes>
              <include>**/Test*.java</include>
              <include>**/*Test.java</include>
              <include>**/*TestCase.java</include>
              <include>**/*Suite.java</include>
            </includes>
            <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
            <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
            <environmentVariables>
              <!--
                Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
                launched by the tests have access to the correct test-time classpath.
              -->
              <SPARK_DIST_CLASSPATH>${test_classpath}</SPARK_DIST_CLASSPATH>
              <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
              <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
              <SPARK_TESTING>1</SPARK_TESTING>
              <JAVA_HOME>${test.java.home}</JAVA_HOME>
              <SPARK_BEELINE_OPTS>-DmyKey=yourValue</SPARK_BEELINE_OPTS>
            </environmentVariables>
            <systemProperties>
              <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
              <derby.system.durability>test</derby.system.durability>
              <java.awt.headless>true</java.awt.headless>
              <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
              <spark.test.home>${spark.test.home}</spark.test.home>
              <spark.testing>1</spark.testing>
              <spark.master.rest.enabled>false</spark.master.rest.enabled>
              <spark.ui.enabled>false</spark.ui.enabled>
              <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
              <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
              <spark.memory.debugFill>true</spark.memory.debugFill>
              <spark.hadoop.hadoop.security.key.provider.path>test:///</spark.hadoop.hadoop.security.key.provider.path>
              <!-- Needed by sql/hive tests. -->
              <test.src.tables>src</test.src.tables>
            </systemProperties>
            <failIfNoTests>false</failIfNoTests>
            <failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
            <excludedGroups>${test.exclude.tags}</excludedGroups>
            <groups>${test.include.tags}</groups>
          </configuration>
          <executions>
            <execution>
              <id>test</id>
              <goals>
                <goal>test</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <!-- Scalatest runs all Scala tests -->
        <plugin>
          <groupId>org.scalatest</groupId>
          <artifactId>scalatest-maven-plugin</artifactId>
          <version>${scalatest-maven-plugin.version}</version>
          <!-- Note config is repeated in surefire config -->
          <configuration>
            <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
            <junitxml>.</junitxml>
            <filereports>SparkTestSuite.txt</filereports>
            <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
            <stderr/>
            <environmentVariables>
              <!--
                Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
                launched by the tests have access to the correct test-time classpath.
              -->
              <SPARK_DIST_CLASSPATH>${test_classpath}</SPARK_DIST_CLASSPATH>
              <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
              <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
              <SPARK_TESTING>1</SPARK_TESTING>
              <JAVA_HOME>${test.java.home}</JAVA_HOME>
            </environmentVariables>
            <systemProperties>
              <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
              <derby.system.durability>test</derby.system.durability>
              <java.awt.headless>true</java.awt.headless>
              <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
              <spark.test.home>${spark.test.home}</spark.test.home>
              <spark.testing>1</spark.testing>
              <spark.ui.enabled>false</spark.ui.enabled>
              <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
              <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
              <spark.test.webdriver.chrome.driver>${spark.test.webdriver.chrome.driver}</spark.test.webdriver.chrome.driver>
              <spark.test.docker.keepContainer>${spark.test.docker.keepContainer}</spark.test.docker.keepContainer>
              <spark.test.docker.removePulledImage>${spark.test.docker.removePulledImage}</spark.test.docker.removePulledImage>
              <!-- Needed by sql/hive tests. -->
              <test.src.tables>__not_used__</test.src.tables>
            </systemProperties>
            <tagsToExclude>${test.exclude.tags},${test.default.exclude.tags}</tagsToExclude>
            <tagsToInclude>${test.include.tags}</tagsToInclude>
          </configuration>
          <executions>
            <execution>
              <id>test</id>
              <goals>
                <goal>test</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-jar-plugin</artifactId>
          <version>3.2.2</version>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-antrun-plugin</artifactId>
          <version>${maven-antrun.version}</version>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-source-plugin</artifactId>
          <version>3.1.0</version>
          <configuration>
            <attach>true</attach>
          </configuration>
          <executions>
            <execution>
              <id>create-source-jar</id>
              <goals>
                <goal>jar-no-fork</goal>
                <goal>test-jar-no-fork</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-clean-plugin</artifactId>
          <version>3.1.0</version>
          <configuration>
            <filesets>
              <fileset>
                <directory>work</directory>
              </fileset>
              <fileset>
                <directory>checkpoint</directory>
              </fileset>
              <fileset>
                <directory>lib_managed</directory>
              </fileset>
              <fileset>
                <directory>metastore_db</directory>
              </fileset>
              <fileset>
                <directory>spark-warehouse</directory>
              </fileset>
              <fileset>
                <directory>dist</directory>
              </fileset>
            </filesets>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-javadoc-plugin</artifactId>
          <version>3.4.0</version>
          <configuration>
            <additionalJOptions>
              <additionalJOption>-Xdoclint:all</additionalJOption>
              <additionalJOption>-Xdoclint:-missing</additionalJOption>
            </additionalJOptions>
            <tags>
              <tag>
                <name>example</name>
                <placement>a</placement>
                <head>Example:</head>
              </tag>
              <tag>
                <name>note</name>
                <placement>a</placement>
                <head>Note:</head>
              </tag>
              <tag>
                <name>group</name>
                <placement>X</placement>
              </tag>
              <tag>
                <name>tparam</name>
                <placement>X</placement>
              </tag>
              <tag>
                <name>constructor</name>
                <placement>X</placement>
              </tag>
              <tag>
                <name>todo</name>
                <placement>X</placement>
              </tag>
              <tag>
                <name>groupname</name>
                <placement>X</placement>
              </tag>
            </tags>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>exec-maven-plugin</artifactId>
          <version>${exec-maven-plugin.version}</version>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-assembly-plugin</artifactId>
          <version>3.1.0</version>
          <configuration>
            <tarLongFileMode>posix</tarLongFileMode>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-shade-plugin</artifactId>
          <version>3.2.4</version>
          <dependencies>
            <dependency>
              <groupId>org.ow2.asm</groupId>
              <artifactId>asm</artifactId>
              <version>9.4</version>
            </dependency>
            <dependency>
              <groupId>org.ow2.asm</groupId>
              <artifactId>asm-commons</artifactId>
              <version>9.4</version>
            </dependency>
          </dependencies>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-install-plugin</artifactId>
          <version>3.0.0-M1</version>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-deploy-plugin</artifactId>
          <version>3.0.0-M1</version>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-dependency-plugin</artifactId>
          <version>3.3.0</version>
          <executions>
            <execution>
              <id>default-cli</id>
              <goals>
                 <goal>build-classpath</goal>
              </goals>
              <configuration>
                <!-- This includes dependencies with 'runtime' and 'compile' scopes;
                     see the docs for includeScope for more details -->
                <includeScope>runtime</includeScope>
              </configuration>
            </execution>
          </executions>
        </plugin>
      </plugins>
    </pluginManagement>

    <plugins>
      <!-- This plugin dumps the test classpath into a file -->
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-dependency-plugin</artifactId>
        <executions>
          <execution>
            <id>generate-test-classpath</id>
            <phase>test-compile</phase>
            <goals>
              <goal>build-classpath</goal>
            </goals>
            <configuration>
              <includeScope>test</includeScope>
              <outputProperty>test_classpath</outputProperty>
            </configuration>
          </execution>
          <execution>
            <id>copy-module-dependencies</id>
            <phase>${build.copyDependenciesPhase}</phase>
            <goals>
              <goal>copy-dependencies</goal>
            </goals>
            <configuration>
              <includeScope>runtime</includeScope>
              <outputDirectory>${jars.target.dir}</outputDirectory>
            </configuration>
          </execution>
        </executions>
      </plugin>

      <!--
        The shade plug-in is used here to create effective pom's (see SPARK-3812), and also
        remove references from the shaded libraries from artifacts published by Spark.
      -->
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <configuration>
          <shadedArtifactAttached>false</shadedArtifactAttached>
          <artifactSet>
            <includes>
              <include>org.spark-project.spark:unused</include>
              <include>org.eclipse.jetty:jetty-io</include>
              <include>org.eclipse.jetty:jetty-http</include>
              <include>org.eclipse.jetty:jetty-proxy</include>
              <include>org.eclipse.jetty:jetty-client</include>
              <include>org.eclipse.jetty:jetty-continuation</include>
              <include>org.eclipse.jetty:jetty-servlet</include>
              <include>org.eclipse.jetty:jetty-servlets</include>
              <include>org.eclipse.jetty:jetty-plus</include>
              <include>org.eclipse.jetty:jetty-security</include>
              <include>org.eclipse.jetty:jetty-util</include>
              <include>org.eclipse.jetty:jetty-server</include>
              <include>com.google.guava:guava</include>
              <include>org.jpmml:*</include>
            </includes>
          </artifactSet>
          <relocations>
            <relocation>
              <pattern>org.eclipse.jetty</pattern>
              <shadedPattern>${spark.shade.packageName}.jetty</shadedPattern>
              <includes>
                <include>org.eclipse.jetty.**</include>
              </includes>
            </relocation>
            <relocation>
              <pattern>com.google.common</pattern>
              <shadedPattern>${spark.shade.packageName}.guava</shadedPattern>
            </relocation>
            <relocation>
              <pattern>org.dmg.pmml</pattern>
              <shadedPattern>${spark.shade.packageName}.dmg.pmml</shadedPattern>
            </relocation>
            <relocation>
              <pattern>org.jpmml</pattern>
              <shadedPattern>${spark.shade.packageName}.jpmml</shadedPattern>
            </relocation>
          </relocations>
        </configuration>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-enforcer-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-source-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.scalastyle</groupId>
        <artifactId>scalastyle-maven-plugin</artifactId>
        <version>1.0.0</version>
        <configuration>
          <verbose>false</verbose>
          <failOnViolation>true</failOnViolation>
          <includeTestSourceDirectory>false</includeTestSourceDirectory>
          <failOnWarning>false</failOnWarning>
          <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
          <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
          <configLocation>scalastyle-config.xml</configLocation>
          <outputFile>${basedir}/target/scalastyle-output.xml</outputFile>
          <inputEncoding>${project.build.sourceEncoding}</inputEncoding>
          <outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
        </configuration>
        <executions>
          <execution>
            <goals>
              <goal>check</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-checkstyle-plugin</artifactId>
        <version>3.1.2</version>
        <configuration>
          <failOnViolation>false</failOnViolation>
          <includeTestSourceDirectory>true</includeTestSourceDirectory>
          <sourceDirectories>
            <directory>${basedir}/src/main/java</directory>
            <directory>${basedir}/src/main/scala</directory>
          </sourceDirectories>
          <testSourceDirectories>
            <directory>${basedir}/src/test/java</directory>
          </testSourceDirectories>
          <configLocation>dev/checkstyle.xml</configLocation>
          <outputFile>${basedir}/target/checkstyle-output.xml</outputFile>
          <inputEncoding>${project.build.sourceEncoding}</inputEncoding>
          <outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
        </configuration>
        <dependencies>
          <dependency>
            <!--
              If you are changing the dependency setting for checkstyle plugin,
              please check project/plugins.sbt too.
            -->
            <groupId>com.puppycrawl.tools</groupId>
            <artifactId>checkstyle</artifactId>
            <version>9.3</version>
          </dependency>
        </dependencies>
        <executions>
          <execution>
            <goals>
              <goal>check</goal>
            </goals>
          </execution>
        </executions>
      </plugin>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-antrun-plugin</artifactId>
        <executions>
          <execution>
            <id>create-tmp-dir</id>
            <phase>generate-test-resources</phase>
            <goals>
              <goal>run</goal>
            </goals>
            <configuration>
              <target>
                <mkdir dir="${project.build.directory}/tmp" />
              </target>
            </configuration>
          </execution>
        </executions>
      </plugin>

      <!-- Enable surefire and scalatest in all children, in one place: -->
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-surefire-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest-maven-plugin</artifactId>
      </plugin>
      <!-- Build test-jar's for all projects, since some projects depend on tests from others -->
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-jar-plugin</artifactId>
        <executions>
          <execution>
            <id>prepare-test-jar</id>
            <phase>${build.testJarPhase}</phase>
            <goals>
              <goal>test-jar</goal>
            </goals>
            <configuration>
              <excludes>
                <exclude>log4j2.properties</exclude>
              </excludes>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.antipathy</groupId>
        <artifactId>mvn-scalafmt_${scala.binary.version}</artifactId>
        <version>1.1.1640084764.9f463a9</version>
        <configuration>
          <validateOnly>${scalafmt.validateOnly}</validateOnly> <!-- (Optional) skip formatting -->
          <skipSources>${scalafmt.skip}</skipSources>
          <skipTestSources>${scalafmt.skip}</skipTestSources>
          <configLocation>dev/.scalafmt.conf</configLocation> <!-- (Optional) config location -->
          <onlyChangedFiles>${scalafmt.changedOnly}</onlyChangedFiles>
        </configuration>
        <executions>
          <execution>
            <phase>validate</phase>
            <goals>
              <goal>format</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
      <!--
        Couple of dependencies are coming in bundle format (bundle is just a normal jar which
        contains OSGi metadata in the manifest). If one don't use OSGi, then a bundle will work as
        any other jar. Since maven doesn't have native bundle support it needs an external plugin
        handle it. If the plugin is not added then the build can't resolve bundle dependencies.
      -->
      <plugin>
        <groupId>org.apache.felix</groupId>
        <artifactId>maven-bundle-plugin</artifactId>
        <version>4.2.0</version>
        <extensions>true</extensions>
      </plugin>
    </plugins>
  </build>

  <profiles>

    <!--
      This profile is enabled automatically by the sbt build. It changes the scope for shaded
      dependencies, since we don't shade it in the artifacts generated by the sbt build.
    -->
    <profile>
      <id>sbt</id>
      <dependencies>
        <dependency>
          <groupId>com.google.guava</groupId>
          <artifactId>guava</artifactId>
          <scope>compile</scope>
        </dependency>
        <dependency>
          <groupId>org.jpmml</groupId>
          <artifactId>pmml-model</artifactId>
          <scope>compile</scope>
        </dependency>
      </dependencies>
    </profile>

    <!-- Ganglia integration is not included by default due to LGPL-licensed code -->
    <profile>
      <id>spark-ganglia-lgpl</id>
      <modules>
        <module>connector/spark-ganglia-lgpl</module>
      </modules>
    </profile>

    <!-- Kinesis integration is not included by default due to ASL-licensed code -->
    <profile>
      <id>kinesis-asl</id>
      <modules>
        <module>connector/kinesis-asl</module>
        <module>connector/kinesis-asl-assembly</module>
      </modules>
    </profile>

    <profile>
      <id>docker-integration-tests</id>
      <modules>
        <module>connector/docker-integration-tests</module>
      </modules>
    </profile>

    <!-- A series of build profiles where customizations for particular Hadoop releases can be made -->

    <!-- Hadoop-a.b.c dependencies can be found at
    http://hadoop.apache.org/docs/ra.b.c/hadoop-project-dist/hadoop-common/dependency-analysis.html
    -->

    <profile>
      <id>hadoop-2</id>
      <properties>
        <!-- make sure to update IsolatedClientLoader whenever this version is changed -->
        <hadoop.version>2.7.4</hadoop.version>
        <curator.version>2.7.1</curator.version>
        <commons-io.version>2.4</commons-io.version>
        <!--
          the declaration site above of these variables explains why we need to re-assign them here
        -->
        <hadoop-client-api.artifact>hadoop-client</hadoop-client-api.artifact>
        <hadoop-client-runtime.artifact>hadoop-yarn-api</hadoop-client-runtime.artifact>
        <hadoop-client-minicluster.artifact>hadoop-client</hadoop-client-minicluster.artifact>
        <gcs-connector.version>hadoop2-2.2.7</gcs-connector.version>
        <!-- SPARK-36547: Please don't upgrade the version below, otherwise there will be an error on building Hadoop 2.7 package -->
        <scala-maven-plugin.version>4.3.0</scala-maven-plugin.version>
      </properties>
    </profile>

    <profile>
      <id>hadoop-3</id>
      <!-- Default hadoop profile. Uses global properties. -->
    </profile>

    <profile>
      <id>yarn</id>
      <modules>
        <module>resource-managers/yarn</module>
        <module>common/network-yarn</module>
      </modules>
    </profile>

    <profile>
      <id>mesos</id>
      <modules>
        <module>resource-managers/mesos</module>
      </modules>
    </profile>

    <profile>
      <id>kubernetes</id>
      <modules>
        <module>resource-managers/kubernetes/core</module>
      </modules>
    </profile>

    <!-- generally not enabled for automated builds, but will run k8s tests -->
    <profile>
      <id>kubernetes-integration-tests</id>
      <modules>
        <module>resource-managers/kubernetes/integration-tests</module>
      </modules>
    </profile>

    <profile>
      <id>hive-thriftserver</id>
      <modules>
        <module>sql/hive-thriftserver</module>
      </modules>
    </profile>

    <profile>
      <id>hadoop-cloud</id>
      <modules>
        <module>hadoop-cloud</module>
      </modules>
    </profile>

    <profile>
      <id>test-java-home</id>
      <activation>
        <property><name>env.JAVA_HOME</name></property>
      </activation>
      <properties>
        <test.java.home>${env.JAVA_HOME}</test.java.home>
      </properties>
    </profile>

    <profile>
      <id>scala-2.12</id>
      <properties>
        <!--
         SPARK-34774 Add this property to ensure change-scala-version.sh can replace the public `scala.version`
         property correctly.
        -->
        <scala.version>2.12.17</scala.version>
      </properties>
      <build>
        <pluginManagement>
          <plugins>
          </plugins>
        </pluginManagement>
      </build>
    </profile>

    <profile>
      <id>scala-2.13</id>
      <properties>
        <scala.version>2.13.8</scala.version>
        <scala.binary.version>2.13</scala.binary.version>
      </properties>
      <build>
        <pluginManagement>
          <plugins>
            <plugin>
              <groupId>net.alchim31.maven</groupId>
              <artifactId>scala-maven-plugin</artifactId>
              <configuration>
                <args>
                  <arg>-unchecked</arg>
                  <arg>-deprecation</arg>
                  <arg>-feature</arg>
                  <arg>-explaintypes</arg>
                  <arg>-target:jvm-1.8</arg>
                  <arg>-Wconf:cat=deprecation:wv,any:e</arg>
                  <!--
                    TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed
                    <arg>-Wunused:imports</arg>
                  -->
                  <arg>-Wconf:cat=scaladoc:wv</arg>
                  <arg>-Wconf:cat=lint-multiarg-infix:wv</arg>
                  <arg>-Wconf:cat=other-nullary-override:wv</arg>
                  <arg>-Wconf:cat=other-match-analysis&amp;site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv</arg>
                  <arg>-Wconf:cat=other-pure-statement&amp;site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv</arg>
                  <arg>-Wconf:cat=other-pure-statement&amp;site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.&lt;local OutputCommitCoordinatorSuite&gt;.futureAction:wv</arg>
                  <!--
                    SPARK-33775 Suppress compilation warnings that contain the following contents.
                    TODO(SPARK-33805): Undo the corresponding deprecated usage suppression rule after fixed.
                  -->
                  <arg>-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s</arg>
                  <arg>-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s</arg>
                  <arg>-Wconf:msg=Auto-application to \`\(\)\` is deprecated:s</arg>
                  <arg>-Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s</arg>
                  <arg>-Wconf:msg=method without a parameter list overrides a method with a single empty one:s</arg>
                  <!--
                    SPARK-35574 Prevent the recurrence of compilation warnings related to
                    `procedure syntax is deprecated`
                  -->
                  <arg>-Wconf:cat=deprecation&amp;msg=procedure syntax is deprecated:e</arg>
                  <!--
                    SPARK-35496 Upgrade Scala to 2.13.7 and suppress:
                    1. `The outer reference in this type test cannot be checked at run time`
                    2. `the type test for pattern TypeA cannot be checked at runtime because it
                     has type parameters eliminated by erasure`
                    3. `abstract type TypeA in type pattern Seq[TypeA] (the underlying of
                    Seq[TypeA]) is unchecked since it is eliminated by erasure`
                    4. `fruitless type test: a value of TypeA cannot also be a TypeB`
                  -->
                  <arg>-Wconf:cat=unchecked&amp;msg=outer reference:s</arg>
                  <arg>-Wconf:cat=unchecked&amp;msg=eliminated by erasure:s</arg>
                  <arg>-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s</arg>
                </args>
                <compilerPlugins combine.self="override">
                </compilerPlugins>
              </configuration>
            </plugin>
          </plugins>
        </pluginManagement>
      </build>

    </profile>

    <!--
     This is a profile to enable the use of the ASF snapshot and staging repositories
     during a build. It is useful when testing against nightly or RC releases of dependencies.
     It MUST NOT be used when building copies of Spark to use in production of for distribution,
     -->
    <profile>
      <id>snapshots-and-staging</id>
      <properties>
        <!-- override point for ASF staging/snapshot repos -->
        <asf.staging>https://repository.apache.org/content/groups/staging/</asf.staging>
        <asf.snapshots>https://repository.apache.org/content/repositories/snapshots/</asf.snapshots>
      </properties>

      <pluginRepositories>
        <pluginRepository>
          <id>ASF Staging</id>
          <url>${asf.staging}</url>
        </pluginRepository>
        <pluginRepository>
          <id>ASF Snapshots</id>
          <url>${asf.snapshots}</url>
          <snapshots>
            <enabled>true</enabled>
          </snapshots>
          <releases>
            <enabled>false</enabled>
          </releases>
        </pluginRepository>

      </pluginRepositories>
      <repositories>
        <repository>
          <id>ASF Staging</id>
          <url>${asf.staging}</url>
        </repository>
        <repository>
          <id>ASF Snapshots</id>
          <url>${asf.snapshots}</url>
          <snapshots>
            <enabled>true</enabled>
          </snapshots>
          <releases>
            <enabled>false</enabled>
          </releases>
        </repository>
      </repositories>
    </profile>

    <!--
      These empty profiles are available in some sub-modules. Declare them here so that
      maven does not complain when they're provided on the command line for a sub-module
      that does not have them.
    -->
    <profile>
      <id>hadoop-provided</id>
      <properties>
        <spark.yarn.isHadoopProvided>true</spark.yarn.isHadoopProvided>
      </properties>
    </profile>
    <profile>
      <id>hive-provided</id>
    </profile>
    <profile>
      <id>orc-provided</id>
    </profile>
    <profile>
      <id>parquet-provided</id>
    </profile>
    <profile>
      <id>sparkr</id>
    </profile>
    <!-- use org.openlabtesting.leveldbjni on aarch64 platform -->
    <profile>
      <id>aarch64</id>
      <properties>
        <leveldbjni.group>org.openlabtesting.leveldbjni</leveldbjni.group>
      </properties>
      <activation>
        <os>
          <family>linux</family>
          <arch>aarch64</arch>
        </os>
      </activation>
    </profile>
    <profile>
      <id>jdwp-test-debug</id>
      <properties>
        <jdwp.arg.line>-agentlib:jdwp=transport=dt_socket,suspend=${test.jdwp.suspend},server=${test.jdwp.server},address=${test.jdwp.address}</jdwp.arg.line>
        <debugArgLine>${jdwp.arg.line}</debugArgLine>
        <maven.surefire.debug>${jdwp.arg.line}</maven.surefire.debug>
        <debugForkedProcess>${test.debug.suite}</debugForkedProcess>
      </properties>
    </profile>
    <profile>
      <id>only-eclipse</id>
      <activation>
        <property>
          <!-- Eclipse M2E plugin exports this property and auto-activates this profile -->
          <name>m2e.version</name>
        </property>
      </activation>
      <build>
        <pluginManagement>
          <plugins>
            <!-- This plugin's configuration is used to store Eclipse m2e settings only. -->
            <!-- It has no influence on the Maven build itself. -->
            <plugin>
              <groupId>org.eclipse.m2e</groupId>
              <artifactId>lifecycle-mapping</artifactId>
              <version>1.0.0</version>
              <configuration>
                <lifecycleMappingMetadata>
                  <pluginExecutions>
                    <pluginExecution>
                      <pluginExecutionFilter>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-dependency-plugin</artifactId>
                        <versionRange>[2.8,)</versionRange>
                        <goals>
                          <goal>build-classpath</goal>
                        </goals>
                      </pluginExecutionFilter>
                      <action>
                        <ignore></ignore>
                      </action>
                    </pluginExecution>
                    <pluginExecution>
                      <pluginExecutionFilter>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-jar-plugin</artifactId>
                        <versionRange>3.2.2</versionRange>
                        <goals>
                          <goal>test-jar</goal>
                        </goals>
                      </pluginExecutionFilter>
                      <action>
                        <ignore></ignore>
                      </action>
                    </pluginExecution>
                    <pluginExecution>
                      <pluginExecutionFilter>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-antrun-plugin</artifactId>
                        <versionRange>[${maven-antrun.version},)</versionRange>
                        <goals>
                          <goal>run</goal>
                        </goals>
                      </pluginExecutionFilter>
                      <action>
                        <ignore></ignore>
                      </action>
                    </pluginExecution>
                  </pluginExecutions>
                </lifecycleMappingMetadata>
              </configuration>
            </plugin>
          </plugins>
        </pluginManagement>
      </build>
    </profile>
  </profiles>
</project>