[SPARK-37600][BUILD] Upgrade to Hadoop 3.3.2

### What changes were proposed in this pull request? This PR aims to upgrade to Hadoop 3.3.2. In addition, it also removes the LZ4 wrapper classes added in SPARK-36669, therefore fixing SPARK-36679. ### Why are the changes needed? Hadoop 3.3.2 has many bug fixes and we also can remove our internal hacked Hadoop codecs. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. Closes apache#34855 from sunchao/SPARK-37600. Authored-by: Chao Sun <sunchao@apple.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
iemejia · Mar 9, 2022 · 4da04fc · 4da04fc
1 parent 35c0e5c
commit 4da04fc
Show file tree

Hide file tree

Showing 11 changed files with 98 additions and 144 deletions.
diff --git a/LICENSE-binary b/LICENSE-binary
@@ -456,6 +456,7 @@ net.sf.py4j:py4j
 org.jpmml:pmml-model
 org.jpmml:pmml-schema
 org.threeten:threeten-extra
+org.jdom:jdom2
 
 python/lib/py4j-*-src.zip
 python/pyspark/cloudpickle.py
@@ -504,6 +505,7 @@ Common Development and Distribution License (CDDL) 1.0
 javax.activation:activation  http://www.oracle.com/technetwork/java/javase/tech/index-jsp-138795.html
 javax.xml.stream:stax-api    https://jcp.org/en/jsr/detail?id=173
 javax.transaction:javax.transaction-api
+javax.xml.bind:jaxb-api
 
 
 Common Development and Distribution License (CDDL) 1.1

diff --git a/NOTICE-binary b/NOTICE-binary
@@ -917,6 +917,9 @@ This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin
 g Package (jaspell): http://jaspell.sourceforge.net/
 License: The BSD License (http://www.opensource.org/licenses/bsd-license.php)
 
+This product includes software developed by the JDOM Project (http://www.jdom.org/)
+License: https://raw.githubusercontent.com/hunterhacker/jdom/master/LICENSE.txt
+
 The snowball stemmers in
   analysis/common/src/java/net/sf/snowball
 were developed by Martin Porter and Richard Boulton.

diff --git a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Compressor.java b/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Compressor.java
diff --git a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Factory.java b/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4Factory.java
diff --git a/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4SafeDecompressor.java b/core/src/main/java/org/apache/hadoop/shaded/net/jpountz/lz4/LZ4SafeDecompressor.java
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -6,11 +6,10 @@ ST4/4.0.4//ST4-4.0.4.jar
 activation/1.1.1//activation-1.1.1.jar
 aircompressor/0.21//aircompressor-0.21.jar
 algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar
-aliyun-java-sdk-core/3.4.0//aliyun-java-sdk-core-3.4.0.jar
-aliyun-java-sdk-ecs/4.2.0//aliyun-java-sdk-ecs-4.2.0.jar
-aliyun-java-sdk-ram/3.0.0//aliyun-java-sdk-ram-3.0.0.jar
-aliyun-java-sdk-sts/3.0.0//aliyun-java-sdk-sts-3.0.0.jar
-aliyun-sdk-oss/3.4.1//aliyun-sdk-oss-3.4.1.jar
+aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar
+aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar
+aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar
+aliyun-sdk-oss/3.13.0//aliyun-sdk-oss-3.13.0.jar
 annotations/17.0.0//annotations-17.0.0.jar
 antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
 antlr4-runtime/4.8//antlr4-runtime-4.8.jar
@@ -26,7 +25,7 @@ automaton/1.11-8//automaton-1.11-8.jar
 avro-ipc/1.11.0//avro-ipc-1.11.0.jar
 avro-mapred/1.11.0//avro-mapred-1.11.0.jar
 avro/1.11.0//avro-1.11.0.jar
-aws-java-sdk-bundle/1.11.901//aws-java-sdk-bundle-1.11.901.jar
+aws-java-sdk-bundle/1.11.1026//aws-java-sdk-bundle-1.11.1026.jar
 azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
 azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
 azure-storage/7.0.1//azure-storage-7.0.1.jar
@@ -67,18 +66,18 @@ generex/1.0.2//generex-1.0.2.jar
 gmetric4j/1.0.10//gmetric4j-1.0.10.jar
 gson/2.2.4//gson-2.2.4.jar
 guava/14.0.1//guava-14.0.1.jar
-hadoop-aliyun/3.3.1//hadoop-aliyun-3.3.1.jar
-hadoop-annotations/3.3.1//hadoop-annotations-3.3.1.jar
-hadoop-aws/3.3.1//hadoop-aws-3.3.1.jar
-hadoop-azure-datalake/3.3.1//hadoop-azure-datalake-3.3.1.jar
-hadoop-azure/3.3.1//hadoop-azure-3.3.1.jar
-hadoop-client-api/3.3.1//hadoop-client-api-3.3.1.jar
-hadoop-client-runtime/3.3.1//hadoop-client-runtime-3.3.1.jar
-hadoop-cloud-storage/3.3.1//hadoop-cloud-storage-3.3.1.jar
-hadoop-cos/3.3.1//hadoop-cos-3.3.1.jar
-hadoop-openstack/3.3.1//hadoop-openstack-3.3.1.jar
+hadoop-aliyun/3.3.2//hadoop-aliyun-3.3.2.jar
+hadoop-annotations/3.3.2//hadoop-annotations-3.3.2.jar
+hadoop-aws/3.3.2//hadoop-aws-3.3.2.jar
+hadoop-azure-datalake/3.3.2//hadoop-azure-datalake-3.3.2.jar
+hadoop-azure/3.3.2//hadoop-azure-3.3.2.jar
+hadoop-client-api/3.3.2//hadoop-client-api-3.3.2.jar
+hadoop-client-runtime/3.3.2//hadoop-client-runtime-3.3.2.jar
+hadoop-cloud-storage/3.3.2//hadoop-cloud-storage-3.3.2.jar
+hadoop-cos/3.3.2//hadoop-cos-3.3.2.jar
+hadoop-openstack/3.3.2//hadoop-openstack-3.3.2.jar
 hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar
-hadoop-yarn-server-web-proxy/3.3.1//hadoop-yarn-server-web-proxy-3.3.1.jar
+hadoop-yarn-server-web-proxy/3.3.2//hadoop-yarn-server-web-proxy-3.3.2.jar
 hive-beeline/2.3.9//hive-beeline-2.3.9.jar
 hive-cli/2.3.9//hive-cli-2.3.9.jar
 hive-common/2.3.9//hive-common-2.3.9.jar
@@ -97,9 +96,9 @@ hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
 httpclient/4.5.13//httpclient-4.5.13.jar
 httpcore/4.4.14//httpcore-4.4.14.jar
+ini4j/0.5.4//ini4j-0.5.4.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
 ivy/2.5.0//ivy-2.5.0.jar
 jackson-annotations/2.13.1//jackson-annotations-2.13.1.jar
@@ -121,10 +120,11 @@ janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
 javolution/5.5.1//javolution-5.5.1.jar
+jaxb-api/2.2.11//jaxb-api-2.2.11.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
 jcl-over-slf4j/1.7.32//jcl-over-slf4j-1.7.32.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
-jdom/1.1//jdom-1.1.jar
+jdom2/2.0.6//jdom2-2.0.6.jar
 jersey-client/2.34//jersey-client-2.34.jar
 jersey-common/2.34//jersey-common-2.34.jar
 jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar
@@ -204,6 +204,9 @@ objenesis/3.2//objenesis-3.2.jar
 okhttp/3.12.12//okhttp-3.12.12.jar
 okio/1.14.0//okio-1.14.0.jar
 opencsv/2.3//opencsv-2.3.jar
+opentracing-api/0.33.0//opentracing-api-0.33.0.jar
+opentracing-noop/0.33.0//opentracing-noop-0.33.0.jar
+opentracing-util/0.33.0//opentracing-util-0.33.0.jar
 orc-core/1.7.3//orc-core-1.7.3.jar
 orc-mapreduce/1.7.3//orc-mapreduce-1.7.3.jar
 orc-shims/1.7.3//orc-shims-1.7.3.jar

diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
@@ -267,6 +267,13 @@
               <groupId>com.google.guava</groupId>
               <artifactId>guava</artifactId>
             </exclusion>
+            <exclusion>
+              <!--
+              This is a code coverage library introduced by aliyun-java-sdk-core, only for testing
+              -->
+              <groupId>org.jacoco</groupId>
+              <artifactId>org.jacoco.agent</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
         <!--

diff --git a/licenses/LICENSE-jdom.txt b/licenses/LICENSE-jdom.txt
@@ -0,0 +1,54 @@
+/*-- 
+
+ Copyright (C) 2000-2012 Jason Hunter & Brett McLaughlin.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions, and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions, and the disclaimer that follows 
+    these conditions in the documentation and/or other materials 
+    provided with the distribution.
+
+ 3. The name "JDOM" must not be used to endorse or promote products
+    derived from this software without prior written permission.  For
+    written permission, please contact <request_AT_jdom_DOT_org>.
+
+ 4. Products derived from this software may not be called "JDOM", nor
+    may "JDOM" appear in their name, without prior written permission
+    from the JDOM Project Management <request_AT_jdom_DOT_org>.
+
+ In addition, we request (but do not require) that you include in the 
+ end-user documentation provided with the redistribution and/or in the 
+ software itself an acknowledgement equivalent to the following:
+     "This product includes software developed by the
+      JDOM Project (http://www.jdom.org/)."
+ Alternatively, the acknowledgment may be graphical using the logos 
+ available at http://www.jdom.org/images/logos.
+
+ THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED.  IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGE.
+
+ This software consists of voluntary contributions made by many 
+ individuals on behalf of the JDOM Project and was originally 
+ created by Jason Hunter <jhunter_AT_jdom_DOT_org> and
+ Brett McLaughlin <brett_AT_jdom_DOT_org>.  For more information
+ on the JDOM Project, please see <http://www.jdom.org/>. 
+
+ */
+
diff --git a/pom.xml b/pom.xml
@@ -120,7 +120,8 @@
     <sbt.project.name>spark</sbt.project.name>
     <slf4j.version>1.7.32</slf4j.version>
     <log4j.version>2.17.1</log4j.version>
-    <hadoop.version>3.3.1</hadoop.version>
+    <!-- make sure to update IsolatedClientLoader whenever this version is changed -->
+    <hadoop.version>3.3.2</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>
     <yarn.version>${hadoop.version}</yarn.version>
     <zookeeper.version>3.6.2</zookeeper.version>
@@ -3427,6 +3428,7 @@
     <profile>
       <id>hadoop-2</id>
       <properties>
+        <!-- make sure to update IsolatedClientLoader whenever this version is changed -->
         <hadoop.version>2.7.4</hadoop.version>
         <curator.version>2.7.1</curator.version>
         <commons-io.version>2.4</commons-io.version>

diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
@@ -48,7 +48,12 @@ object MimaExcludes {
     // [SPARK-37780][SQL] QueryExecutionListener support SQLConf as constructor parameter
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.util.ExecutionListenerManager.this"),
     // [SPARK-37786][SQL] StreamingQueryListener support use SQLConf.get to get corresponding SessionState's SQLConf
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryManager.this")
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryManager.this"),
+
+    // [SPARK-37600][BUILD] Upgrade to Hadoop 3.3.2
+    ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Compressor"),
+    ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Factory"),
+    ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4SafeDecompressor")
   )
 
   // Exclude rules for 3.2.x from 3.1.1

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -69,7 +69,7 @@ private[hive] object IsolatedClientLoader extends Logging {
             // If the error message contains hadoop, it is probably because the hadoop
             // version cannot be resolved.
             val fallbackVersion = if (VersionUtils.isHadoop3) {
-              "3.3.1"
+              "3.3.2"
             } else {
               "2.7.4"
             }