Skip to content

Commit

Permalink
HBASE-20332 shaded mapreduce module shouldn't include hadoop
Browse files Browse the repository at this point in the history 
* modify the jar checking script to take args; make hadoop stuff optional
* separate out checking the artifacts that have hadoop vs those that don't.
* * Unfortunately means we need two modules for checking things
* * put in a safety check that the support script for checking jar contents is maintained in both modules
* * have to carve out an exception for o.a.hadoop.metrics2. :(
* fix duplicated class warning
* clean up dependencies in hbase-server and some modules that depend on it.
* allow Hadoop to have its own htrace where it needs it
* add a precommit check to make sure we're not using old htrace imports
  • Loading branch information
@busbey
busbey committed Jun 18, 2018
1 parent ac5bb81 commit f1b536b
Show file tree
Hide file tree
Showing 25 changed files with 789 additions and 265 deletions.
12 changes: 5 additions & 7 deletions hbase-backup/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
Expand Down Expand Up @@ -264,9 +260,6 @@
<value>3.0</value>
</property>
</activation>
<properties>
<hadoop.version>3.0-SNAPSHOT</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand All @@ -276,6 +269,11 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
</profile>
</profiles>
Expand Down
11 changes: 11 additions & 0 deletions hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@
<!DOCTYPE suppressions PUBLIC
"-//Puppy Crawl//DTD Suppressions 1.0//EN"
"http://www.puppycrawl.com/dtds/suppressions_1_0.dtd">
<!--
TODO Update to use the message suppression filter once we can update
to checkstyle 8.6+
<!DOCTYPE suppressions PUBLIC
"-//Checkstyle//DTD SuppressionFilter Configuration 1.2//EN"
"https://checkstyle.org/dtds/suppressions_1_2.dtd">
-->
<!--
/**
* Licensed to the Apache Software Foundation (ASF) under one
Expand Down Expand Up @@ -39,4 +46,8 @@
<suppress checks="EmptyBlockCheck" files="TBoundedThreadPoolServer.java"/>
<suppress checks="EqualsHashCode" files="StartcodeAgnosticServerName.java"/>
<suppress checks="MethodLength" files="Branch1CoprocessorMethods.java"/>
<!--
TODO use message filter once we can upgrade to checkstyle 8.6+
<suppress checks="IllegalImport" message="org\.apache\.htrace\.core"/>
-->
</suppressions>
9 changes: 8 additions & 1 deletion hbase-checkstyle/src/main/resources/hbase/checkstyle.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@
<property name="processJavadoc" value="true"/>
</module>
<module name="IllegalImport">
<!--
TODO include the o.a.htrace package for HTrace v3 once we can upgrade
to checkstyle 8.6 plus and use a message filter to suppress
errors from proper use of o.a.htrace.core
org.apache.htrace,
-->
<property name="illegalPkgs" value="
com.google.common,
io.netty,
Expand All @@ -87,7 +93,8 @@
org.apache.curator.shaded,
org.apache.hadoop.classification,
org.apache.htrace.shaded,
org.codehaus.jackson"/>
org.codehaus.jackson,
org.htrace"/>
<property name="illegalClasses" value="
org.apache.commons.logging.Log,
org.apache.commons.logging.LogFactory"/>
Expand Down
4 changes: 0 additions & 4 deletions hbase-client/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
Expand Down
12 changes: 0 additions & 12 deletions hbase-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,12 +314,6 @@
<artifactId>hadoop-common</artifactId>
<!--FYI This pulls in hadoop's guava. Its needed for Configuration
at least-->
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
Expand Down Expand Up @@ -366,12 +360,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
Expand Down
14 changes: 0 additions & 14 deletions hbase-endpoint/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,12 +260,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down Expand Up @@ -296,10 +290,6 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down Expand Up @@ -343,10 +333,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down
12 changes: 0 additions & 12 deletions hbase-examples/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
Expand Down Expand Up @@ -287,12 +281,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
Expand Down
4 changes: 0 additions & 4 deletions hbase-external-blockcache/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,10 +224,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down
6 changes: 0 additions & 6 deletions hbase-hadoop2-compat/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,12 +167,6 @@ limitations under the License.
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down
6 changes: 0 additions & 6 deletions hbase-it/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,12 +335,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down
30 changes: 18 additions & 12 deletions hbase-mapreduce/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,15 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<exclusions>
<!-- commons-logging is only used by hbase-http's HttpRequestLog and hbase-server's
HBaseTestingUtil. We don't need either of those here, so execlude it.
-->
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
Expand Down Expand Up @@ -246,10 +255,19 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<!-- jackson(s) used by PerformanceEvaluation and it looks like TableMapReduceUtil -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
</dependencies>
<profiles>
<!-- Skip the tests in this module -->
Expand Down Expand Up @@ -284,10 +302,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
Expand Down Expand Up @@ -334,10 +348,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
Expand Down Expand Up @@ -377,10 +387,6 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
Expand Down
4 changes: 0 additions & 4 deletions hbase-replication/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
Expand Down
19 changes: 13 additions & 6 deletions hbase-rest/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,19 @@
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
</dependency>
<dependency>
<!-- We *might* need this for XMLStreamReader use in RemoteAdmin
TODO figure out if we can remove it.
-->
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
<exclusions>
<exclusion>
<groupId>stax</groupId>
<artifactId>stax-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<!--For JspC used in ant task-->
<groupId>org.glassfish.web</groupId>
Expand Down Expand Up @@ -374,12 +387,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down
4 changes: 0 additions & 4 deletions hbase-rsgroup/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,10 +198,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
Expand Down
Loading

0 comments on commit f1b536b

Please sign in to comment.