From 4cba6eb42031b1a4cc3308833116ca5d9ccb1a89 Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Tue, 6 Jan 2015 12:02:08 -0800 Subject: [PATCH] SPARK-4159 [CORE] Maven build doesn't run JUnit test suites This PR: - Reenables `surefire`, and copies config from `scalatest` (which is itself an old fork of `surefire`, so similar) - Tells `surefire` to test only Java tests - Enables `surefire` and `scalatest` for all children, and in turn eliminates some duplication. For me this causes the Scala and Java tests to be run once each, it seems, as desired. It doesn't affect the SBT build but works for Maven. I still need to verify that all of the Scala tests and Java tests are being run. Author: Sean Owen Closes #3651 from srowen/SPARK-4159 and squashes the following commits: 2e8a0af [Sean Owen] Remove specialized SPARK_HOME setting for REPL, YARN tests as it appears to be obsolete 12e4558 [Sean Owen] Append to unit-test.log instead of overwriting, so that both surefire and scalatest output is preserved. Also standardize/correct comments a bit. e6f8601 [Sean Owen] Reenable Java tests by reenabling surefire with config cloned from scalatest; centralize test config in the parent --- bagel/pom.xml | 11 ----- bagel/src/test/resources/log4j.properties | 4 +- core/pom.xml | 18 -------- core/src/test/resources/log4j.properties | 4 +- examples/pom.xml | 5 --- external/flume-sink/pom.xml | 9 ---- .../src/test/resources/log4j.properties | 3 +- external/flume/pom.xml | 11 ----- .../flume/src/test/resources/log4j.properties | 5 +-- external/kafka/pom.xml | 11 ----- .../kafka/src/test/resources/log4j.properties | 5 +-- external/mqtt/pom.xml | 11 ----- .../mqtt/src/test/resources/log4j.properties | 5 +-- external/twitter/pom.xml | 11 ----- .../src/test/resources/log4j.properties | 5 +-- external/zeromq/pom.xml | 11 ----- .../src/test/resources/log4j.properties | 5 +-- extras/java8-tests/pom.xml | 15 ------- .../src/test/resources/log4j.properties | 2 +- extras/kinesis-asl/pom.xml | 11 ----- .../src/test/resources/log4j.properties | 5 ++- graphx/pom.xml | 11 ----- graphx/src/test/resources/log4j.properties | 4 +- mllib/pom.xml | 11 ----- mllib/src/test/resources/log4j.properties | 4 +- network/common/pom.xml | 5 --- network/shuffle/pom.xml | 5 --- pom.xml | 45 +++++++++++++++++-- repl/pom.xml | 14 ------ repl/src/test/resources/log4j.properties | 4 +- sql/catalyst/pom.xml | 10 ----- sql/core/pom.xml | 11 ----- sql/hive-thriftserver/pom.xml | 9 ---- sql/hive/pom.xml | 5 --- streaming/pom.xml | 10 ----- streaming/src/test/resources/log4j.properties | 5 +-- tools/pom.xml | 9 ---- yarn/pom.xml | 14 ------ yarn/src/test/resources/log4j.properties | 4 +- 39 files changed, 70 insertions(+), 277 deletions(-) diff --git a/bagel/pom.xml b/bagel/pom.xml index 0327ffa402671..3bcd38fa3245c 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -44,11 +44,6 @@ org.eclipse.jetty jetty-server - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -58,11 +53,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/bagel/src/test/resources/log4j.properties b/bagel/src/test/resources/log4j.properties index 789869f72e3b0..853ef0ed2986f 100644 --- a/bagel/src/test/resources/log4j.properties +++ b/bagel/src/test/resources/log4j.properties @@ -15,10 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file bagel/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/core/pom.xml b/core/pom.xml index c5c41b2b5de42..d9a49c9e08afc 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -276,11 +276,6 @@ selenium-java test - - org.scalatest - scalatest_${scala.binary.version} - test - org.mockito mockito-all @@ -326,19 +321,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.scalatest - scalatest-maven-plugin - - - test - - test - - - - - org.apache.maven.plugins diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 9dd05f17f012b..287c8e3563503 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -15,10 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file core/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/examples/pom.xml b/examples/pom.xml index 8713230e1e8ed..bdc5d0562f3e1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -244,11 +244,6 @@ algebird-core_${scala.binary.version} 0.8.1 - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 72618b6515f83..71f595d0a6800 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -65,11 +65,6 @@ - - org.scalatest - scalatest_${scala.binary.version} - test - org.scala-lang scala-library @@ -91,10 +86,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.scalatest - scalatest-maven-plugin - org.apache.avro avro-maven-plugin diff --git a/external/flume-sink/src/test/resources/log4j.properties b/external/flume-sink/src/test/resources/log4j.properties index 4411d6e20c52a..2a58e99817224 100644 --- a/external/flume-sink/src/test/resources/log4j.properties +++ b/external/flume-sink/src/test/resources/log4j.properties @@ -17,9 +17,8 @@ # Set everything to be logged to the file streaming/target/unit-tests.log log4j.rootCategory=INFO, file -# log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/external/flume/pom.xml b/external/flume/pom.xml index a682f0e8471d8..0374262212e08 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -61,11 +61,6 @@ - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -85,11 +80,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/external/flume/src/test/resources/log4j.properties b/external/flume/src/test/resources/log4j.properties index 4411d6e20c52a..9697237bfa1a3 100644 --- a/external/flume/src/test/resources/log4j.properties +++ b/external/flume/src/test/resources/log4j.properties @@ -15,11 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file streaming/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file -# log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index b3f44471cd326..b29b0509656ba 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -74,11 +74,6 @@ 3.2 test - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -98,11 +93,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/external/kafka/src/test/resources/log4j.properties b/external/kafka/src/test/resources/log4j.properties index 4411d6e20c52a..9697237bfa1a3 100644 --- a/external/kafka/src/test/resources/log4j.properties +++ b/external/kafka/src/test/resources/log4j.properties @@ -15,11 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file streaming/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file -# log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index d478267b605ba..560c8b9d18276 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -46,11 +46,6 @@ org.eclipse.paho.client.mqttv3 1.0.1 - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -76,11 +71,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/external/mqtt/src/test/resources/log4j.properties b/external/mqtt/src/test/resources/log4j.properties index 4411d6e20c52a..9697237bfa1a3 100644 --- a/external/mqtt/src/test/resources/log4j.properties +++ b/external/mqtt/src/test/resources/log4j.properties @@ -15,11 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file streaming/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file -# log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index 000ace1446e5e..da6ffe7662f63 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -46,11 +46,6 @@ twitter4j-stream 3.0.3 - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -70,11 +65,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/external/twitter/src/test/resources/log4j.properties b/external/twitter/src/test/resources/log4j.properties index 4411d6e20c52a..64bfc5745088f 100644 --- a/external/twitter/src/test/resources/log4j.properties +++ b/external/twitter/src/test/resources/log4j.properties @@ -15,11 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file streaming/target/unit-tests.log +# Set everything to be logged to the filetarget/unit-tests.log log4j.rootCategory=INFO, file -# log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index 29c452093502e..2fb5f0ed2f57c 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -46,11 +46,6 @@ akka-zeromq_${scala.binary.version} ${akka.version} - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -70,11 +65,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/external/zeromq/src/test/resources/log4j.properties b/external/zeromq/src/test/resources/log4j.properties index 4411d6e20c52a..9697237bfa1a3 100644 --- a/external/zeromq/src/test/resources/log4j.properties +++ b/external/zeromq/src/test/resources/log4j.properties @@ -15,11 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file streaming/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file -# log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml index c8477a6566311..0fb431808bacd 100644 --- a/extras/java8-tests/pom.xml +++ b/extras/java8-tests/pom.xml @@ -60,11 +60,6 @@ junit-interface test - - org.scalatest - scalatest_${scala.binary.version} - test - @@ -159,16 +154,6 @@ - - org.scalatest - scalatest-maven-plugin - - - test - none - - - diff --git a/extras/java8-tests/src/test/resources/log4j.properties b/extras/java8-tests/src/test/resources/log4j.properties index bb0ab319a0080..287c8e3563503 100644 --- a/extras/java8-tests/src/test/resources/log4j.properties +++ b/extras/java8-tests/src/test/resources/log4j.properties @@ -18,7 +18,7 @@ # Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml index c0d3a61119113..c815eda52bda7 100644 --- a/extras/kinesis-asl/pom.xml +++ b/extras/kinesis-asl/pom.xml @@ -57,11 +57,6 @@ aws-java-sdk ${aws.java.sdk.version} - - org.scalatest - scalatest_${scala.binary.version} - test - org.mockito mockito-all @@ -86,11 +81,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/extras/kinesis-asl/src/test/resources/log4j.properties b/extras/kinesis-asl/src/test/resources/log4j.properties index d9d08f68687d3..853ef0ed2986f 100644 --- a/extras/kinesis-asl/src/test/resources/log4j.properties +++ b/extras/kinesis-asl/src/test/resources/log4j.properties @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file -# log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/graphx/pom.xml b/graphx/pom.xml index 9982b36f9b62f..91db799d244ad 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -49,11 +49,6 @@ org.eclipse.jetty jetty-server - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -63,11 +58,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - diff --git a/graphx/src/test/resources/log4j.properties b/graphx/src/test/resources/log4j.properties index 9dd05f17f012b..287c8e3563503 100644 --- a/graphx/src/test/resources/log4j.properties +++ b/graphx/src/test/resources/log4j.properties @@ -15,10 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file core/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/mllib/pom.xml b/mllib/pom.xml index 0a6dda0ab8c80..2198757481684 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -80,11 +80,6 @@ org.apache.commons commons-math3 - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -129,12 +124,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - org.scalatest - scalatest-maven-plugin - - ../python diff --git a/mllib/src/test/resources/log4j.properties b/mllib/src/test/resources/log4j.properties index a469badf603c6..9697237bfa1a3 100644 --- a/mllib/src/test/resources/log4j.properties +++ b/mllib/src/test/resources/log4j.properties @@ -15,10 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the file core/target/unit-tests.log +# Set everything to be logged to the file target/unit-tests.log log4j.rootCategory=INFO, file log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/network/common/pom.xml b/network/common/pom.xml index baca859fa5011..245a96b8c4038 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -75,11 +75,6 @@ mockito-all test - - org.scalatest - scalatest_${scala.binary.version} - test - diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index 12468567c3aed..5bfa1ac9c373e 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -83,11 +83,6 @@ mockito-all test - - org.scalatest - scalatest_${scala.binary.version} - test - diff --git a/pom.xml b/pom.xml index 05f59a9b4140b..46ff211f91160 100644 --- a/pom.xml +++ b/pom.xml @@ -256,7 +256,7 @@ 1.0.0 @@ -266,6 +266,15 @@ 2.3.7 provided + + + org.scalatest + scalatest_${scala.binary.version} + test + @@ -935,19 +944,38 @@ true + org.apache.maven.plugins maven-surefire-plugin - 2.17 + 2.18 + - - true + + **/Test*.java + **/*Test.java + **/*TestCase.java + **/*Suite.java + + ${project.build.directory}/surefire-reports + -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m + + true + ${session.executionRootDirectory} + 1 + false + false + ${test_classpath} + true + + org.scalatest scalatest-maven-plugin 1.0 + ${project.build.directory}/surefire-reports . @@ -1159,6 +1187,15 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.scalatest + scalatest-maven-plugin + diff --git a/repl/pom.xml b/repl/pom.xml index 9b2290429fee5..97165e024926e 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -86,11 +86,6 @@ org.slf4j jul-to-slf4j - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -115,15 +110,6 @@ true - - org.scalatest - scalatest-maven-plugin - - - ${basedir}/.. - - - org.codehaus.mojo diff --git a/repl/src/test/resources/log4j.properties b/repl/src/test/resources/log4j.properties index 52098993f5c3c..e7e4a4113174a 100644 --- a/repl/src/test/resources/log4j.properties +++ b/repl/src/test/resources/log4j.properties @@ -15,10 +15,10 @@ # limitations under the License. # -# Set everything to be logged to the repl/target/unit-tests.log +# Set everything to be logged to the target/unit-tests.log log4j.rootCategory=INFO, file log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.append=false +log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 1caa297e24e37..a1947fb022e54 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -50,11 +50,6 @@ spark-core_${scala.binary.version} ${project.version} - - org.scalatest - scalatest_${scala.binary.version} - test - org.scalacheck scalacheck_${scala.binary.version} @@ -65,11 +60,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.scalatest - scalatest-maven-plugin - -