Allow snippet version overrides to be done by major Hadoop version.

trahasch · Jan 10, 2014 · 51a3d33 · 51a3d33
1 parent fb237f6
commit 51a3d33
Show file tree

Hide file tree

Showing 75 changed files with 26 additions and 52,548 deletions.
diff --git a/README b/README
@@ -27,19 +27,20 @@ Then you can build the code with:
 
 % mvn package -DskipTests
 
-By default Hadoop 1.2.1 is used. This can be changed by specifying the hadoop.version
-property, e.g.
+By default Hadoop 1.2.1 is used. This can be changed by specifying the
+hadoop.version property, e.g.
 
 % mvn package -DskipTests -Dhadoop.version=1.2.0
 
-There are also profiles for different Hadoop major versions,
-specified in hadoop-meta/pom.xml. For example, to use Hadoop 2:
+There are profiles for different Hadoop major versions and distributions,
+specified in hadoop-meta/pom.xml, and they are specified using the hadoop.distro
+property. For example, to use the default version of Hadoop 2:
 
-% mvn package -DskipTests -P apache-2
+% mvn package -DskipTests -Dhadoop.distro=apache-2
 
 Again, you can specify hadoop.version to use a particular Hadoop 2 version:
 
-% mvn package -DskipTests -P apache-2 -Dhadoop.version=2.1.1-beta
+% mvn package -DskipTests -Dhadoop.distro=apache-2 -Dhadoop.version=2.1.1-beta
 
 You should then be able to run the examples from the book.
 

diff --git a/ch02/src/main/examples/max_temperature_py/2.2.0/input.txt b/ch02/src/main/examples/max_temperature_py/2.2.0/input.txt
diff --git a/ch02/src/main/examples/max_temperature_py/2.2.0/output/part-00000 b/ch02/src/main/examples/max_temperature_py/2.2.0/output/part-00000
diff --git a/...ples/max_temperature_py/0.23.10/input.txt → ...n/examples/max_temperature_py/2/input.txt b/...ples/max_temperature_py/0.23.10/input.txt → ...n/examples/max_temperature_py/2/input.txt
diff --git a/..._temperature_py/0.23.10/output/part-00000 → ...es/max_temperature_py/2/output/part-00000 b/..._temperature_py/0.23.10/output/part-00000 → ...es/max_temperature_py/2/output/part-00000
diff --git a/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02907099999-r-00000 b/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02907099999-r-00000
diff --git a/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02950099999-r-00000 b/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02950099999-r-00000
diff --git a/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02960099999-r-00000 b/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02960099999-r-00000
diff --git a/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02972099999-r-00000 b/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02972099999-r-00000
diff --git a/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02981099999-r-00000 b/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-02981099999-r-00000
diff --git a/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-22707099999-r-00000 b/...PartitionByStationUsingMultipleOutputs/0.24.0-SNAPSHOT/output/station-22707099999-r-00000
diff --git a/ch07/src/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/input.txt b/ch07/src/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/input.txt
diff --git a/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029070-99999-r-00000 b/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029070-99999-r-00000
diff --git a/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029500-99999-r-00000 b/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029500-99999-r-00000
diff --git a/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029600-99999-r-00000 b/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029600-99999-r-00000
diff --git a/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029720-99999-r-00000 b/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029720-99999-r-00000
diff --git a/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029810-99999-r-00000 b/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/029810-99999-r-00000
diff --git a/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/227070-99999-r-00000 b/...rc/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/227070-99999-r-00000
diff --git a/ch07/src/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/part-r-00000 b/ch07/src/main/examples/PartitionByStationUsingMultipleOutputs/2.2.0/output/part-r-00000
diff --git a/...ionUsingMultipleOutputs/0.23.10/input.txt → ...ByStationUsingMultipleOutputs/2/input.txt b/...ionUsingMultipleOutputs/0.23.10/input.txt → ...ByStationUsingMultipleOutputs/2/input.txt
diff --git a/...tputs/0.23.10/output/029070-99999-r-00000 → ...ipleOutputs/2/output/029070-99999-r-00000 b/...tputs/0.23.10/output/029070-99999-r-00000 → ...ipleOutputs/2/output/029070-99999-r-00000
diff --git a/...tputs/0.23.10/output/029500-99999-r-00000 → ...ipleOutputs/2/output/029500-99999-r-00000 b/...tputs/0.23.10/output/029500-99999-r-00000 → ...ipleOutputs/2/output/029500-99999-r-00000
diff --git a/...tputs/0.23.10/output/029600-99999-r-00000 → ...ipleOutputs/2/output/029600-99999-r-00000 b/...tputs/0.23.10/output/029600-99999-r-00000 → ...ipleOutputs/2/output/029600-99999-r-00000
diff --git a/...tputs/0.23.10/output/029720-99999-r-00000 → ...ipleOutputs/2/output/029720-99999-r-00000 b/...tputs/0.23.10/output/029720-99999-r-00000 → ...ipleOutputs/2/output/029720-99999-r-00000
diff --git a/...tputs/0.23.10/output/029810-99999-r-00000 → ...ipleOutputs/2/output/029810-99999-r-00000 b/...tputs/0.23.10/output/029810-99999-r-00000 → ...ipleOutputs/2/output/029810-99999-r-00000
diff --git a/...tputs/0.23.10/output/227070-99999-r-00000 → ...ipleOutputs/2/output/227070-99999-r-00000 b/...tputs/0.23.10/output/227070-99999-r-00000 → ...ipleOutputs/2/output/227070-99999-r-00000
diff --git a/...ltipleOutputs/0.23.10/output/part-r-00000 → ...singMultipleOutputs/2/output/part-r-00000 b/...ltipleOutputs/0.23.10/output/part-r-00000 → ...singMultipleOutputs/2/output/part-r-00000
diff --git a/.../src/main/examples/PartitionByStationYearUsingMultipleOutputs/0.23.10/output/part-r-00000 b/.../src/main/examples/PartitionByStationYearUsingMultipleOutputs/0.23.10/output/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029070-99999/1901/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029070-99999/1901/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029070-99999/1902/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029070-99999/1902/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029500-99999/1901/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029500-99999/1901/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029500-99999/1902/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029500-99999/1902/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029600-99999/1901/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029600-99999/1901/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029600-99999/1902/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029600-99999/1902/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029720-99999/1901/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029720-99999/1901/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029720-99999/1902/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029720-99999/1902/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029810-99999/1901/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029810-99999/1901/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029810-99999/1902/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/029810-99999/1902/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/227070-99999/1901/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/227070-99999/1901/part-r-00000
diff --git a/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/227070-99999/1902/part-r-00000 b/...onByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/227070-99999/1902/part-r-00000
diff --git a/...n/examples/PartitionByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/part-r-00000 b/...n/examples/PartitionByStationYearUsingMultipleOutputs/0.24.0-SNAPSHOT/output/part-r-00000
diff --git a/ch07/src/main/examples/PartitionByStationYearUsingMultipleOutputs/2.2.0/input.txt b/ch07/src/main/examples/PartitionByStationYearUsingMultipleOutputs/2.2.0/input.txt
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029070-99999/1901/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029070-99999/1901/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029070-99999/1902/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029070-99999/1902/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029500-99999/1901/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029500-99999/1901/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029500-99999/1902/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029500-99999/1902/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029600-99999/1901/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029600-99999/1901/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029600-99999/1902/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029600-99999/1902/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029720-99999/1901/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029720-99999/1901/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029720-99999/1902/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029720-99999/1902/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029810-99999/1901/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029810-99999/1901/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029810-99999/1902/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/029810-99999/1902/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/227070-99999/1901/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/227070-99999/1901/part-r-00000
diff --git a/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/227070-99999/1902/part-r-00000 b/...es/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/227070-99999/1902/part-r-00000
diff --git a/ch07/src/main/examples/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/part-r-00000 b/ch07/src/main/examples/PartitionByStationYearUsingMultipleOutputs/2.2.0/output/part-r-00000
diff --git a/...earUsingMultipleOutputs/0.23.10/input.txt → ...ationYearUsingMultipleOutputs/2/input.txt b/...earUsingMultipleOutputs/0.23.10/input.txt → ...ationYearUsingMultipleOutputs/2/input.txt
diff --git a/....10/output/029070-99999/1901/part-r-00000 → ...s/2/output/029070-99999/1901/part-r-00000 b/....10/output/029070-99999/1901/part-r-00000 → ...s/2/output/029070-99999/1901/part-r-00000
diff --git a/....10/output/029070-99999/1902/part-r-00000 → ...s/2/output/029070-99999/1902/part-r-00000 b/....10/output/029070-99999/1902/part-r-00000 → ...s/2/output/029070-99999/1902/part-r-00000
diff --git a/....10/output/029500-99999/1901/part-r-00000 → ...s/2/output/029500-99999/1901/part-r-00000 b/....10/output/029500-99999/1901/part-r-00000 → ...s/2/output/029500-99999/1901/part-r-00000
diff --git a/....10/output/029500-99999/1902/part-r-00000 → ...s/2/output/029500-99999/1902/part-r-00000 b/....10/output/029500-99999/1902/part-r-00000 → ...s/2/output/029500-99999/1902/part-r-00000
diff --git a/....10/output/029600-99999/1901/part-r-00000 → ...s/2/output/029600-99999/1901/part-r-00000 b/....10/output/029600-99999/1901/part-r-00000 → ...s/2/output/029600-99999/1901/part-r-00000
diff --git a/....10/output/029600-99999/1902/part-r-00000 → ...s/2/output/029600-99999/1902/part-r-00000 b/....10/output/029600-99999/1902/part-r-00000 → ...s/2/output/029600-99999/1902/part-r-00000
diff --git a/....10/output/029720-99999/1901/part-r-00000 → ...s/2/output/029720-99999/1901/part-r-00000 b/....10/output/029720-99999/1901/part-r-00000 → ...s/2/output/029720-99999/1901/part-r-00000
diff --git a/....10/output/029720-99999/1902/part-r-00000 → ...s/2/output/029720-99999/1902/part-r-00000 b/....10/output/029720-99999/1902/part-r-00000 → ...s/2/output/029720-99999/1902/part-r-00000
diff --git a/....10/output/029810-99999/1901/part-r-00000 → ...s/2/output/029810-99999/1901/part-r-00000 b/....10/output/029810-99999/1901/part-r-00000 → ...s/2/output/029810-99999/1901/part-r-00000
diff --git a/....10/output/029810-99999/1902/part-r-00000 → ...s/2/output/029810-99999/1902/part-r-00000 b/....10/output/029810-99999/1902/part-r-00000 → ...s/2/output/029810-99999/1902/part-r-00000
diff --git a/....10/output/227070-99999/1901/part-r-00000 → ...s/2/output/227070-99999/1901/part-r-00000 b/....10/output/227070-99999/1901/part-r-00000 → ...s/2/output/227070-99999/1901/part-r-00000
diff --git a/....10/output/227070-99999/1902/part-r-00000 → ...s/2/output/227070-99999/1902/part-r-00000 b/....10/output/227070-99999/1902/part-r-00000 → ...s/2/output/227070-99999/1902/part-r-00000
diff --git a/...tputs/0.24.0-SNAPSHOT/output/part-r-00000 → ...singMultipleOutputs/2/output/part-r-00000 b/...tputs/0.24.0-SNAPSHOT/output/part-r-00000 → ...singMultipleOutputs/2/output/part-r-00000
diff --git a/ch08/src/main/examples/JoinRecordWithStationName/0.24.0-SNAPSHOT/output/part-r-00000 b/ch08/src/main/examples/JoinRecordWithStationName/0.24.0-SNAPSHOT/output/part-r-00000
diff --git a/ch08/src/main/examples/JoinRecordWithStationName/2.2.0/input.txt b/ch08/src/main/examples/JoinRecordWithStationName/2.2.0/input.txt
diff --git a/ch08/src/main/examples/JoinRecordWithStationName/2.2.0/output/part-r-00000 b/ch08/src/main/examples/JoinRecordWithStationName/2.2.0/output/part-r-00000
diff --git a/...inRecordWithStationName/0.23.10/input.txt → ...les/JoinRecordWithStationName/2/input.txt b/...inRecordWithStationName/0.23.10/input.txt → ...les/JoinRecordWithStationName/2/input.txt
diff --git a/...thStationName/0.23.10/output/part-r-00000 → ...cordWithStationName/2/output/part-r-00000 b/...thStationName/0.23.10/output/part-r-00000 → ...cordWithStationName/2/output/part-r-00000
diff --git a/snippet/src/test/java/ExamplesIT.java b/snippet/src/test/java/ExamplesIT.java
@@ -92,6 +92,7 @@ public static Collection<Object[]> data() {
   private File actualOutputDir = new File(PROJECT_BASE_DIR, "output");
   private static Map<String, String> env;
   private static String version;
+  private static String majorVersion;
   private static String mode;
 
   public ExamplesIT(File example) {
@@ -125,7 +126,20 @@ public static void setUpClass() throws IOException {
     }
     assertNotNull("Version not found", version);
     System.out.printf("version=%s\n", version);
-
+
+    majorVersion = version.substring(0, version.indexOf('.'));
+    // Treat 0.2x as Hadoop 1 or 2 for the purposes of these tests
+    if ("0".equals(majorVersion)) {
+      if (version.startsWith("0.20")) {
+        majorVersion = "1";
+      } else if (version.startsWith("0.21") || version.startsWith("0.22") ||
+          version.startsWith("0.23")) {
+        majorVersion = "2";
+      }
+    }
+    assertNotNull("Major version not found", majorVersion);
+    System.out.printf("majorVersion=%s\n", majorVersion);
+
   }
 
   @Before
@@ -184,10 +198,13 @@ public void test() throws Exception {
   }
 
   private File findBaseExampleDirectory(File example) {
-    // Look in base/<version>/<mode> then base/<version> then base/<mode>
+    // Look in base/<version>/<mode> then base/<version> then base/<major-version>/<mode>
+    // then base/<major-version> then base/<mode>
     File[] candidates = {
         new File(new File(example, version), mode),
         new File(example, version),
+        new File(new File(example, majorVersion), mode),
+        new File(example, majorVersion),
         new File(example, mode),
     };
     for (File candidate : candidates) {