Skip to content

Commit

Permalink
Tidy up example testing a bit.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomwhite
tomwhite committed Oct 20, 2011
1 parent f849f04 commit 89cf0da
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 94 deletions.
72 changes: 36 additions & 36 deletions hadoop-meta/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,16 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/JoinRecordWithStationName.java</exclude>
<exclude>**/LookupRecordByTemperature.java</exclude>
<exclude>**/LookupRecordsByTemperature.java</exclude>
<exclude>**/MissingTemperatureFields.java</exclude>
<exclude>**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
<exclude>**/PartitionByStationYearUsingMultipleOutputs.java</exclude>
<exclude>**/MaxTemperatureWithMultipleInputs.java</exclude>
<exclude>**/PartitionByStationUsingMultipleOutputs.java</exclude>
<exclude>**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch07/**/MaxTemperatureWithMultipleInputs.java</exclude>
<exclude>**/ch07/**/PartitionByStationUsingMultipleOutputs.java</exclude>
<exclude>**/ch07/**/PartitionByStationYearUsingMultipleOutputs.java</exclude>
<exclude>**/ch08/**/JoinRecordWithStationName.java</exclude>
<exclude>**/ch08/**/LookupRecordByTemperature.java</exclude>
<exclude>**/ch08/**/LookupRecordsByTemperature.java</exclude>
<exclude>**/ch08/**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch08/**/MissingTemperatureFields.java</exclude>
<exclude>**/ch08/**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/ch08/**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
</excludes>
</configuration>
</plugin>
Expand Down Expand Up @@ -93,16 +93,16 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/JoinRecordWithStationName.java</exclude>
<exclude>**/LookupRecordByTemperature.java</exclude>
<exclude>**/LookupRecordsByTemperature.java</exclude>
<exclude>**/MissingTemperatureFields.java</exclude>
<exclude>**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
<exclude>**/PartitionByStationYearUsingMultipleOutputs.java</exclude>
<exclude>**/MaxTemperatureWithMultipleInputs.java</exclude>
<exclude>**/PartitionByStationUsingMultipleOutputs.java</exclude>
<exclude>**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch07/**/MaxTemperatureWithMultipleInputs.java</exclude>
<exclude>**/ch07/**/PartitionByStationUsingMultipleOutputs.java</exclude>
<exclude>**/ch07/**/PartitionByStationYearUsingMultipleOutputs.java</exclude>
<exclude>**/ch08/**/JoinRecordWithStationName.java</exclude>
<exclude>**/ch08/**/LookupRecordByTemperature.java</exclude>
<exclude>**/ch08/**/LookupRecordsByTemperature.java</exclude>
<exclude>**/ch08/**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch08/**/MissingTemperatureFields.java</exclude>
<exclude>**/ch08/**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/ch08/**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
</excludes>
</configuration>
</plugin>
Expand Down Expand Up @@ -133,16 +133,16 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/JoinRecordWithStationName.java</exclude>
<exclude>**/LookupRecordByTemperature.java</exclude>
<exclude>**/LookupRecordsByTemperature.java</exclude>
<exclude>**/MissingTemperatureFields.java</exclude>
<exclude>**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
<exclude>**/PartitionByStationYearUsingMultipleOutputs.java</exclude>
<exclude>**/MaxTemperatureWithMultipleInputs.java</exclude>
<exclude>**/PartitionByStationUsingMultipleOutputs.java</exclude>
<exclude>**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch07/**/MaxTemperatureWithMultipleInputs.java</exclude>
<exclude>**/ch07/**/PartitionByStationUsingMultipleOutputs.java</exclude>
<exclude>**/ch07/**/PartitionByStationYearUsingMultipleOutputs.java</exclude>
<exclude>**/ch08/**/JoinRecordWithStationName.java</exclude>
<exclude>**/ch08/**/LookupRecordByTemperature.java</exclude>
<exclude>**/ch08/**/LookupRecordsByTemperature.java</exclude>
<exclude>**/ch08/**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch08/**/MissingTemperatureFields.java</exclude>
<exclude>**/ch08/**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/ch08/**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
</excludes>
</configuration>
</plugin>
Expand Down Expand Up @@ -173,12 +173,12 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/LookupRecordByTemperature.java</exclude>
<exclude>**/LookupRecordsByTemperature.java</exclude>
<exclude>**/MissingTemperatureFields.java</exclude>
<exclude>**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
<exclude>**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch08/**/LookupRecordByTemperature.java</exclude>
<exclude>**/ch08/**/LookupRecordsByTemperature.java</exclude>
<exclude>**/ch08/**/MaxTemperatureByStationNameUsingDistributedCacheFileApi.java</exclude>
<exclude>**/ch08/**/MissingTemperatureFields.java</exclude>
<exclude>**/ch08/**/SortByTemperatureToMapFile.java</exclude>
<exclude>**/ch08/**/SortByTemperatureUsingTotalOrderPartitioner.java</exclude>
</excludes>
</configuration>
</plugin>
Expand Down
29 changes: 0 additions & 29 deletions snippet/conf/pseudo/mapred-site.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,4 @@
<name>mapred.job.tracker</name>
<value>localhost:8021</value>
</property>

<!--property>
<name>mapred.jobtracker.taskScheduler</name>
<value>org.apache.hadoop.mapred.FairScheduler</value>
</property-->
<property>
<name>mapred.fairscheduler.allocation.file</name>
<value>/Users/tom/dev/hadoop-conf/fairshare.xml</value>
</property>
<property>
<name>mapred.fairscheduler.preemption</name>
<value>true</value>
</property>
<property>
<name>mapred.fairscheduler.update.interval</name>
<value>100</value>
</property>
<property>
<name>mapred.fairscheduler.preemption.interval</name>
<value>100</value>
</property>
<!--property>
<name>mapred.fairscheduler.poolnameproperty</name>
<value>pool.name</value>
</property>
<property>
<name>pool.name</name>
<value>\${mapreduce.job.user.name}</value>
</property-->
</configuration>
56 changes: 27 additions & 29 deletions snippet/src/test/java/ExamplesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,11 @@ public static Collection<Object[]> data() {
fail(dir + " does not exist");
}
for (File file : dir.listFiles()) {
data.add(new Object[] { file });
// so we can see which test corresponds to which file
System.out.printf("%s: %s\n", i++, file);
if (file.isDirectory()) {
data.add(new Object[] { file });
// so we can see which test corresponds to which file
System.out.printf("%s: %s\n", i++, file);
}
}
}
return data;
Expand Down Expand Up @@ -134,39 +136,17 @@ public void setUp() throws IOException {
@Test
public void test() throws Exception {
System.out.println("Running " + example);
File inputFile;
File expectedOutputDir;

// First look for data for the particular version and mode
File versionedExample = new File(example, version);
File modeExample = new File(example, mode);
File versionedModeExample = new File(versionedExample, mode);
if (versionedModeExample.exists()) {
inputFile = new File(versionedModeExample, "input.txt");
expectedOutputDir = new File(versionedModeExample, "output");
// if empty then skip
assumeTrue(inputFile.exists());
} else if (versionedExample.exists()) { // Then just for a particular version
inputFile = new File(versionedExample, "input.txt");
expectedOutputDir = new File(versionedExample, "output");
// if empty then skip
assumeTrue(inputFile.exists());
} else if (modeExample.exists()) { // Then just for a particular mode
inputFile = new File(modeExample, "input.txt");
expectedOutputDir = new File(modeExample, "output");
// if empty then skip
assumeTrue(inputFile.exists());
} else { // Otherwise use the standard fallback
inputFile = new File(example, "input.txt");
expectedOutputDir = new File(example, "output");
}

File exampleDir = findBaseExampleDirectory(example);
File inputFile = new File(exampleDir, "input.txt");
System.out.println("Running input " + inputFile);

String systemOut = execute(inputFile.getAbsolutePath());
System.out.println(systemOut);

execute(new File("src/test/resources/copyoutput.sh").getAbsolutePath());

File expectedOutputDir = new File(exampleDir, "output");
if (!expectedOutputDir.exists()) {
FileUtils.copyDirectory(actualOutputDir, expectedOutputDir);
fail(expectedOutputDir + " does not exist - creating.");
Expand Down Expand Up @@ -196,6 +176,24 @@ public void test() throws Exception {
}
System.out.println("Completed " + example);
}

private File findBaseExampleDirectory(File example) {
// Look in base/<version>/<mode> then base/<version> then base/<mode>
File[] candidates = {
new File(new File(example, version), mode),
new File(example, version),
new File(example, mode),
};
for (File candidate : candidates) {
if (candidate.exists()) {
File inputFile = new File(candidate, "input.txt");
// if no input file then skip test
assumeTrue(inputFile.exists());
return candidate;
}
}
return example;
}

private static String execute(String commandLine) throws ExecuteException, IOException {
ByteArrayOutputStream stdout = new ByteArrayOutputStream();
Expand Down

0 comments on commit 89cf0da

Please sign in to comment.