Skip to content

Commit

Permalink
Change example code directory structure to be more maven-like: src/ma…
Browse files Browse the repository at this point in the history
…in/chxx to chxx/src/main.
  • Loading branch information
tomwhite committed Jul 23, 2010
1 parent f5d3605 commit cabdb31
Show file tree
Hide file tree
Showing 279 changed files with 64 additions and 54 deletions.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
34 changes: 22 additions & 12 deletions build.xml
Original file line number Diff line number Diff line change
Expand Up @@ -92,18 +92,28 @@

<mkdir dir="${classes.dir}"/>

<javac destdir="${classes.dir}" debug="true" deprecation="false" optimize="false" failonerror="true" includes="${src.includes}" excludes="ch13/,ch15/">
<src path="${src.dir}"/>
<javac destdir="${classes.dir}" debug="true" deprecation="false" optimize="false" failonerror="true" includes="${src.includes}">
<src path="common/${src.dir}"/>
<src path="ch02/${src.dir}"/>
<src path="ch03/${src.dir}"/>
<src path="ch04/${src.dir}"/>
<src path="ch05/${src.dir}"/>
<src path="ch06/${src.dir}"/>
<src path="ch08/${src.dir}"/>
<src path="ch11/${src.dir}"/>
<src path="ch12/${src.dir}"/>
<src path="ch14/${src.dir}"/>
<classpath refid="classpath"/>
</javac>

<javac destdir="${classes.dir}" debug="true" deprecation="false" optimize="false" failonerror="true" includes="${src.test.includes}">
<src path="${src.test.dir}"/>
<src path="common/${src.test.dir}"/>
<src path="book/${src.test.dir}"/>
<classpath refid="classpath"/>
</javac>

<copy todir="${classes.dir}">
<fileset dir="${src.dir}"/>
<fileset dir="."/>
<mapper type="regexp" from=".*?/java/(.*)\.(properties|txt|xml)$$" to="\1.\2"/>
</copy>

Expand All @@ -114,7 +124,7 @@
<mkdir dir="${classes.dir}"/>

<javac destdir="${classes.dir}" debug="true" deprecation="false" optimize="false" failonerror="true" includes="${src.includes}">
<src path="${src.dir}/ch13"/>
<src path="ch13/${src.dir}"/>
<classpath refid="hbase.classpath"/>
</javac>

Expand Down Expand Up @@ -149,13 +159,13 @@
output="xml:withMessages"
outputFile="${findbugs.report.xmlfile}">
<auxClasspath refid="classpath"/>
<sourcePath path="${src.dir}/common/java" />
<sourcePath path="${src.dir}/ch02/java" />
<sourcePath path="${src.dir}/ch03/java" />
<sourcePath path="${src.dir}/ch04/java" />
<sourcePath path="${src.dir}/ch05/java" />
<sourcePath path="${src.dir}/ch06/java" />
<sourcePath path="${src.dir}/ch09/java" />
<sourcePath path="common/${src.dir}/common/java" />
<sourcePath path="ch02/${src.dir}/java" />
<sourcePath path="ch03/${src.dir}/java" />
<sourcePath path="ch04/${src.dir}/java" />
<sourcePath path="ch05/${src.dir}/java" />
<sourcePath path="ch06/${src.dir}/java" />
<sourcePath path="ch09/${src.dir}/java" />
<class location="${build.dir}/job.jar" />
</findbugs>

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
hadoop jar $HADOOP_INSTALL/contrib/streaming/hadoop-*-streaming.jar \
-input input/ncdc/sample.txt \
-output output \
-mapper src/main/ch02/ruby/max_temperature_map.rb \
-reducer src/main/ch02/ruby/max_temperature_reduce.rb
-mapper ch02/src/main/ruby/max_temperature_map.rb \
-reducer ch02/src/main/ruby/max_temperature_reduce.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
hadoop jar $HADOOP_INSTALL/contrib/streaming/hadoop-*-streaming.jar \
-input input/ncdc/all \
-output output \
-mapper "ch02/src/main/ruby/max_temperature_map.rb | sort | ch02/src/main/ruby/max_temperature_reduce.rb" \
-reducer ch02/src/main/ruby/max_temperature_reduce.rb \
-file ch02/src/main/ruby/max_temperature_map.rb \
-file ch02/src/main/ruby/max_temperature_reduce.rb
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
10 changes: 5 additions & 5 deletions src/main/ch02/sh/max_temp.sh → ch02/src/main/sh/max_temp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ hadoop MaxTemperature input/ncdc/sample.txt output
cat output/part-00000
: ^^ max_temp_java_output
: vv max_temp_ruby_map
cat input/ncdc/sample.txt | src/main/ch02/ruby/max_temperature_map.rb
cat input/ncdc/sample.txt | ch02/src/main/ruby/max_temperature_map.rb
: ^^ max_temp_ruby_map
: vv max_temp_ruby_pipeline
cat input/ncdc/sample.txt | src/main/ch02/ruby/max_temperature_map.rb | \
sort | src/main/ch02/ruby/max_temperature_reduce.rb
cat input/ncdc/sample.txt | ch02/src/main/ruby/max_temperature_map.rb | \
sort | ch02/src/main/ruby/max_temperature_reduce.rb
: ^^ max_temp_ruby_pipeline
: vv max_temp_python_pipeline
cat input/ncdc/sample.txt | src/main/ch02/python/max_temperature_map.py | \
sort | src/main/ch02/python/max_temperature_reduce.py
cat input/ncdc/sample.txt | ch02/src/main/python/max_temperature_map.py | \
sort | ch02/src/main/python/max_temperature_reduce.py
: ^^ max_temp_python_pipeline

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
20 changes: 20 additions & 0 deletions ch08/src/main/python/mean_max_daily_temp.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
STREAM="hadoop jar $HADOOP_INSTALL/contrib/streaming/hadoop-*-streaming.jar -conf conf/hadoop-localhost.xml"

$STREAM \
-D stream.num.map.output.key.fields=2 \
-input input/ncdc/all \
-output out_max_daily \
-mapper ch08/src/main/python/max_daily_temp_map.py \
-reducer ch08/src/main/python/max_daily_temp_reduce.py \
-file ch08/src/main/python/max_daily_temp_map.py \
-file ch08/src/main/python/max_daily_temp_reduce.py

$STREAM \
-D stream.num.map.output.key.fields=2 \
-input out_max_daily \
-output out_mean_max_daily \
-mapper ch08/src/main/python/mean_max_daily_temp_map.py \
-reducer ch08/src/main/python/mean_max_daily_temp_reduce.py \
-file ch08/src/main/python/mean_max_daily_temp_map.py \
-file ch08/src/main/python/mean_max_daily_temp_reduce.py

Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ org.apache.hadoop.mapred.lib.KeyFieldBasedComparator \
-D mapred.text.key.comparator.options="-k1n -k2nr" \
-input input/ncdc/all \
-output output_secondarysort_streaming \
-mapper src/main/ch08/python/secondary_sort_map.py \
-mapper ch08/src/main/python/secondary_sort_map.py \
-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
-reducer src/main/ch08/python/secondary_sort_reduce.py \
-file src/main/ch08/python/secondary_sort_map.py \
-file src/main/ch08/python/secondary_sort_reduce.py
-reducer ch08/src/main/python/secondary_sort_reduce.py \
-file ch08/src/main/python/secondary_sort_map.py \
-file ch08/src/main/python/secondary_sort_reduce.py

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
-- max_temp_filter_stream.pig
DEFINE is_good_quality `is_good_quality.py`
SHIP ('src/main/ch11/python/is_good_quality.py');
SHIP ('ch11/src/main/python/is_good_quality.py');
records = LOAD 'input/ncdc/micro-tab/sample.txt'
AS (year:chararray, temperature:int, quality:int);
filtered_records = STREAM records THROUGH is_good_quality
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ADD FILE /Users/tom/workspace/hadoop-book/src/main/ch12/python/is_good_quality.py;
ADD FILE /Users/tom/workspace/hadoop-book/src/main/ch12/python/max_temperature_reduce.py;
ADD FILE /Users/tom/workspace/hadoop-book/ch12/src/main/python/is_good_quality.py;
ADD FILE /Users/tom/workspace/hadoop-book/ch12/src/main/python/max_temperature_reduce.py;

FROM records2
SELECT TRANSFORM(year, temperature, quality)
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion snippet/bin/grunter.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ cd "$bin"/../..
rm -rf $actual/ch11/grunt
mkdir -p $actual/ch11/grunt

for f in src/main/ch11/grunt/*.grunt; do
for f in ch11/src/main/grunt/*.grunt; do
out=$f.output.txt
pig -x local < $f 2> /dev/null \
| grep -v INFO \
Expand Down

This file was deleted.

20 changes: 0 additions & 20 deletions src/main/ch08/python/mean_max_daily_temp.sh

This file was deleted.

0 comments on commit cabdb31

Please sign in to comment.