-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathyarn.txt
57 lines (37 loc) · 2.14 KB
/
yarn.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
yarn jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar
source code:
https://github.com/apache/hadoop-common/tree/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples
Pi example:
yarn jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 16 1000
# see: http://arxiv.org/abs/1008.3171
# Sample Yarn (Non Map-Reduce application)
wget http://mirror.catn.com/pub/apache/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz
tar xvf apache-maven-3.2.3-bin.tar.gz
sudo mv apache-maven-3.2.3 /usr/local/apache-maven
export M2_HOME=/usr/local/apache-maven
export M2=$M2_HOME/bin
export PATH=$M2:$PATH
hadoop jar /usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell.jar org.apache.hadoop.yarn.applications.distributedshell.Client --jar /usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell.jar --shell_command "touch /tmp/test123" --num_containers 1
# see source code
https://github.com/apache/hadoop-common/tree/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell
# to see logs it might be necessary to add sandbox to the /etc/hosts
sudo vi /etc/hosts
127.0.0.1 sandbox.hortonworks.com
# Note, consider Weave: https://github.com/continuuity/weave
Join example:
https://github.com/apache/hadoop-common/blob/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
# run it:
javac -cp `hadoop classpath`:. -d java/target java/src/org/apache/*
jar -cvf ./examples.jar -C java/target .
hdfs dfs -mkdir /user/root/join
hdfs dfs -copyFromLocal join/* /user/root/join/
# testing
hdfs dfs -rm -f -r /user/root/joinout
yarn jar ./examples.jar org.apache.Join \
-outKey org.apache.hadoop.io.Text \
-joinOp outer \
join/a.txt join/b.txt joinout
hdfs dfs -text /user/root/joinout/part-r-00000
# see
https://github.com/hanborq/hadoop/blob/master/src/mapred/org/apache/hadoop/mapred/join/CompositeInputFormat.java
https://github.com/facebook/hadoop-20/blob/master/src/mapred/org/apache/hadoop/mapred/join/InnerJoinRecordReader.java