Fix dependency conflict between tez and vineyard. (v6d-io#1701)

Fixes v6d-io#1682 Signed-off-by: vegetableysm <yuanshumin.ysm@alibaba-inc.com>
SighingSnow · Dec 29, 2023 · f21f437 · f21f437
1 parent 7575eee
commit f21f437
Show file tree

Hide file tree

Showing 18 changed files with 4,517 additions and 32 deletions.
diff --git a/java/hive/docker/README.rst b/java/hive/docker/README.rst
@@ -12,7 +12,7 @@ Build Hive Docker Image with Hadoop
 
 ### Build docker images
 ```bash
-    cd v6d/java/hive/distributed
+    cd v6d/java/hive/docker
     ./build.sh
 ```
 
@@ -23,15 +23,15 @@ Build Hive Docker Image with Hadoop
 
 ### Start sql server for hive metastore
 ```bash
-    cd v6d/java/hive/distributed/docker/mysql
+    cd v6d/java/hive/docker/dependency/mysql
     docker-compose -f mysql-compose.yaml up -d
     # You can change the password in mysql-compose.yaml and hive-site.xml
 ```
 
 ### Run hadoop & hive docker images
 ```bash
-    cd v6d/java/hive/distributed/docker
-    docker-compose -f docker-compose.yaml up -d
+    cd v6d/java/hive/docker
+    docker-compose -f docker-compose-distributed.yaml up -d
 ```
 
 ### Prepare tez jars
@@ -53,14 +53,6 @@ Build Hive Docker Image with Hadoop
     docker exec -it hive-hiveserver2 beeline -u "jdbc:hive2://hive-hiveserver2:10000" -n root
 ```
 
-```sql
-    -- in beeline
-    drop table test_hive1;
-    create table test_hive1(field int);
-    insert into table test_hive1 values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
-    select * from test_hive1;
-```
-
 Using vineyard as storage
 -----------------
 
@@ -100,9 +92,7 @@ Using vineyard as storage
 ```sql
     -- in beeline
     drop table test_vineyard;
-    create table test_vineyard(field int)
-    stored as Vineyard
-    location "vineyard:///user/hive_remote/warehouse/test_vineyard";
+    create table test_vineyard(field int);
     insert into table test_vineyard values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
     select * from test_vineyard;
 ```
diff --git a/java/hive/docker/build.sh b/java/hive/docker/build.sh
@@ -42,7 +42,7 @@ else
     fi
 fi
 
-cp -R ./dependency/* "$WORK_DIR/"
+cp -R ./dependency/images/ "$WORK_DIR/"
 cp ../target/vineyard-hive-0.1-SNAPSHOT.jar "$WORK_DIR/images/"
 
 tar -xzf "$WORK_DIR/apache-hive-$HIVE_VERSION-bin.tar.gz" -C "$WORK_DIR/"

diff --git a/java/hive/docker/dependency/images/Dockerfile b/java/hive/docker/dependency/images/Dockerfile
@@ -1,5 +1,10 @@
-FROM registry.cn-hangzhou.aliyuncs.com/bigdata_cloudnative/hadoop:v1
+FROM apache/hadoop:3.3.5
 
+RUN mkdir -p /opt/apache/; \
+    mv /opt/hadoop/ /opt/apache/
+ENV HADOOP_HOME=/opt/apache/hadoop \
+    HADOOP_CONF_DIR=/opt/apache/hadoop/etc/hadoop \
+    HIVE_HOME=/opt/apache/hive
 COPY ./vineyard-hive-0.1-SNAPSHOT.jar ${HADOOP_HOME}/share/hadoop/common/
 
 # prepare hdoop config
@@ -19,15 +24,15 @@ COPY hive /opt/apache/hive/
 COPY hive-config/ /hive-config
 COPY hive-config-distributed/ /hive-config-distributed
 
+ENV PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:${HIVE_HOME}/bin:${PATH}
+
 COPY bootstrap.sh /opt/apache/
 COPY mysql-connector-java-5.1.49/mysql-connector-java-5.1.49-bin.jar ${HIVE_HOME}/lib/
 
-RUN sudo yum -y install unzip
-RUN wget http://www.vuln.cn/wp-content/uploads/2019/08/libstdc.so_.6.0.26.zip -q && unzip libstdc.so_.6.0.26.zip
-RUN cp libstdc++.so.6.0.26 /usr/lib64
-RUN rm /usr/lib64/libstdc++.so.6
-RUN ln -s /usr/lib64/libstdc++.so.6.0.26 /usr/lib64/libstdc++.so.6
-
-RUN sudo yum -y install vim
-
-#RUN yum -y install which
+RUN sudo yum -y install unzip; \
+    wget http://www.vuln.cn/wp-content/uploads/2019/08/libstdc.so_.6.0.26.zip -q && unzip libstdc.so_.6.0.26.zip; \
+    sudo cp libstdc++.so.6.0.26 /usr/lib64; \
+    sudo rm /usr/lib64/libstdc++.so.6; \
+    sudo ln -s /usr/lib64/libstdc++.so.6.0.26 /usr/lib64/libstdc++.so.6; \
+    sudo yum -y install vim; \
+    rm libstdc.so_.6.0.26.zip libstdc++.so.6.0.26;
diff --git a/java/hive/docker/dependency/images/bootstrap.sh b/java/hive/docker/dependency/images/bootstrap.sh
@@ -12,41 +12,41 @@ start_hdfs_namenode() {
 	fi
 	${HADOOP_HOME}/bin/hdfs --loglevel INFO namenode
 
-	tail -f ${HADOOP_HOME}/logs/*namenode*.log
+	tail -f /var/log/hadoop/*namenode*.log
 }
 
 start_hdfs_datanode() {
     wait_for $1 $2
 
 	${HADOOP_HOME}/bin/hdfs --loglevel INFO --daemon start datanode
-    tail -f ${HADOOP_HOME}/logs/*datanode*.log	
+    tail -f /var/log/hadoop/*datanode*.log	
 }
 
 start_yarn_resourcemanager() {
     ${HADOOP_HOME}/bin/yarn --loglevel INFO --daemon start resourcemanager
-    tail -f ${HADOOP_HOME}/logs/*resourcemanager*.log
+    tail -f /var/log/hadoop/*resourcemanager*.log
 }
 
 start_yarn_nodemanager() {
 	wait_for $1 $2
 
 	${HADOOP_HOME}/bin/yarn --loglevel INFO --daemon start nodemanager
-	tail -f ${HADOOP_HOME}/logs/*nodemanager*.log
+	tail -f /var/log/hadoop/*nodemanager*.log
 }
 
 start_yarn_proxyserver() {
 	wait_for $1 $2
 
 	${HADOOP_HOME}/bin/yarn --loglevel INFO --daemon start proxyserver
-	tail -f ${HADOOP_HOME}/logs/*proxyserver*.log
+	tail -f /var/log/hadoop/*proxyserver*.log
 }
 
 start_mr_historyserver() {
 
     wait_for $1 $2
 
 	${HADOOP_HOME}/bin/mapred --loglevel INFO  --daemon  start historyserver
-	tail -f ${HADOOP_HOME}/logs/*historyserver*.log
+	tail -f /var/log/hadoop/*historyserver*.log
 }
 
 start_hive_metastore() {