miscellaneous updates and enhacements for shell scripts (#1923)

* miscellaneous updates and enhacements for shell scripts * More style updates * More enhance shell scripts * Small style enhance for function definition * Remove update_katib_ksonnet.sh * Update styles for update_components.sh * Minor shell style enhance
kubeflow · Nov 16, 2018 · 76107ff · 76107ff
1 parent 2ea26eb
commit 76107ff
Show file tree

Hide file tree

Showing 44 changed files with 573 additions and 612 deletions.
diff --git a/bootstrap/build_image.sh b/bootstrap/build_image.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # A simple script to build the Docker images.
 # This is intended to be invoked as a step in Argo to build the docker image.
@@ -15,8 +15,8 @@ BUILDER_IMG=gcr.io/kubeflow-images-public/bootstrapper-builder
 BUILDER_IMG_VERSION=$(head -1 ${CONTEXT_DIR}/glide.lock | cut -d ' ' -f 2)
 
 # Wait for the Docker daemon to be available.
-until docker ps
-do sleep 3
+until docker ps; do
+  sleep 3
 done
 
 # pull builder image from GCR or build it from local if required one doesn't exist.

diff --git a/bootstrap/debug.sh b/bootstrap/debug.sh
@@ -6,25 +6,22 @@
 # See the [developer_guide.md](./developer_guide.md) for additional details.
 #
 
-cleanup()
-{
+cleanup() {
   if [[ -n $portforwardcommand ]]; then
     echo killing $portforwardcommand
     pkill -f $portforwardcommand
   fi
 }
 trap cleanup EXIT
 
-portforward()
-{
+portforward() {
   local pod=$1 namespace=$2 from_port=$3 to_port=$4 cmd
   cmd='kubectl port-forward $pod ${from_port}:${to_port} --namespace=$namespace 2>&1>/dev/null &'
   portforwardcommand="${cmd% 2>&1>/dev/null &}"
   eval $cmd
 }
 
-waitforpod()
-{
+waitforpod() {
   local cmd="kubectl get pods --no-headers -oname --selector=app=kubeflow-bootstrapper --field-selector=status.phase=Running --namespace=kubeflow-admin | sed 's/^pod.*\///'" found=$(eval "$cmd")
   while [[ -z $found ]]; do
     sleep 1
@@ -33,8 +30,7 @@ waitforpod()
   echo $found
 }
 
-waitforever()
-{
+waitforever() {
   which gsleep >/dev/null
   if [[ $? == 1 ]]; then
     while true; do

diff --git a/bootstrap/start.sh b/bootstrap/start.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Startup script.
 # The purpose of this script is to create a unix user
@@ -14,23 +14,23 @@ set -x
 
 # If it's run by deployment job, don't switch users
 if [ -v "DEPLOY_JOB" ]; then
-    cd /kubeflow
-    # Remove fake kubeconfig and start k8s proxy
-    rm ~/.kube/config
-    kubectl proxy --port=8111 &
+  cd /kubeflow
+  # Remove fake kubeconfig and start k8s proxy
+  rm ~/.kube/config
+  kubectl proxy --port=8111 &
 
-    # Recreate env since we have proper k8s creds
-    # TODO (inc0): Replace this with bootstrapper when we confirm that it works inside cluster
-    ks env rm default
-    ks env add default --server=http://127.0.0.1:8111
-    ks env set default --namespace ${NAMESPACE}
+  # Recreate env since we have proper k8s creds
+  # TODO (inc0): Replace this with bootstrapper when we confirm that it works inside cluster
+  ks env rm default
+  ks env add default --server=http://127.0.0.1:8111
+  ks env set default --namespace ${NAMESPACE}
 
-    if [ ! -d /opt/kubeflow/app/kubeflow ]; then
-        ks generate kubeflow-core kubeflow-core
-        cp -R /kubeflow /opt/kubeflow/app
-    fi
-    cd /opt/kubeflow/app/kubeflow
-    tail -f /dev/null
+  if [ ! -d /opt/kubeflow/app/kubeflow ]; then
+    ks generate kubeflow-core kubeflow-core
+    cp -R /kubeflow /opt/kubeflow/app
+  fi
+  cd /opt/kubeflow/app/kubeflow
+  tail -f /dev/null
 fi
 groupadd -g ${GROUP_ID} ${GROUP}
 useradd -r -u ${USER_ID} -g ${GROUP} --shell=/bin/bash ${USER}

diff --git a/bootstrap/test_delete.sh b/bootstrap/test_delete.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Script to delete kubeflow-ci namespace; deployment and source repo in sandbox project.
 # This is intended to be invoked as a step in Argo.
@@ -17,10 +17,10 @@ gcloud container clusters get-credentials ${CLUSTER} --zone ${ZONE} --project ku
 # delete test namespace
 kubectl delete namespace ${NAMESPACE}
 
-for i in 1 2 3 4 5
-do gcloud -q deployment-manager deployments delete ${DEPLOYMENT} --project=kubeflow-ci-deploy && break || sleep 30
+for i in 1 2 3 4 5; do
+  gcloud -q deployment-manager deployments delete ${DEPLOYMENT} --project=kubeflow-ci-deploy && break || sleep 30
 done
 
-for i in 1 2 3 4 5
-do gcloud -q source repos delete kubeflow-ci-deploy-kubeflow-config --project=kubeflow-ci-deploy && break || sleep 30
+for i in 1 2 3 4 5; do
+  gcloud -q source repos delete kubeflow-ci-deploy-kubeflow-config --project=kubeflow-ci-deploy && break || sleep 30
 done
diff --git a/bootstrap/test_setup.sh b/bootstrap/test_setup.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # A simple script to setup deployment service.
 # This is intended to be invoked as a step in Argo.

diff --git a/build/check_boilerplate.sh b/build/check_boilerplate.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2015 The Kubeflow Authors All rights reserved.
 #

diff --git a/components/centraldashboard/build_image.sh b/components/centraldashboard/build_image.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # A simple script to build the Docker images.
 # This is intended to be invoked as a step in Argo to build the docker image.
@@ -13,8 +13,8 @@ CONTEXT_DIR=$(dirname "$DOCKERFILE")
 PROJECT="${GCP_PROJECT}"
 
 # Wait for the Docker daemon to be available.
-until docker ps
-do sleep 3
+until docker ps; do
+  sleep 3
 done
 
 gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
@@ -25,4 +25,4 @@ echo "GCP Project: "$PROJECT
 
 echo "Building centraldashboard using gcloud build"
 gcloud builds submit --tag=${IMAGE}:${TAG} --project=${PROJECT} .
-echo "Finished building image"
+echo "Finished building image"
diff --git a/components/gcp-click-to-deploy/ks-app/create_gcp_deploy_sa.sh b/components/gcp-click-to-deploy/ks-app/create_gcp_deploy_sa.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Creates a service account for running the GCP deploy app.
 # We need a service account to work with external DNS.
@@ -39,15 +39,14 @@ gcloud --project=${PROJECT} --quiet iam service-accounts create ${NAME}
 declare -a roles=("dns.admin")
 
 # Add policy bindings
-for ROLE in "${roles[@]}"
-do   
-   gcloud projects add-iam-policy-binding ${DNS_PROJECT} \
-     --member serviceAccount:${EMAIL} \
-     --role roles/${ROLE}
+for ROLE in "${roles[@]}"; do
+  gcloud projects add-iam-policy-binding ${DNS_PROJECT} \
+    --member serviceAccount:${EMAIL} \
+    --role roles/${ROLE}
 done
 
 # Get a new service account key
-SECRET_FILE=~/tmp/${EMAIL}.json 
+SECRET_FILE=~/tmp/${EMAIL}.json
 rm -f ~${SECRET_FILE}
 gcloud --project=${PROJECT} iam service-accounts keys create ${SECRET_FILE} --iam-account ${EMAIL}
 

diff --git a/components/k8s-model-server/http-proxy/build_image.sh b/components/k8s-model-server/http-proxy/build_image.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # A simple script to build the Docker images.
 # This is intended to be invoked as a step in Argo to build the docker image.
@@ -11,12 +11,12 @@ CONTEXT_DIR=$(dirname "$DOCKERFILE")
 IMAGE=$2
 
 # Wait for the Docker daemon to be available.
-until docker ps
-do sleep 3
+until docker ps; do
+  sleep 3
 done
 
 docker build --pull -t ${IMAGE} \
-	-f ${DOCKERFILE} ${CONTEXT_DIR}
+  -f ${DOCKERFILE} ${CONTEXT_DIR}
 
 gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
-gcloud docker -- push ${IMAGE}
+gcloud docker -- push ${IMAGE}
diff --git a/components/k8s-model-server/images/build_image.sh b/components/k8s-model-server/images/build_image.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # A simple script to build the Docker images.
 # This is intended to be invoked as a step in Argo to build the docker image.
@@ -11,12 +11,12 @@ CONTEXT_DIR=$(dirname "$DOCKERFILE")
 IMAGE=$2
 
 # Wait for the Docker daemon to be available.
-until docker ps
-do sleep 3
+until docker ps; do
+  sleep 3
 done
 
 docker build --pull -t ${IMAGE} \
-	-f ${DOCKERFILE} ${CONTEXT_DIR}
+  -f ${DOCKERFILE} ${CONTEXT_DIR}
 
 gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
-gcloud docker -- push ${IMAGE}
+gcloud docker -- push ${IMAGE}
diff --git a/components/k8s-model-server/inception-client/run.sh b/components/k8s-model-server/inception-client/run.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2018 The Kubeflow Authors All rights reserved.
 #
@@ -17,11 +17,11 @@
 SERVER=$1
 PORT=$2
 
-if [ -z $SERVER ] ; then
+if [ -z $SERVER ]; then
   SERVER=$INCEPTION_SERVICE_HOST
 fi
 
-if [ -z $PORT ] ; then
+if [ -z $PORT ]; then
   PORT=$INCEPTION_SERVICE_PORT
 fi
 

diff --git a/components/tensorflow-notebook-image/build_image.sh b/components/tensorflow-notebook-image/build_image.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # A simple script to build the Docker images.
 # This is intended to be invoked as a step in Argo to build the docker image.
@@ -16,41 +16,40 @@ TAG=$3
 #
 # TODO(jlewi): We should take in the json config file and then parse that.
 CONFIG_FILE=$4
-BASE_IMAGE=$(jq -r .BASE_IMAGE  ${CONFIG_FILE})
-TF_PACKAGE=$(jq -r .TF_PACKAGE  ${CONFIG_FILE})
+BASE_IMAGE=$(jq -r .BASE_IMAGE ${CONFIG_FILE})
+TF_PACKAGE=$(jq -r .TF_PACKAGE ${CONFIG_FILE})
 TF_PACKAGE_PY_27=$(jq -r .TF_PACKAGE_PY_27 ${CONFIG_FILE})
 TF_SERVING_VERSION=$(jq -r .TF_SERVING_VERSION ${CONFIG_FILE})
 TFMA_VERSION=$(jq -r .TFMA_VERSION ${CONFIG_FILE})
 TFDV_VERSION=$(jq -r .TFDV_VERSION ${CONFIG_FILE})
 
-
 # JQ returns null for non defined values.
 if [ ${BASE_IMAGE} == "null" ]; then
-   BASE_IMAGE=""
+  BASE_IMAGE=""
 fi
 
 if [ ${TFMA_VERSION} == "null" ]; then
-   TFMA_VERSION=""
+  TFMA_VERSION=""
 fi
 
 if [ ${TFDV_VERSION} == "null" ]; then
-   TFDV_VERSION=""
+  TFDV_VERSION=""
 fi
 
 # Wait for the Docker daemon to be available.
-until docker ps
-do sleep 3
+until docker ps; do
+  sleep 3
 done
 
 docker build --pull \
-        --build-arg "BASE_IMAGE=${BASE_IMAGE}" \
-        --build-arg "TF_PACKAGE=${TF_PACKAGE}" \
-        --build-arg "TF_PACKAGE_PY_27=${TF_PACKAGE_PY_27}" \
-        --build-arg "TF_SERVING_VERSION=${TF_SERVING_VERSION}" \
-        --build-arg "TFMA_VERSION=${TFMA_VERSION}" \
-        --build-arg "TFDV_VERSION=${TFDV_VERSION}" \
-        -t "${IMAGE}:${TAG}" \
-	-f ${DOCKERFILE} ${CONTEXT_DIR}
+  --build-arg "BASE_IMAGE=${BASE_IMAGE}" \
+  --build-arg "TF_PACKAGE=${TF_PACKAGE}" \
+  --build-arg "TF_PACKAGE_PY_27=${TF_PACKAGE_PY_27}" \
+  --build-arg "TF_SERVING_VERSION=${TF_SERVING_VERSION}" \
+  --build-arg "TFMA_VERSION=${TFMA_VERSION}" \
+  --build-arg "TFDV_VERSION=${TFDV_VERSION}" \
+  -t "${IMAGE}:${TAG}" \
+  -f ${DOCKERFILE} ${CONTEXT_DIR}
 
 gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
 gcloud docker -- push "${IMAGE}:${TAG}"

diff --git a/components/tensorflow-notebook-image/install.sh b/components/tensorflow-notebook-image/install.sh
@@ -1,28 +1,28 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # An install script to be called from the DOCKERFILE
 set -ex
 conda create -n py2 python=2
 
 source activate py2
-pip install --upgrade pip 
+pip install --upgrade pip
 
-# TFX packages only supports python 2 
+# TFX packages only supports python 2
 pip --no-cache-dir install \
-    ipykernel \
-    ${TF_PACKAGE_PY_27} \
-    tensorflow-transform \
-    tensorflow-serving-api===${TF_SERVING_VERSION}    
+  ipykernel \
+  ${TF_PACKAGE_PY_27} \
+  tensorflow-transform \
+  tensorflow-serving-api===${TF_SERVING_VERSION}
 
 python -m ipykernel install
 
 # TODO(jlewi): Does TFDV not have jupyter extensions?
 if [[ $TFDV_VERSION ]]; then
-    pip install --no-cache-dir tensorflow-data-validation==$TFDV_VERSION
+  pip install --no-cache-dir tensorflow-data-validation==$TFDV_VERSION
 fi
 
 # tensorflow-model-analysis is only supported for TF 1.6 and above
-# TODO temporarily remove tensorflow-model-analysis because of gpu problem 
+# TODO temporarily remove tensorflow-model-analysis because of gpu problem
 # https://github.com/kubeflow/kubeflow/pull/1759
 if [[ $TFMA_VERSION ]]; then
   # if you find any dependency problems in the future
@@ -31,7 +31,7 @@ if [[ $TFMA_VERSION ]]; then
   pip install --no-cache-dir tensorflow-model-analysis==$TFMA_VERSION
   # We use --system because if we don't the config is written to the home directory
   # and the changes are lost when we mount a PV over them.
-  jupyter nbextension enable --py --system widgetsnbextension  
+  jupyter nbextension enable --py --system widgetsnbextension
 fi
 
 # TODO a quick fix for tensorflow_serving_api when install gpu
@@ -46,9 +46,9 @@ jupyter labextension install @jupyter-widgets/jupyterlab-manager
 
 # Install common packages from requirements.txt for both python2 and python
 pip --no-cache-dir install -r /tmp/requirements.txt
-source activate py2 
-pip --no-cache-dir install -r /tmp/requirements.txt 
-    
+source activate py2
+pip --no-cache-dir install -r /tmp/requirements.txt
+
 # Do chown in this layer for significant size savings
-chown -R ${NB_USER}:users $HOME 
-chown -R ${NB_USER}:users $CONDA_DIR
+chown -R ${NB_USER}:users $HOME
+chown -R ${NB_USER}:users $CONDA_DIR