Create deploy job (kubeflow#673)

* Create deploy job * move job to examples and remove options * make job a statefulset * move app to pvc * add TODOs * address review
hiramf · Apr 28, 2018 · 108313a · 108313a
1 parent 85a3283
commit 108313a
Show file tree

Hide file tree

Showing 7 changed files with 122 additions and 5 deletions.
diff --git a/.gitignore b/.gitignore
@@ -18,7 +18,7 @@ vendor/
 flymd.*
 
 # vim .swp files
-.swp
+*.swp
 
 # Files created by Gogland IDE
 .idea/

diff --git a/bootstrap/Dockerfile b/bootstrap/Dockerfile
@@ -22,7 +22,7 @@ RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.ta
     --path-update=false \
     --usage-reporting=false
 
-RUN ln -sf /google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud    
+RUN ln -sf /google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud
 
 # Install kubectl
 # We don't install via gcloud because we want 1.10 which is newer than what's in gcloud.
@@ -46,6 +46,23 @@ RUN cd /opt/ksonnet && \
     cp /tmp/go/bin/ks /usr/local/bin && \
     chmod a+rx /usr/local/bin/ks
 
+# Pull kubeflow ksonnet repos
+
+# Ksonnet needs to have kubernetes env setup even for non-k8s actions.
+# TODO (inc0): remove this when we move to disk registry
+COPY kubeconfig_fake.yml /root/.kube/config
+
+ARG github_token
+
+# TODO (inc0): remove this when we move to disk registry
+RUN export GITHUB_TOKEN=${github_token} && cd / && ks init kubeflow && \
+    cd kubeflow && \
+    ks registry add kubeflow github.com/kubeflow/kubeflow/tree/master/kubeflow && \
+    ks pkg install kubeflow/core && \
+    ks pkg install kubeflow/tf-serving && \
+    ks pkg install kubeflow/tf-job && unset GITHUB_TOKEN
+COPY deploy.sh /kubeflow
+
 RUN mkdir -p /opt/kubeflow
 
 COPY --from=builder /opt/kubeflow/bootstrapper /opt/kubeflow/
@@ -64,4 +81,5 @@ ENV USER_ID 1000
 ENV GROUP_ID 1000
 ENV GROUP kubeflow
 
+WORKDIR /opt/kubeflow
 ENTRYPOINT ["/opt/kubeflow/start.sh"]
diff --git a/bootstrap/Makefile b/bootstrap/Makefile
@@ -21,7 +21,7 @@ all: build
 # To build without the cache set the environment variable
 # export DOCKER_BUILD_OPTS=--no-cache
 build:
-	docker build ${DOCKER_BUILD_OPTS} -t $(IMG):$(TAG) .
+	docker build ${DOCKER_BUILD_OPTS} -t $(IMG):$(TAG) --build-arg github_token=${GITHUB_TOKEN} .
 	docker tag $(IMG):$(TAG) $(IMG):latest
 	@echo Built $(IMG):latest
 	@echo Built $(IMG):$(TAG)

diff --git a/bootstrap/deploy.sh b/bootstrap/deploy.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Shortcut script to run ks apply from within cluster
+ks apply default --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
diff --git a/bootstrap/examples/kubeflow_toolbox.yaml b/bootstrap/examples/kubeflow_toolbox.yaml
@@ -0,0 +1,57 @@
+# Deploy kubeflow to namespace "kubeflow"
+
+# TODO(inc0): Create rbac role bindings
+# Headless service because StatefulSet requires it
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kubeflow-toolbox
+spec:
+  clusterIP: "None"
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: kubeflow-toolbox-pvc
+  labels:
+    app: kubeflow-toolbox
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kubeflow-toolbox
+spec:
+  selector:
+    matchLabels:
+      app: kubeflow-toolbox
+  serviceName: kubeflow-toolbox
+  template:
+    metadata:
+      name: kubeflow-toolbox
+      labels:
+        app: kubeflow-toolbox
+    spec:
+      containers:
+      - name: kubeflow-toolbox
+        image: gcr.io/kubeflow-images-public/bootstrapper
+        env:
+        - name: NAMESPACE
+          value: "kubeflow"
+        - name: DEPLOY_JOB
+          value: "TRUE"
+        volumeMounts:
+        - name: kubeflow-toolbox-pvc
+          mountPath: /opt/kubeflow/app
+      volumes:
+      - name: kubeflow-toolbox-pvc
+        persistentVolumeClaim:
+          claimName: kubeflow-toolbox-pvc
diff --git a/bootstrap/kubeconfig_fake.yml b/bootstrap/kubeconfig_fake.yml
@@ -0,0 +1,17 @@
+
+# Fake kubeconfig file to embed into bootstrap image - tricks ksonnet to allow
+# running prefetch ops
+
+apiVersion: v1
+clusters:
+- cluster:
+    server: http://localhost
+  name: fake
+contexts:
+- context:
+    cluster: fake
+    user: ""
+  name: fake
+current-context: fake
+kind: Config
+preferences: {}
diff --git a/bootstrap/start.sh b/bootstrap/start.sh
@@ -1,12 +1,33 @@
 #!/bin/bash
 #
-# Startup script. 
+# Startup script.
 # The purpose of this script is to create a unix user
 # with the same id as the user on the host system to
 # that we can read the user's home directory on the host
 # so that we can uss kubeconfig, gcloud config and other things.
 set -x
+
+# If it's run by deployment job, don't switch users
+if [ -v "DEPLOY_JOB" ]; then
+    cd /kubeflow
+    # Remove fake kubeconfig and start k8s proxy
+    rm ~/.kube/config
+    kubectl proxy --port=8111 &
+
+    # Recreate env since we have proper k8s creds
+    # TODO (inc0): Replace this with bootstrapper when we confirm that it works inside cluster
+    ks env rm default
+    ks env add default --server=http://127.0.0.1:8111
+    ks env set default --namespace ${NAMESPACE}
+
+    if [ ! -d /opt/kubeflow/app/kubeflow ]; then
+        ks generate kubeflow-core kubeflow-core
+        cp -R /kubeflow /opt/kubeflow/app
+    fi
+    cd /opt/kubeflow/app/kubeflow
+    tail -f /dev/null
+fi
 groupadd -g ${GROUP_ID} ${GROUP}
 useradd -r -u ${USER_ID} -g ${GROUP} --shell=/bin/bash ${USER}
 cd /home/${USER}
-su ${USER}
+su ${USER}
-Original file line number
+Diff line change
@@ Expand Up / @@ -18,7 +18,7 @@ vendor/ @@
     flymd.*
     # vim .swp files
-    .swp
+    *.swp
     # Files created by Gogland IDE
     .idea/
@@ Expand Down @@