Turn release.py into a binary to build the artifacts for all the diff…

…erent contexts (#133) * Turn release.py into a binary to build the artifacts for all the different modes. * We have multiple contexts in which we want to build the images and other artifacts * local * Tests * releases * This CL changes release.py so handle all three cases. * This increases code reuse and provides greater consistency. * Prior to this CL building the operator image didn't build other artifacts; notably the helm package. * This change is intended to facilitate transitioning to a workflow system for running more complex test and release workflows. * Currently the test infrastructure is using runner.py to build the images. The tests don't actually build the helm package. * We want to replace runner.py with a binary (release.py) that can be invoked to build the artifacts. This way our test/release workflow can just invoke that binary.
kubeflow · Nov 14, 2017 · f27b8de · f27b8de
1 parent ad34ef2
commit f27b8de
Show file tree

Hide file tree

Showing 7 changed files with 357 additions and 79 deletions.
diff --git a/.pylintrc b/.pylintrc
@@ -56,7 +56,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,missing-docstring,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,relative-import,invalid-name,bad-continuation,no-member,locally-disabled,fixme,import-error
+disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,missing-docstring,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,relative-import,invalid-name,bad-continuation,no-member,locally-disabled,fixme,import-error,too-many-locals
 
 
 [REPORTS]

diff --git a/developer_guide.md b/developer_guide.md
@@ -4,8 +4,8 @@
 Create a symbolic link inside your GOPATH to the location you checked out the code
 
 ```sh
-mkdir -p ${GOPATH}/src/github.com/jlewi
-ln -sf ${GIT_TRAINING} ${GOPATH}/src/mlkube.io
+mkdir -p ${GOPATH}/src/github.com/tensorflow
+ln -sf ${GIT_TRAINING} ${GOPATH}/src/github.com/tensorflow/k8s
 ```
 
   * GIT_TRAINING should be the location where you checked out https://github.com/tensorflow/k8s
@@ -27,6 +27,22 @@ Build it
 go install github.com/tensorflow/k8s/cmd/tf_operator
 ```
 
+## Building all the artifacts.
+
+To build the following artifacts:
+
+  * Docker image for the operator
+  * Helm chart for deploying it
+
+You can run
+
+```sh
+python -m release.py --registry=${REGISTRY}
+```
+  * The docker image will be tagged into your registry
+  * The helm chart will be created in **./bin**
+
+
 ## Running the Operator Locally
 
 Running the operator locally (as opposed to deploying it on a K8s cluster) is convenient for debugging/development.

diff --git a/images/tf_operator/build_and_push.py b/images/tf_operator/build_and_push.py
@@ -13,6 +13,8 @@
 
 import yaml
 
+# TODO(jlewi): build_and_push.py should be obsolete. We should be able to
+# use py/release.py
 
 def GetGitHash(root_dir):
   # The image tag is based on the githash.

diff --git a/py/build_and_push_image.py b/py/build_and_push_image.py
@@ -11,19 +11,24 @@
 
 import jinja2
 
-def GetGitHash():
+def GetGitHash(root_dir=None):
   # The image tag is based on the githash.
-  git_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])
+  git_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"],
+                                     cwd=root_dir).decode("utf-8")
   git_hash = git_hash.strip()
-  modified_files = subprocess.check_output(["git", "ls-files", "--modified"])
-  untracked_files = subprocess.check_output(["git", "ls-files", "--others",
-                                             "--exclude-standard"])
+
+  modified_files = subprocess.check_output(["git", "ls-files", "--modified"],
+                                           cwd=root_dir)
+  untracked_files = subprocess.check_output(
+      ["git", "ls-files", "--others", "--exclude-standard"], cwd=root_dir)
   if modified_files or untracked_files:
-    diff = subprocess.check_output(["git", "diff"])
+    diff = subprocess.check_output(["git", "diff"], cwd=root_dir)
+
     sha = hashlib.sha256()
     sha.update(diff)
     diffhash = sha.hexdigest()[0:7]
     git_hash = "{0}-dirty-{1}".format(git_hash, diffhash)
+
   return git_hash
 
 def run_and_stream(cmd):