Merge pull request kubernetes#30424 from fejta/e2e2

Automatic merge from submit-queue Create a kubekins-e2e image with runner and dependencies Update dockerized-e2e-runner.sh to just run an image with the appropriate e2e environment variables set. e2e-runner.sh, upload-to-gcs.sh and e2e.go now live inside the container.
lavalamp · Aug 17, 2016 · d13add0 · d13add0
2 parents 361dd00 + 40078b9
commit d13add0
Show file tree

Hide file tree

Showing 6 changed files with 145 additions and 30 deletions.
diff --git a/hack/jenkins/dockerized-e2e-runner.sh b/hack/jenkins/dockerized-e2e-runner.sh
@@ -35,6 +35,7 @@ env \
   -u PATH \
   -u PWD \
   -u WORKSPACE \
+  -u GOROOT \
   >${WORKSPACE}/env.list
 
 docker_extra_args=()
@@ -47,6 +48,7 @@ if [[ "${JENKINS_ENABLE_DOCKER_IN_DOCKER:-}" =~ ^[yY]$ ]]; then
     )
 fi
 
+echo "Starting..."
 docker run --rm=true -i \
   -v "${WORKSPACE}/_artifacts":/workspace/_artifacts \
   -v /etc/localtime:/etc/localtime:ro \
@@ -61,5 +63,4 @@ docker run --rm=true -i \
   -e "WORKSPACE=/workspace" \
   ${KUBEKINS_SERVICE_ACCOUNT_FILE:+-e "KUBEKINS_SERVICE_ACCOUNT_FILE=/service-account.json"} \
   "${docker_extra_args[@]:+${docker_extra_args[@]}}" \
-  gcr.io/google_containers/kubekins-test:go1.6.3-docker1.9.1-rev3 \
-  bash -c "bash <(curl -fsS --retry 3 --keepalive-time 2 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh')"
+  gcr.io/google-containers/kubekins-e2e:v20160810
diff --git a/hack/jenkins/e2e-image/Dockerfile b/hack/jenkins/e2e-image/Dockerfile
@@ -0,0 +1,50 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file creates a build environment for building and running kubernetes
+# unit and integration tests
+
+FROM gcr.io/google-containers/kubekins-test:v20160810
+MAINTAINER  Erick Fejta <fejta@google.com>
+
+# Defaults of all e2e runs
+ENV E2E_UP=true \
+    E2E_TEST=true \
+    E2E_DOWN=true
+
+# Customize these as appropriate
+ENV E2E_PUBLISH_GREEN_VERSION=false \
+    GINKGO_PARALLEL_NODES=25 \
+    GINKGO_TEST_ARGS="--ginkgo.focus=\[Conformance\]" \
+    INSTANCE_PREFIX=jenkins-e2e \
+    KUBERNETES_PROVIDER=gce
+
+# Variables specific to GCP
+ENV FAIL_ON_GCP_RESOURCE_LEAK=true \
+    JOB_NAME=kubernetes-e2e-gce-conformance \
+    KUBE_GCE_INSTANCE_PREFIX=jenkins-e2e \
+    KUBE_GCE_NETWORK=jenkins-e2e \
+    KUBE_GCE_ZONE=us-central1-f
+
+# Variable specific to the machine:
+# KUBEKINS_SERVICE_ACCOUNT_FILE
+# JENKINS_GCE_SSH_PRIVATE_KEY_FILE
+# JENKINS_GCE_SSH_PUBLIC_KEY_FILE
+# JENKINS_AWS_SSH_PRIVATE_KEY_FILE
+# JENKINS_AWS_SSH_PUBLIC_KEY_FILE
+# JENKINS_AWS_CREDENTIALS_FILE
+
+ADD ["sh2ju.sh", "e2e-runner.sh", "e2e.go", "upload-to-gcs.sh", "${WORKSPACE}/"]
+
+ENTRYPOINT ["bash", "-c", "${WORKSPACE}/e2e-runner.sh"]
diff --git a/hack/jenkins/e2e-image/Makefile b/hack/jenkins/e2e-image/Makefile
@@ -0,0 +1,43 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+IMG = gcr.io/google-containers/kubekins-e2e
+TAG = $(shell date +v%Y%m%d)
+
+e2e:
+	cp ../../e2e.go .
+
+e2e-runner:
+	cp ../e2e-runner.sh .
+
+sh2ju:
+	cp ../../../third_party/forked/shell2junit/sh2ju.sh .
+
+upload-to-gcs.sh:
+	cp ../upload-to-gcs.sh .
+
+clean:
+	rm -rf e2e.go e2e-runner.sh sh2ju.sh upload-to-gcs.sh
+
+all: push
+
+build: e2e e2e-runner sh2ju upload-to-gcs
+	docker build -t $(IMG):$(TAG) .
+	docker tag -f $(IMG):$(TAG) $(IMG):latest
+	@echo Built $(IMG):$(TAG) and tagged with latest
+
+push: build
+	gcloud docker push $(IMG):$(TAG)
+	gcloud docker push $(IMG):latest
+	@echo Pushed $(IMG) with :latest and :$(TAG) tags
diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh
@@ -21,8 +21,19 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
+if [[ -z "${PROJECT:-}" ]]; then
+  echo "ERROR: unset PROJECT" >&2
+  exit 1
+fi
+
 # include shell2junit library
-source <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/third_party/forked/shell2junit/sh2ju.sh')
+sh2ju="$(dirname "${0}")/sh2ju.sh"
+if [[ -f "${sh2ju}" ]]; then
+  source "${sh2ju}"
+else
+  echo "TODO(fejta): stop pulling sh2ju.sh"
+  source <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/third_party/forked/shell2junit/sh2ju.sh')
+fi
 
 # Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
 ARTIFACTS=${WORKSPACE}/_artifacts
@@ -347,7 +358,13 @@ cd kubernetes
 # Upload build start time and k8s version to GCS, but not on PR Jenkins.
 # On PR Jenkins this is done before the build.
 if [[ ! "${JOB_NAME}" =~ -pull- ]]; then
+  upload_to_gcs="$(dirname "${0}")/upload-to-gcs.sh"
+  if [[ -f "${upload_to_gcs}" ]]; then
+    JENKINS_BUILD_STARTED=true "${upload_to_gcs}"
+  else
+    echo "TODO(fejta): stop pulling upload-to-gcs.sh"
     JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 --keepalive-time 2 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")
+  fi
 fi
 
 # When run inside Docker, we need to make sure all files are world-readable
@@ -398,9 +415,12 @@ if [[ "${USE_KUBEMARK:-}" == "true" ]]; then
   e2e_go_args+=("--kubemark=true")
 fi
 
-go run ./hack/e2e.go \
-  ${E2E_OPT:-} \
-  "${e2e_go_args[@]}"
+e2e_go="$(dirname "${0}")/e2e.go"
+if [[ ! -f "${e2e_go}" ]]; then
+  echo "TODO(fejta): stop using head version of e2e.go"
+  e2e_go="./hack/e2e.go"
+fi
+go run "${e2e_go}" ${E2E_OPT:-} "${e2e_go_args[@]}"
 
 if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" ]]; then
   # Use plaintext version file packaged with kubernetes.tar.gz

diff --git a/hack/jenkins/test-image/Dockerfile b/hack/jenkins/test-image/Dockerfile
@@ -18,14 +18,14 @@
 FROM golang:1.6.3
 MAINTAINER  Jeff Lowdermilk <jeffml@google.com>
 
-ENV WORKSPACE               /workspace
-ENV TERM                    xterm
-# Note: 1.11+ changes the format of the tarball, so that line likely will need to be
-# changed.
-ENV DOCKER_VERSION          1.9.1
-
+# Setup workspace and symlink to gopath
 WORKDIR /workspace
+RUN mkdir -p /go/src/k8s.io/kubernetes /workspace \
+    && ln -s /go/src/k8s.io/kubernetes /workspace/kubernetes
+ENV WORKSPACE=/workspace \
+    TERM=xterm
 
+# Install linux packages
 # bc is needed by shell2junit
 # dnsutils is needed by federation cluster scripts.
 # file is used when uploading test artifacts to GCS.
@@ -43,12 +43,13 @@ RUN apt-get -o Acquire::Check-Valid-Until=false update && apt-get install -y \
 	--no-install-recommends \
 	&& rm -rf /var/lib/apt/lists/*
 
-RUN curl -L "https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz" |\
-  tar -C /usr/bin -xvzf- --strip-components=3 usr/local/bin/docker
-
-RUN go get github.com/golang/lint/golint
+# Install docker
+# Note: 1.11+ changes the tarball format
+RUN curl -L "https://get.docker.com/builds/Linux/x86_64/docker-1.9.1.tgz" \
+    | tar -C /usr/bin -xvzf- --strip-components=3 usr/local/bin/docker
 
-RUN mkdir -p /go/src/k8s.io/kubernetes \
-	&& ln -s /go/src/k8s.io/kubernetes /workspace/kubernetes
+# Install any go packages
+# TODO(fejta): migrate this to a unit test image
+RUN go get \
+        github.com/golang/lint/golint
 
-RUN /bin/bash
diff --git a/hack/jenkins/test-image/Makefile b/hack/jenkins/test-image/Makefile
@@ -12,18 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-all: push
+IMG = gcr.io/google-containers/kubekins-test
+TAG = $(shell date +v%Y%m%d)
 
-# Tag format: $GO_VERSION-$EMBEDDED_DOCKER_VERSION-$REVISION
-# These versions are specified in the Dockerfile
-TAG = go1.6.3-docker1.9.1-rev3
+all: build
 
-container:
-	docker build -t gcr.io/google_containers/kubekins-test .
-	docker tag gcr.io/google_containers/kubekins-test gcr.io/google_containers/kubekins-test:$(TAG)
+build:
+	docker build -t $(IMG):$(TAG) .
+	docker tag -f $(IMG):$(TAG) $(IMG):latest
+	@echo Built $(IMG):$(TAG) and tagged with $(IMG):latest
 
-push: container
-	gcloud docker push gcr.io/google_containers/kubekins-test # Push image tagged as latest to repository
-	gcloud docker push gcr.io/google_containers/kubekins-test:$(TAG) # Push version tagged image to repository (since this image is already pushed it will simply create or update version tag)
+push: build
+	gcloud docker push $(IMG):$(TAG)
+	gcloud docker push $(IMG):latest
+	@echo Pushed $(IMG):latest and $(IMG):$(TAG)
 
-clean: