Limit the number of concurrent tests in integration.go

Integration test often time out because the machine is loaded. Instead of increasing timeout, this change hopes to address the issue by limiting the number of tests running simultaneously. Add a new flag in integration.go to specify the maximum number of concurrent tests. Set the default in travis and shippable configurations to be 4.
nikhiljindal · Apr 9, 2015 · faf47b6 · faf47b6
1 parent 2215a64
commit faf47b6
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 13 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -24,7 +24,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"
diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go
@@ -67,6 +67,8 @@ var (
 	fakeDocker1, fakeDocker2 dockertools.FakeDockerClient
 	// API version that should be used by the client to talk to the server.
 	apiVersion string
+	// Limit the number of concurrent tests.
+	maxConcurrency int
 )
 
 type fakeKubeletClient struct{}
@@ -947,6 +949,8 @@ type testFunc func(*client.Client)
 
 func addFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&apiVersion, "apiVersion", latest.Version, "API version that should be used by the client for communicating with the server")
+	fs.IntVar(
+		&maxConcurrency, "maxConcurrency", -1, "Maximum number of tests to be run simultaneously. Unlimited if set to negative.")
 }
 
 func main() {
@@ -994,16 +998,26 @@ func main() {
 		},
 	}
 
-	var wg sync.WaitGroup
-	wg.Add(len(testFuncs))
-	for i := range testFuncs {
-		f := testFuncs[i]
-		go func() {
-			f(kubeClient)
-			wg.Done()
-		}()
+	// Only run at most maxConcurrency tests in parallel.
+	numFinishedTests := 0
+	for numFinishedTests < len(testFuncs) {
+		numTestsToRun := len(testFuncs) - numFinishedTests
+		if maxConcurrency > 0 && numTestsToRun > maxConcurrency {
+			numTestsToRun = maxConcurrency
+		}
+		glog.Infof("Running %d tests in parallel.", numTestsToRun)
+		var wg sync.WaitGroup
+		wg.Add(numTestsToRun)
+		for i := 0; i < numTestsToRun; i++ {
+			f := testFuncs[i+numFinishedTests]
+			go func() {
+				f(kubeClient)
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		numFinishedTests += numTestsToRun
 	}
-	wg.Wait()
 
 	// Check that kubelet tried to make the containers.
 	// Using a set to list unique creation attempts. Our fake is

diff --git a/hack/test-integration.sh b/hack/test-integration.sh
@@ -26,7 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 # Comma separated list of API Versions that should be tested.
 KUBE_TEST_API_VERSIONS=${KUBE_TEST_API_VERSIONS:-"v1beta1,v1beta3"}
-
+KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY:-"-1"}
 
 cleanup() {
   kube::etcd::cleanup
@@ -44,7 +44,8 @@ runTests() {
 
   kube::log::status "Running integration test scenario"
 
-  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1"
+  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1" \
+  --maxConcurrency="${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY}"
 
   cleanup
 }

diff --git a/shippable.yml b/shippable.yml
@@ -33,7 +33,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"