Merge pull request moby#6943 from tiborvass/parallel_compile_test

Parallel compile test binaries
rjnagal · Jul 28, 2014 · 5ef1126 · 5ef1126
2 parents d09e682 + 4cfe806
commit 5ef1126
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 27 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -44,6 +44,7 @@ RUN	apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \
 	lxc=1.0* \
 	mercurial \
 	pandoc \
+	parallel \
 	reprepro \
 	ruby1.9.1 \
 	ruby1.9.1-dev \

diff --git a/hack/make.sh b/hack/make.sh
@@ -42,17 +42,17 @@ echo
 DEFAULT_BUNDLES=(
 	validate-dco
 	validate-gofmt
-	
+
 	binary
-	
+
 	test-unit
 	test-integration
 	test-integration-cli
-	
+
 	dynbinary
 	dyntest-unit
 	dyntest-integration
-	
+
 	cover
 	cross
 	tgz
@@ -156,6 +156,31 @@ go_test_dir() {
 	)
 }
 
+# Compile phase run by parallel in test-unit. No support for coverpkg
+go_compile_test_dir() {
+	dir=$1
+	out_file="$DEST/precompiled/$dir.test"
+	testcover=()
+	if [ "$HAVE_GO_TEST_COVER" ]; then
+		# if our current go install has -cover, we want to use it :)
+		mkdir -p "$DEST/coverprofiles"
+		coverprofile="docker${dir#.}"
+		coverprofile="$DEST/coverprofiles/${coverprofile//\//-}"
+		testcover=( -cover -coverprofile "$coverprofile" ) # missing $coverpkg
+	fi
+	if [ "$BUILDFLAGS_FILE" ]; then
+		readarray -t BUILDFLAGS < "$BUILDFLAGS_FILE"
+	fi
+	(
+		cd "$dir"
+		go test "${testcover[@]}" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS -c
+	)
+	[ $? -ne 0 ] && return 1
+	mkdir -p "$(dirname "$out_file")"
+	mv "$dir/$(basename "$dir").test" "$out_file"
+	echo "Precompiled: github.com/dotcloud/docker${dir#.}"
+}
+
 # This helper function walks the current directory looking for directories
 # holding certain files ($1 parameter), and prints their paths on standard
 # output, one per line.

diff --git a/hack/make/test-unit b/hack/make/test-unit
@@ -2,6 +2,7 @@
 set -e
 
 DEST=$1
+: ${PARALLEL_JOBS:=$(nproc)}
 
 RED=$'\033[31m'
 GREEN=$'\033[32m'
@@ -22,34 +23,62 @@ bundle_test_unit() {
 			TESTDIRS=$(find_dirs '*_test.go')
 		fi
 
-		TESTS_FAILED=()
-		for test_dir in $TESTDIRS; do
-			echo
+		if command -v parallel &> /dev/null; then (
+			# accomodate parallel to be able to access variables
+			export SHELL="$BASH"
+			export HOME="$(mktemp -d)"
+			mkdir -p "$HOME/.parallel"
+			touch "$HOME/.parallel/ignored_vars"
+			export -f go_compile_test_dir
+			export LDFLAGS="$LDFLAGS $LDFLAGS_STATIC_DOCKER"
+			export TESTFLAGS
+			export HAVE_GO_TEST_COVER
+			export DEST
+			# some hack to export array variables
+			export BUILDFLAGS_FILE="$HOME/buildflags_file"
+			( IFS=$'\n'; echo "${BUILDFLAGS[*]}" ) > "$BUILDFLAGS_FILE"
 
-			if ! LDFLAGS="$LDFLAGS $LDFLAGS_STATIC_DOCKER" go_test_dir "$test_dir"; then
-				TESTS_FAILED+=("$test_dir")
-				echo
-				echo "${RED}Tests failed: $test_dir${TEXTRESET}"
-				sleep 1 # give it a second, so observers watching can take note
-			fi
-		done
+			echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --halt 2 --env _ go_compile_test_dir
+			rm -rf "$HOME"
+		) else
+			# aww, no "parallel" available - fall back to boring
+			for test_dir in $TESTDIRS; do
+				go_compile_test_dir "$test_dir"
+			done
+		fi
+		echo "$TESTDIRS" | go_run_test_dir
+	}
+}
 
+go_run_test_dir() {
+	TESTS_FAILED=()
+	while read dir; do
 		echo
-		echo
-		echo
-
-		# if some tests fail, we want the bundlescript to fail, but we want to
-		# try running ALL the tests first, hence TESTS_FAILED
-		if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then
-			echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}"
+		echo '+ go test' $TESTFLAGS "github.com/dotcloud/docker${dir#.}"
+		precompiled="$DEST/precompiled/$dir.test"
+		if ! ( cd "$dir" && "$precompiled" ); then
+			TESTS_FAILED+=("$dir")
 			echo
-			false
-		else
-			echo "${GREEN}Test success${TEXTRESET}"
-			echo
-			true
+			echo "${RED}Tests failed: $dir${TEXTRESET}"
+			sleep 1 # give it a second, so observers watching can take note
 		fi
-	}
+	done
+
+	echo
+	echo
+	echo
+
+	# if some tests fail, we want the bundlescript to fail, but we want to
+	# try running ALL the tests first, hence TESTS_FAILED
+	if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then
+		echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}"
+		echo
+		false
+	else
+		echo "${GREEN}Test success${TEXTRESET}"
+		echo
+		true
+	fi
 }
 
 exec > >(tee -a $DEST/test.log) 2>&1