Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

run focus tests in parallel where possible #15848

Merged
merged 1 commit into from
Aug 22, 2017
Merged

run focus tests in parallel where possible #15848

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,18 +210,15 @@ test-tools:
# SUITE: Which Bash entrypoint under test/extended/ to use. Don't include the
# ending `.sh`. Ex: `core`.
# FOCUS: Literal string to pass to `--ginkgo.focus=`
# The FOCUS env variable is handled by the respective suite scripts.
#
# Example:
# make test-extended SUITE=core
# make test-extended SUITE=conformance FOCUS=pods
#
SUITE ?= conformance
ifneq ($(strip $(FOCUS)),)
FOCUS_ARG=--ginkgo.focus="$(FOCUS)"
else
FOCUS_ARG=
endif
test-extended:
test/extended/$(SUITE).sh $(FOCUS_ARG)
test/extended/$(SUITE).sh
.PHONY: test-extended

# Run All-in-one OpenShift server.
Expand Down
10 changes: 5 additions & 5 deletions test/extended/builds/run_policy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ var _ = g.Describe("[builds][Slow] using build configuration runPolicy", func()
}
if build.Status.Phase == buildapi.BuildPhaseRunning {
latency := lastCompletion.Sub(time.Now())
o.Expect(latency).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after last completed build")
o.Expect(latency).To(o.BeNumerically("<", 15*time.Second), "next build should have started less than 15s after last completed build")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Intended?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, running in parallel seems to have thrown the timing off a bit by slowing things down.


// Ignore events from complete builds (if there are any) if we already
// verified the build.
Expand Down Expand Up @@ -209,14 +209,14 @@ var _ = g.Describe("[builds][Slow] using build configuration runPolicy", func()
}
if build.Name == "sample-serial-build-2" {
duration := time.Now().Sub(cancelTime)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after canceled build")
o.Expect(duration).To(o.BeNumerically("<", 15*time.Second), "next build should have started less than 15s after canceled build")
err := oc.Run("cancel-build").Args("sample-serial-build-2").Execute()
o.Expect(err).ToNot(o.HaveOccurred())
cancelTime2 = time.Now()
}
if build.Name == "sample-serial-build-3" {
duration := time.Now().Sub(cancelTime2)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after canceled build")
o.Expect(duration).To(o.BeNumerically("<", 15*time.Second), "next build should have started less than 15s after canceled build")
break
}
}
Expand Down Expand Up @@ -261,7 +261,7 @@ var _ = g.Describe("[builds][Slow] using build configuration runPolicy", func()
}
if build.Name == "sample-serial-build-fail-2" {
duration := time.Now().Sub(failTime)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after failed build")
o.Expect(duration).To(o.BeNumerically("<", 15*time.Second), "next build should have started less than 15s after failed build")
if build.Status.CompletionTimestamp != nil {
o.Expect(build.Status.StartTimestamp).ToNot(o.BeNil(), "failed builds should have a valid start time")
o.Expect(build.Status.CompletionTimestamp).ToNot(o.BeNil(), "failed builds should have a valid completion time")
Expand All @@ -271,7 +271,7 @@ var _ = g.Describe("[builds][Slow] using build configuration runPolicy", func()
}
if build.Name == "sample-serial-build-fail-3" {
duration := time.Now().Sub(failTime2)
o.Expect(duration).To(o.BeNumerically("<", 10*time.Second), "next build should have started less than 10s after failed build")
o.Expect(duration).To(o.BeNumerically("<", 15*time.Second), "next build should have started less than 15s after failed build")
if build.Status.CompletionTimestamp != nil {
o.Expect(build.Status.StartTimestamp).ToNot(o.BeNil(), "failed builds should have a valid start time")
o.Expect(build.Status.CompletionTimestamp).ToNot(o.BeNil(), "failed builds should have a valid completion time")
Expand Down
3 changes: 2 additions & 1 deletion test/extended/builds/s2i_env.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ var _ = g.Describe("[builds][Slow] s2i build with environment file in sources",
o.Expect(err).NotTo(o.HaveOccurred())

g.By("starting a test build")
br, _ := exutil.StartBuildAndWait(oc, "test", "--from-dir", "test/extended/testdata/sti-environment-build-app")
path := exutil.FixturePath("testdata", "sti-environment-build-app")
br, _ := exutil.StartBuildAndWait(oc, "test", "--from-dir", path)
br.AssertSuccess()

g.By("getting the Docker image reference from ImageStream")
Expand Down
10 changes: 10 additions & 0 deletions test/extended/networking.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,12 @@ function run-extended-tests() {
local test_args="--test.v '--ginkgo.skip=${skip_regex}' \
'--ginkgo.focus=${focus_regex}' ${TEST_EXTRA_ARGS}"

# this ${FOCUS} value will override the $focus_regex in the same way that
# the --ginkgo.focus argument did previously.
if [[ -n "${FOCUS:-}" ]]; then
test_args="${test_args} --ginkgo.focus=${FOCUS}"
fi

if [[ -n "${dlv_debug}" ]]; then
# run tests using delve debugger
local extended_test; extended_test="$( os::util::find::built_binary extended.test )"
Expand Down Expand Up @@ -283,6 +289,10 @@ esac

TEST_EXTRA_ARGS="$@"

if [[ "$@[@]" =~ "ginkgo.focus" ]]; then
os::log::fatal "the --ginkgo.focus flag is no longer supported, use FOCUS=foo <suite.sh> instead."
fi

if [[ -n "${OPENSHIFT_SKIP_BUILD:-}" ]] &&
os::util::find::built_binary 'extended.test' >/dev/null 2>&1; then
os::log::warning "Skipping rebuild of test binary due to OPENSHIFT_SKIP_BUILD=1"
Expand Down
31 changes: 22 additions & 9 deletions test/extended/setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,28 @@

# If invoked with arguments, executes the test directly.
function os::test::extended::focus () {
if [[ $# -ne 0 ]]; then
os::log::info "Running custom: $*"
os::test::extended::test_list "$@"
if [[ "${TEST_COUNT}" -eq 0 ]]; then
os::log::error "No tests would be run"
exit 1
fi
extended.test "$@"
exit $?
if [[ "$@[@]" =~ "ginkgo.focus" ]]; then
os::log::fatal "the --ginkgo.focus flag is no longer supported, use FOCUS=foo <suite.sh> instead."
exit 1
fi
if [[ -n "${FOCUS:-}" ]]; then
exitstatus=0

# first run anything that isn't explicitly declared [Serial], and matches the $FOCUS, in a parallel mode.
os::log::info "Running parallel tests N=${PARALLEL_NODES:-<default>} with focus ${FOCUS}"
TEST_PARALLEL="${PARALLEL_NODES:-5}" TEST_REPORT_FILE_NAME=core_parallel os::test::extended::run -- -ginkgo.skip "\[Serial\]" -test.timeout 6h ${TEST_EXTENDED_ARGS-} || exitstatus=$?

# Then run everything that requires serial and matches the $FOCUS, serially.
# there is bit of overlap here because not all serial tests declare [Serial], so they might have run in the
# parallel section above. Hopefully your focus was precise enough to exclude them, and we should be adding
# the [Serial] tag to them as needed.
os::log::info ""
os::log::info "Running serial tests with focus ${FOCUS}"
TEST_REPORT_FILE_NAME=core_parallel os::test::extended::run -- -suite "serial.conformance.openshift.io" -test.timeout 6h ${TEST_EXTENDED_ARGS-} || exitstatus=$?

os::test::extended::merge_junit

exit $exitstatus
fi
}

Expand Down
19 changes: 13 additions & 6 deletions test/extended/templates/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,16 +94,23 @@ func EnsureTSB(tsbOC *exutil.CLI) (osbclient.Client, *exec.Cmd) {
if !exists {
e2e.Logf("Installing TSB onto the cluster for testing")
_, _, err := tsbOC.AsAdmin().WithoutNamespace().Run("create", "namespace").Args(tsbNS).Outputs()
o.Expect(err).NotTo(o.HaveOccurred())
// If template tests run in parallel this could be created twice, we don't really care.
if err != nil {
e2e.Logf("Error creating TSB namespace %s: %v \n", tsbNS, err)
}
configPath := exutil.FixturePath("..", "..", "examples", "templateservicebroker", "templateservicebroker-template.yaml")
stdout, _, err := tsbOC.WithoutNamespace().Run("process").Args("-f", configPath).Outputs()
o.Expect(err).NotTo(o.HaveOccurred())
if err != nil {
e2e.Logf("Error processing TSB template at %s: %v \n", configPath, err)
}
err = tsbOC.WithoutNamespace().AsAdmin().Run("create").Args("-f", "-").InputString(stdout).Execute()
// this is weird, but we have to ignore this error in case some of this already exists (race between tests and the like)
//o.Expect(err).NotTo(o.HaveOccurred())
err = WaitForDaemonSetStatus(tsbOC.AdminKubeClient(), &extensions.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "apiserver", Namespace: tsbNS}})
o.Expect(err).NotTo(o.HaveOccurred())
if err != nil {
// If template tests run in parallel this could be created twice, we don't really care.
e2e.Logf("Error creating TSB resources: %v \n", err)
}
}
err := WaitForDaemonSetStatus(tsbOC.AdminKubeClient(), &extensions.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: "apiserver", Namespace: tsbNS}})
o.Expect(err).NotTo(o.HaveOccurred())

// we're trying to test the TSB, not the service. We're outside all the normal networks. Run a portforward to a particular
// pod and test that
Expand Down