[test][scalability] Retry performance test on failure (kubernetes-sig…

…s#3020) * [test][scalability] Retry performance test on failure * Review remarks
KunWuLuan · Sep 11, 2024 · 3009775 · 3009775
1 parent 6711364
commit 3009775
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 2 deletions.
diff --git a/Makefile-test.mk b/Makefile-test.mk
@@ -139,13 +139,18 @@ run-performance-scheduler: envtest performance-scheduler-runner minimalkueue
 		--generatorConfig=$(SCALABILITY_GENERATOR_CONFIG) \
 		--minimalKueue=$(MINIMALKUEUE_RUNNER) $(SCALABILITY_EXTRA_ARGS) $(SCALABILITY_SCRAPE_ARGS)
 
-.PHONY: test-performance-scheduler
-test-performance-scheduler: gotestsum run-performance-scheduler
+.PHONY: test-performance-scheduler-once
+test-performance-scheduler-once: gotestsum run-performance-scheduler
 	$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml -- $(GO_TEST_FLAGS) ./test/performance/scheduler/checker  \
 		--summary=$(SCALABILITY_RUN_DIR)/summary.yaml \
 		--cmdStats=$(SCALABILITY_RUN_DIR)/minimalkueue.stats.yaml \
 		--range=$(PROJECT_DIR)/test/performance/scheduler/default_rangespec.yaml
 
+PERFORMANCE_RETRY_COUNT?=2
+.PHONY: test-performance-scheduler
+test-performance-scheduler:
+	ARTIFACTS=$(ARTIFACTS) ./hack/performance-retry.sh $(PERFORMANCE_RETRY_COUNT)
+
 .PHONY: run-performance-scheduler-in-cluster
 run-performance-scheduler-in-cluster: envtest performance-scheduler-runner
 	mkdir -p $(ARTIFACTS)/run-performance-scheduler-in-cluster

diff --git a/hack/performance-retry.sh b/hack/performance-retry.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "$#" -lt 1 ]]; then
+	echo "Usage: ${0} <num-retries>"
+	exit 1
+fi
+
+
+for i in $(seq 1 "$1"); do
+	echo "Try ${i}"
+	err=0
+	make test-performance-scheduler-once || err=$?
+	if [ $err -eq 0 ]; then
+		break
+	else
+		if [ "$i" -lt "$1" ]; then
+			mv "${ARTIFACTS}/junit.xml" "${ARTIFACTS}/junit-fail-${i}.xml" || echo "Unable to back-up ${ARTIFACTS}/junit.xml"
+			mv "${ARTIFACTS}/run-performance-scheduler" "${ARTIFACTS}/run-performance-scheduler-fail-${i}" || echo "Unable to back-up ${ARTIFACTS}/run-performance-scheduler"
+		else 
+			exit $err
+		fi
+	fi
+done