Skip to content

Commit

Permalink
[tests] Add ginkgo tests time analysis (kubernetes-sigs#3035)
Browse files Browse the repository at this point in the history
* [tests] Add ginkgo tests time analysis

* Review remarks
  • Loading branch information
trasc authored Sep 13, 2024
1 parent 9d413ca commit ccf57f1
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 7 deletions.
19 changes: 14 additions & 5 deletions Makefile-test.mk
Original file line number Diff line number Diff line change
Expand Up @@ -68,37 +68,40 @@ test: gotestsum ## Run tests.
$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml -- $(GO_TEST_FLAGS) $(shell $(GO_CMD) list ./... | grep -v '/test/') -coverpkg=./... -coverprofile $(ARTIFACTS)/cover.out

.PHONY: test-integration
test-integration: gomod-download envtest ginkgo dep-crds kueuectl ## Run tests.
test-integration: gomod-download envtest ginkgo dep-crds kueuectl ginkgo-top ## Run tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \
KUEUE_BIN=$(PROJECT_DIR)/bin \
ENVTEST_K8S_VERSION=$(ENVTEST_K8S_VERSION) \
API_LOG_LEVEL=$(INTEGRATION_API_LOG_LEVEL) \
$(GINKGO) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --race --junit-report=junit.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)
$(GINKGO) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --race --junit-report=junit.xml --json-report=integration.json --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)
$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/integration.json > $(ARTIFACTS)/integration-top.yaml

CREATE_KIND_CLUSTER ?= true
.PHONY: test-e2e
test-e2e: kustomize ginkgo yq gomod-download dep-crds kueuectl run-test-e2e-$(E2E_KIND_VERSION:kindest/node:v%=%)
test-e2e: kustomize ginkgo yq gomod-download dep-crds kueuectl ginkgo-top run-test-e2e-$(E2E_KIND_VERSION:kindest/node:v%=%)

.PHONY: test-multikueue-e2e
test-multikueue-e2e: kustomize ginkgo yq gomod-download dep-crds run-test-multikueue-e2e-$(E2E_KIND_VERSION:kindest/node:v%=%)
test-multikueue-e2e: kustomize ginkgo yq gomod-download dep-crds ginkgo-top run-test-multikueue-e2e-$(E2E_KIND_VERSION:kindest/node:v%=%)


E2E_TARGETS := $(addprefix run-test-e2e-,${E2E_K8S_VERSIONS})
MULTIKUEUE-E2E_TARGETS := $(addprefix run-test-multikueue-e2e-,${E2E_K8S_VERSIONS})
.PHONY: test-e2e-all
test-e2e-all: ginkgo $(E2E_TARGETS) $(MULTIKUEUE-E2E_TARGETS)
test-e2e-all: ginkgo ginkgo-top $(E2E_TARGETS) $(MULTIKUEUE-E2E_TARGETS)

FORCE:

run-test-e2e-%: K8S_VERSION = $(@:run-test-e2e-%=%)
run-test-e2e-%: FORCE
@echo Running e2e for k8s ${K8S_VERSION}
E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/e2e-test.sh
$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml

run-test-multikueue-e2e-%: K8S_VERSION = $(@:run-test-multikueue-e2e-%=%)
run-test-multikueue-e2e-%: FORCE
@echo Running multikueue e2e for k8s ${K8S_VERSION}
E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/multikueue-e2e-test.sh
$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml

SCALABILITY_RUNNER := $(PROJECT_DIR)/bin/performance-scheduler-runner
.PHONY: performance-scheduler-runner
Expand Down Expand Up @@ -160,3 +163,9 @@ run-performance-scheduler-in-cluster: envtest performance-scheduler-runner
--o $(ARTIFACTS)/run-performance-scheduler-in-cluster \
--generatorConfig=$(SCALABILITY_GENERATOR_CONFIG) \
--qps=1000 --burst=2000 --timeout=15m $(SCALABILITY_SCRAPE_ARGS)

.PHONY: ginkgo-top
ginkgo-top:
cd $(PROJECT_DIR)/hack/internal/tools && \
go mod download && \
$(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o $(PROJECT_DIR)/bin/ginkgo-top ./ginkgo-top
2 changes: 1 addition & 1 deletion hack/e2e-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,4 @@ startup
kind_load
kueue_deploy
# shellcheck disable=SC2086
$GINKGO $GINKGO_ARGS --junit-report=junit.xml --output-dir="$ARTIFACTS" -v ./test/e2e/singlecluster/...
$GINKGO $GINKGO_ARGS --junit-report=junit.xml --json-report=e2e.json --output-dir="$ARTIFACTS" -v ./test/e2e/singlecluster/...
102 changes: 102 additions & 0 deletions hack/internal/tools/ginkgo-top/ginkgo-top.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"cmp"
"encoding/json"
"flag"
"fmt"
"os"
"slices"
"time"

gtypes "github.com/onsi/ginkgo/v2/types"
"gopkg.in/yaml.v3"
)

type SpecDescription struct {
Suite string
ContainerHierarchy []string
Name string
Duration time.Duration
Events []Event
}

type Event struct {
Message string
Location string
Duration time.Duration
}

var (
reportPath = flag.String("i", "ginkgo.json", "input file name")
maxCount = flag.Uint("maxCount", 0, "maximum number of items")
withEvents = flag.Bool("withEvents", true, "show events")
)

func main() {
flag.Parse()
reprtData, err := os.ReadFile(*reportPath)
if err != nil {
panic(err)
}

report := []gtypes.Report{}
err = json.Unmarshal(reprtData, &report)
if err != nil {
panic(err)
}

flattenSpecs := []SpecDescription{}
for _, r := range report {
for _, s := range r.SpecReports {
sd := SpecDescription{
Suite: r.SuiteDescription,
ContainerHierarchy: s.ContainerHierarchyTexts,
Name: s.LeafNodeText,
Duration: s.RunTime,
}
if *withEvents {
for _, ev := range s.SpecEvents {
if ev.Duration > 0 {
event := Event{
Location: ev.CodeLocation.String(),
Message: fmt.Sprintf("%s (%s)", ev.Message, ev.NodeType),
Duration: ev.Duration,
}
sd.Events = append(sd.Events, event)
}
}
slices.SortFunc(sd.Events, func(a, b Event) int {
return cmp.Compare(b.Duration, a.Duration)
})
}
flattenSpecs = append(flattenSpecs, sd)
}
}
slices.SortFunc(flattenSpecs, func(a, b SpecDescription) int {
return cmp.Compare(b.Duration, a.Duration)
})

yout := yaml.NewEncoder(os.Stdout)
if *maxCount > 0 {
yout.Encode(flattenSpecs[:min(int(*maxCount), len(flattenSpecs))])
} else {
yout.Encode(flattenSpecs)
}
}
2 changes: 1 addition & 1 deletion hack/multikueue-e2e-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -121,4 +121,4 @@ kind_load
kueue_deploy

# shellcheck disable=SC2086
$GINKGO $GINKGO_ARGS --junit-report=junit.xml --output-dir="$ARTIFACTS" -v ./test/e2e/multikueue/...
$GINKGO $GINKGO_ARGS --junit-report=junit.xml --json-report=e2e.json --output-dir="$ARTIFACTS" -v ./test/e2e/multikueue/...

0 comments on commit ccf57f1

Please sign in to comment.