karmada-io · chaosi-zju · Dec 31, 2024
diff --git a/.github/workflows/installation-cli.yaml b/.github/workflows/installation-cli.yaml
@@ -44,7 +44,7 @@ jobs:
           hack/cli-testing-environment.sh
 
           # run a single e2e
-          export KUBECONFIG=${HOME}/karmada/karmada-apiserver.config
+          export KUBECONFIG=${HOME}/.kube/karmada-host.config:${HOME}/karmada/karmada-apiserver.config
           GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
           ginkgo -v --race --trace -p  --focus="[BasicPropagation] propagation testing deployment propagation testing"  ./test/e2e/
       - name: export logs
@@ -87,7 +87,7 @@ jobs:
           hack/cli-testing-init-with-config.sh
 
           # run a single e2e
-          export KUBECONFIG=${HOME}/karmada/karmada-apiserver.config
+          export KUBECONFIG=${HOME}/.kube/karmada-host.config:${HOME}/karmada/karmada-apiserver.config
           GO111MODULE=on go install github.com/onsi/ginkgo/v2/ginkgo
           ginkgo -v --race --trace -p  --focus="[BasicPropagation] propagation testing deployment propagation testing"  ./test/e2e/
       - name: export logs for config test

diff --git a/go.mod b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/onsi/gomega v1.34.1
 	github.com/opensearch-project/opensearch-go v1.1.0
 	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/common v0.55.0
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
@@ -134,7 +135,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.4.2 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect

diff --git a/test/e2e/framework/cluster.go b/test/e2e/framework/cluster.go
@@ -310,7 +310,15 @@ func WaitClusterFitWith(c client.Client, clusterName string, fit func(cluster *c
 
 // LoadRESTClientConfig creates a rest.Config using the passed kubeconfig. If context is empty, current context in kubeconfig will be used.
 func LoadRESTClientConfig(kubeconfig string, context string) (*rest.Config, error) {
-	loader := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}
+	var loader *clientcmd.ClientConfigLoadingRules
+	if strings.Contains(kubeconfig, ":") {
+		// kubeconfig is a list of kubeconfig files in form of "file1:file2:file3"
+		loader = &clientcmd.ClientConfigLoadingRules{Precedence: strings.Split(kubeconfig, ":")}
+	} else {
+		// kubeconfig is a single file
+		loader = &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}
+	}
+
 	loadedConfig, err := loader.Load()
 	if err != nil {
 		return nil, err

diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go
@@ -0,0 +1,74 @@
+/*
+Copyright 2023 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	"github.com/prometheus/common/model"
+	"k8s.io/component-base/metrics/testutil"
+
+	testhelper "github.com/karmada-io/karmada/test/helper"
+)
+
+var _ = ginkgo.Describe("metrics testing", func() {
+	var grabber *testhelper.Grabber
+
+	const workqueueQueueDuration = "workqueue_queue_duration_seconds_sum"
+
+	ginkgo.BeforeEach(func() {
+		var err error
+		grabber, err = testhelper.NewMetricsGrabber(context.TODO(), hostKubeClient)
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+	})
+
+	ginkgo.Context("workqueue metrics testing", func() {
+
+		ginkgo.It("verify whether workqueue metrics exist", func() {
+			var metrics *testutil.Metrics
+			var workqueueQueueDurationMetrics model.Samples
+
+			ginkgo.By("1. grab metrics from karmada controller", func() {
+				// the output format of `metrics` is like:
+				// [{
+				//	"metric": {
+				//		"__name__": "workqueue_queue_duration_seconds_sum",
+				//		"controller": "work-status-controller",
+				//		"name": "work-status-controller"
+				//	},
+				//	"value": [0, "0.12403110800000001"]
+				//}]
+				var err error
+				metrics, err = grabber.GrabMetricsFromKarmadaControllerManager(context.TODO())
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+			})
+
+			ginkgo.By("2. verify whether workqueue metrics exist", func() {
+				workqueueQueueDurationMetrics = (*metrics)[workqueueQueueDuration]
+				gomega.Expect(workqueueQueueDurationMetrics.Len()).Should(gomega.BeNumerically(">", 0))
+			})
+
+			ginkgo.By("3. verify the value of work-status-controller metric greater than 0", func() {
+				workStatusMetric := testhelper.GetMetricByName(workqueueQueueDurationMetrics, "work-status-controller")
+				gomega.Expect(workStatusMetric).ShouldNot(gomega.BeNil())
+				gomega.Expect(workStatusMetric.Value).Should(gomega.BeNumerically(">", 0))
+			})
+		})
+	})
+})
diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go
@@ -103,11 +103,13 @@ var (
 )
 
 var (
+	hostContext           string
 	karmadaContext        string
 	kubeconfig            string
 	karmadactlPath        string
 	restConfig            *rest.Config
 	karmadaHost           string
+	hostKubeClient        kubernetes.Interface
 	kubeClient            kubernetes.Interface
 	karmadaClient         karmada.Interface
 	dynamicClient         dynamic.Interface
@@ -125,7 +127,8 @@ func init() {
 	// eg. ginkgo -v --race --trace --fail-fast -p --randomize-all ./test/e2e/ -- --poll-interval=5s --poll-timeout=5m
 	flag.DurationVar(&pollInterval, "poll-interval", 5*time.Second, "poll-interval defines the interval time for a poll operation")
 	flag.DurationVar(&pollTimeout, "poll-timeout", 300*time.Second, "poll-timeout defines the time which the poll operation times out")
-	flag.StringVar(&karmadaContext, "karmada-context", karmadaContext, "Name of the cluster context in control plane kubeconfig file.")
+	flag.StringVar(&hostContext, "host-context", "karmada-host", "Name of the host cluster context in control plane kubeconfig file.")
+	flag.StringVar(&karmadaContext, "karmada-context", "karmada-apiserver", "Name of the karmada cluster context in control plane kubeconfig file.")
 }
 
 func TestE2E(t *testing.T) {
@@ -148,6 +151,13 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	gomega.Expect(karmadactlPath).ShouldNot(gomega.BeEmpty())
 
 	clusterProvider = cluster.NewProvider()
+
+	restConfig, err = framework.LoadRESTClientConfig(kubeconfig, hostContext)
+	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+
+	hostKubeClient, err = kubernetes.NewForConfig(restConfig)
+	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+
 	restConfig, err = framework.LoadRESTClientConfig(kubeconfig, karmadaContext)
 	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 

diff --git a/test/helper/metrics.go b/test/helper/metrics.go
@@ -0,0 +1,136 @@
+/*
+Copyright 2023 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helper
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/prometheus/common/model"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/component-base/metrics/testutil"
+	"k8s.io/klog/v2"
+)
+
+const (
+	karmadaNamespace                     = "karmada-system"
+	karmadaControllerManagerDeploy       = "karmada-controller-manager"
+	karmadaControllerManagerPort         = 8080
+	karmadaControllerManagerLeaderMetric = "leader_election_master_status"
+)
+
+// Grabber is used to grab metrics from karmada components
+type Grabber struct {
+	hostKubeClient        clientset.Interface
+	controllerManagerPods []string
+}
+
+// NewMetricsGrabber creates a new metrics grabber
+func NewMetricsGrabber(ctx context.Context, c clientset.Interface) (*Grabber, error) {
+	grabber := Grabber{hostKubeClient: c}
+	regKarmadaControllerManager := regexp.MustCompile(karmadaControllerManagerDeploy + "-.*")
+
+	podList, err := c.CoreV1().Pods(karmadaNamespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if len(podList.Items) < 1 {
+		klog.Warningf("Can't find any pods in namespace %s to grab metrics from", karmadaNamespace)
+	}
+	for _, pod := range podList.Items {
+		if regKarmadaControllerManager.MatchString(pod.Name) {
+			grabber.controllerManagerPods = append(grabber.controllerManagerPods, pod.Name)
+		}
+	}
+	return &grabber, nil
+}
+
+// GrabMetricsFromKarmadaControllerManager fetch metrics from the leader of karmada controller manager
+func (g *Grabber) GrabMetricsFromKarmadaControllerManager(ctx context.Context) (*testutil.Metrics, error) {
+	var output string
+	var lastMetricsFetchErr error
+
+	var result *testutil.Metrics
+	// judge which pod is the leader of karmada controller manager
+	for _, podName := range g.controllerManagerPods {
+		if metricsWaitErr := wait.PollUntilContextTimeout(ctx, time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) {
+			output, lastMetricsFetchErr = GetMetricsFromPod(ctx, g.hostKubeClient, podName, karmadaNamespace, karmadaControllerManagerPort)
+			return lastMetricsFetchErr == nil, nil
+		}); metricsWaitErr != nil {
+			klog.Errorf("error waiting for %s to expose metrics: %v; %v", podName, metricsWaitErr, lastMetricsFetchErr)
+			continue
+		}
+
+		podMetrics := testutil.Metrics{}
+		metricsParseErr := testutil.ParseMetrics(output, &podMetrics)
+		if metricsParseErr != nil {
+			klog.Errorf("failed to parse metrics for %s: %v", podName, metricsParseErr)
+			continue
+		}
+
+		if !isKarmadaControllerManagerLeader(podMetrics[karmadaControllerManagerLeaderMetric]) {
+			klog.Infof("skip fetch %s since it is not the leader pod", podName)
+			continue
+		}
+
+		result = &podMetrics
+		break
+	}
+
+	if result == nil {
+		return nil, fmt.Errorf("failed to fetch metrics from the leader of karmada controller manager")
+	}
+	return result, nil
+}
+
+// GetMetricsFromPod retrieves metrics data.
+func GetMetricsFromPod(ctx context.Context, client clientset.Interface, podName string, namespace string, port int) (string, error) {
+	rawOutput, err := client.CoreV1().RESTClient().Get().
+		Namespace(namespace).
+		Resource("pods").
+		SubResource("proxy").
+		Name(fmt.Sprintf("%s:%d", podName, port)).
+		Suffix("metrics").
+		Do(ctx).Raw()
+	if err != nil {
+		return "", err
+	}
+	return string(rawOutput), nil
+}
+
+func isKarmadaControllerManagerLeader(samples model.Samples) bool {
+	for _, sample := range samples {
+		if sample.Metric["name"] == karmadaControllerManagerDeploy && sample.Value > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// GetMetricByName returns the metric value with the given name.
+func GetMetricByName(samples model.Samples, name string) *model.Sample {
+	for _, sample := range samples {
+		if sample.Metric["name"] == model.LabelValue(name) {
+			return sample
+		}
+	}
+	return nil
+}