e2e: add API call budget check

openshift · Feb 22, 2022 · ab06110 · ab06110
1 parent f3b29fc
commit ab06110
Show file tree

Hide file tree

Showing 27 changed files with 6,154 additions and 2 deletions.
diff --git a/go.mod b/go.mod
@@ -28,6 +28,7 @@ require (
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.51.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
+	github.com/prometheus/common v0.28.0
 	github.com/spf13/cobra v1.2.1
 	github.com/tombuildsstuff/giovanni v0.18.0
 	github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50
@@ -91,14 +92,15 @@ require (
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca // indirect
 	github.com/pborman/uuid v1.2.0 // indirect
-	github.com/prometheus/common v0.28.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

diff --git a/go.sum b/go.sum
@@ -836,6 +836,7 @@ github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqx
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -959,6 +960,7 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=

diff --git a/test/e2e/autorepair_test.go b/test/e2e/autorepair_test.go
@@ -76,6 +76,7 @@ func TestAutoRepair(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred(), "failed to wait for new node to become available")
 
 	e2eutil.EnsureNoCrashingPods(t, ctx, client, hostedCluster)
+	e2eutil.EnsureAPIBudget(t, ctx, client, hostedCluster)
 }
 
 func ec2Client(awsCredsFile, region string) *ec2.EC2 {

diff --git a/test/e2e/autoscaling_test.go b/test/e2e/autoscaling_test.go
@@ -110,6 +110,7 @@ func TestAutoscaling(t *testing.T) {
 	_ = e2eutil.WaitForNReadyNodes(t, testContext, guestClient, numNodes)
 
 	e2eutil.EnsureNoCrashingPods(t, ctx, client, hostedCluster)
+	e2eutil.EnsureAPIBudget(t, ctx, client, hostedCluster)
 }
 
 func newWorkLoad(njobs int32, memoryRequest resource.Quantity, nodeSelector, image string, zone string) *batchv1.Job {

diff --git a/test/e2e/control_plane_upgrade_test.go b/test/e2e/control_plane_upgrade_test.go
@@ -60,4 +60,5 @@ func TestUpgradeControlPlane(t *testing.T) {
 
 	e2eutil.EnsureNodeCountMatchesNodePoolReplicas(t, testContext, client, guestClient, hostedCluster.Namespace)
 	e2eutil.EnsureNoCrashingPods(t, ctx, client, hostedCluster)
+	e2eutil.EnsureAPIBudget(t, ctx, client, hostedCluster)
 }
diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go
@@ -19,6 +19,13 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	routev1 "github.com/openshift/api/route/v1"
+
+	promapi "github.com/prometheus/client_golang/api"
+	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	promconfig "github.com/prometheus/common/config"
+	prommodel "github.com/prometheus/common/model"
+
 	hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests"
 )
@@ -233,7 +240,7 @@ func WaitForConditionsOnHostedControlPlane(t *testing.T, ctx context.Context, cl
 }
 
 func EnsureNoCrashingPods(t *testing.T, ctx context.Context, client crclient.Client, hostedCluster *hyperv1.HostedCluster) {
-	t.Run("No controlplane pods crash", func(t *testing.T) {
+	t.Run("EnsureNoCrashingPods", func(t *testing.T) {
 		namespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name).Name
 
 		var podList corev1.PodList
@@ -283,3 +290,148 @@ func EnsureNodeCountMatchesNodePoolReplicas(t *testing.T, ctx context.Context, h
 		}
 	})
 }
+
+func EnsureAPIBudget(t *testing.T, ctx context.Context, client crclient.Client, hostedCluster *hyperv1.HostedCluster) {
+	t.Run("EnsureAPIBudget", func(t *testing.T) {
+
+		// Get hypershift-operator token
+		operatorServiceAccount := &corev1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "operator",
+				Namespace: "hypershift",
+			},
+		}
+		if err := client.Get(ctx, crclient.ObjectKeyFromObject(operatorServiceAccount), operatorServiceAccount); err != nil {
+			t.Fatalf("failed to get hypershift operator service account: %v", err)
+		}
+		var secretName string
+		for _, secret := range operatorServiceAccount.Secrets {
+			if strings.HasPrefix(secret.Name, "operator-token-") {
+				secretName = secret.Name
+				break
+			}
+		}
+		if secretName == "" {
+			t.Fatal("failed to determine hypershift operator token secret name")
+		}
+		tokenSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      secretName,
+				Namespace: "hypershift",
+			},
+		}
+		if err := client.Get(ctx, crclient.ObjectKeyFromObject(tokenSecret), tokenSecret); err != nil {
+			t.Fatalf("failed to get hypershift operator token secret: %v", err)
+		}
+		token, ok := tokenSecret.Data["token"]
+		if !ok {
+			t.Fatal("token secret did not contain a token value")
+		}
+
+		// Get thanos-querier endpoint
+		promRoute := &routev1.Route{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "thanos-querier",
+				Namespace: "openshift-monitoring",
+			},
+		}
+		if err := client.Get(ctx, crclient.ObjectKeyFromObject(promRoute), promRoute); err != nil {
+			t.Skip("unable to get prometheus route, skipping")
+		}
+		if len(promRoute.Status.Ingress) == 0 {
+			t.Skip("unable to get prometheus ingress, skipping")
+		}
+		promEndpoint := fmt.Sprintf("https://%s", promRoute.Status.Ingress[0].Host)
+
+		// Create prometheus client
+		cfg := promconfig.HTTPClientConfig{
+			Authorization: &promconfig.Authorization{
+				Type:        "Bearer",
+				Credentials: promconfig.Secret(token),
+			},
+			TLSConfig: promconfig.TLSConfig{
+				InsecureSkipVerify: true,
+			},
+		}
+		rt, err := promconfig.NewRoundTripperFromConfig(cfg, "e2e-budget-checker")
+		if err != nil {
+			t.Fatalf("failed to get create round tripper: %v", err)
+		}
+		client, err := promapi.NewClient(promapi.Config{
+			Address:      promEndpoint,
+			RoundTripper: rt,
+		})
+		if err != nil {
+			t.Fatalf("failed to get create prometheus client: %v", err)
+		}
+		v1api := promv1.NewAPI(client)
+
+		// Compare metrics against budgets
+		namespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name).Name
+		budgets := []struct {
+			name   string
+			query  string
+			budget float64
+		}{
+			{
+				name:   "control-plane-operator read",
+				query:  fmt.Sprintf("sum(hypershift:controlplane:component_api_requests_total{method=\"GET\", namespace=~\"%s\"}) by (pod)", namespace),
+				budget: 600,
+			},
+			{
+				name:   "control-plane-operator mutate",
+				query:  fmt.Sprintf("sum(hypershift:controlplane:component_api_requests_total{method!=\"GET\", namespace=~\"%s\"}) by (pod)", namespace),
+				budget: 400,
+			},
+			{
+				name:   "control-plane-operator no 404 deletes",
+				query:  fmt.Sprintf("sum(hypershift:controlplane:component_api_requests_total{method=\"DELETE\", code=\"404\", namespace=~\"%s\"}) by (pod)", namespace),
+				budget: 0,
+			},
+			// hypershift-operator budget can not be per HC so metric will be significantly under budget for all but the last test(s) to complete on a particular test cluster
+			// These budgets will also need to scale up with additional tests that create HostedClusters
+			{
+				name:   "hypershift-operator read",
+				query:  "sum(hypershift:operator:component_api_requests_total{method=\"GET\"})",
+				budget: 1500,
+			},
+			{
+				name:   "hypershift-operator mutate",
+				query:  "sum(hypershift:operator:component_api_requests_total{method!=\"GET\"})",
+				budget: 2000,
+			},
+			{
+				name:   "hypershift-operator no 404 deletes",
+				query:  "sum(hypershift:operator:component_api_requests_total{method=\"DELETE\", code=\"404\"})",
+				budget: 0,
+			},
+		}
+
+		for _, budget := range budgets {
+			t.Run(budget.name, func(t *testing.T) {
+				result, _, err := v1api.Query(ctx, budget.query, time.Now())
+				if err != nil {
+					t.Fatalf("failed to query prometheus: %v", err)
+				}
+				vector, ok := result.(prommodel.Vector)
+				if !ok {
+					t.Fatal("expected vector result")
+				}
+				if len(vector) == 0 {
+					if budget.budget == 0 {
+						t.Log("no samples returned for query with zero budget, skipping check")
+					} else {
+						t.Errorf("no samples returned for query with non-zero budget, failed check")
+					}
+				}
+				for _, sample := range vector {
+					if float64(sample.Value) > budget.budget {
+						t.Errorf("over budget: budget: %.0f, actual: %.0f", budget.budget, sample.Value)
+					} else {
+						t.Logf("within budget: budget: %.0f, actual: %.0f", budget.budget, sample.Value)
+					}
+				}
+			})
+		}
+	})
+}
diff --git a/vendor/github.com/jpillora/backoff/LICENSE b/vendor/github.com/jpillora/backoff/LICENSE
diff --git a/vendor/github.com/jpillora/backoff/README.md b/vendor/github.com/jpillora/backoff/README.md