Merge pull request #5229 from openshift/rapid-deployment-detection

NO-JIRA: test/e2e: detect rapidly updating deployments
openshift · Dec 13, 2024 · 618a614 · 618a614
2 parents dc7bd65 + d97908d
commit 618a614
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 0 deletions.
diff --git a/test/e2e/util/hypershift_framework.go b/test/e2e/util/hypershift_framework.go
@@ -141,6 +141,7 @@ func (h *hypershiftTest) after(hostedCluster *hyperv1.HostedCluster, platform hy
 		EnsureAllContainersHavePullPolicyIfNotPresent(t, context.Background(), h.client, hostedCluster)
 		EnsureHCPContainersHaveResourceRequests(t, context.Background(), h.client, hostedCluster)
 		EnsureNoPodsWithTooHighPriority(t, context.Background(), h.client, hostedCluster)
+		EnsureNoRapidDeploymentRollouts(t, context.Background(), h.client, hostedCluster)
 		NoticePreemptionOrFailedScheduling(t, context.Background(), h.client, hostedCluster)
 		EnsureAllRoutesUseHCPRouter(t, context.Background(), h.client, hostedCluster)
 		EnsureNetworkPolicies(t, context.Background(), h.client, hostedCluster)

diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go
@@ -553,6 +553,23 @@ func NoticePreemptionOrFailedScheduling(t *testing.T, ctx context.Context, clien
 	})
 }
 
+func EnsureNoRapidDeploymentRollouts(t *testing.T, ctx context.Context, client crclient.Client, hostedCluster *hyperv1.HostedCluster) {
+	const maxAllowedGeneration = 10
+	t.Run("EnsureNoRapidDeploymentRollouts", func(t *testing.T) {
+		namespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name)
+
+		var deploymentList appsv1.DeploymentList
+		if err := client.List(ctx, &deploymentList, crclient.InNamespace(namespace)); err != nil {
+			t.Fatalf("failed to list deployments in namespace %s: %v", namespace, err)
+		}
+		for _, deployment := range deploymentList.Items {
+			if deployment.Generation > maxAllowedGeneration {
+				t.Errorf("Rapidly updating deployment detected! Deployment %s exceeds the max allowed generation of %d", deployment.Name, maxAllowedGeneration)
+			}
+		}
+	})
+}
+
 func EnsureNoPodsWithTooHighPriority(t *testing.T, ctx context.Context, client crclient.Client, hostedCluster *hyperv1.HostedCluster) {
 	// Priority of the etcd priority class, nothing should ever exceed this.
 	const maxAllowedPriority = 100002000