E2E: Add ServiceBGPStatus tests

Signed-off-by: Ori Braunshtein <obraunsh@redhat.com>
metallb · Jan 19, 2025 · 51bae2a · 51bae2a
1 parent 2d2c0fd
commit 51bae2a
Show file tree

Hide file tree

Showing 5 changed files with 242 additions and 5 deletions.
diff --git a/e2etest/bgptests/bgp.go b/e2etest/bgptests/bgp.go
@@ -23,9 +23,12 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sort"
 	"strings"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+
 	frrk8sv1beta1 "github.com/metallb/frr-k8s/api/v1beta1"
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -38,6 +41,7 @@ import (
 	"go.universe.tf/e2etest/pkg/k8sclient"
 	"go.universe.tf/e2etest/pkg/mac"
 	"go.universe.tf/e2etest/pkg/metallb"
+	"go.universe.tf/e2etest/pkg/status"
 	metallbv1beta1 "go.universe.tf/metallb/api/v1beta1"
 	metallbv1beta2 "go.universe.tf/metallb/api/v1beta2"
 
@@ -48,6 +52,7 @@ import (
 	"go.universe.tf/e2etest/pkg/ipfamily"
 	testservice "go.universe.tf/e2etest/pkg/service"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
@@ -600,6 +605,190 @@ var _ = ginkgo.Describe("BGP", func() {
 		ginkgo.Entry("IPV4 with Secret Ref set for BGPPeer CR", ipfamily.IPv4),
 		ginkgo.Entry("IPV6 with Secret Ref set for BGPPeer CR", ipfamily.IPv6))
 
+	ginkgo.DescribeTable("ServiceBGPStatus", func(ipFamily ipfamily.Family, poolAddresses []string, tweak testservice.Tweak) {
+		peers := metallb.PeersForContainers(FRRContainers, ipFamily)
+		sort.Slice(peers, func(i, j int) bool {
+			return peers[i].Name < peers[j].Name
+		})
+		peersNames := []string{}
+		for _, p := range peers {
+			peersNames = append(peersNames, p.Name)
+		}
+
+		bgpAdv := metallbv1beta1.BGPAdvertisement{
+			ObjectMeta: metav1.ObjectMeta{Name: "empty", Namespace: ConfigUpdater.Namespace()},
+			Spec:       metallbv1beta1.BGPAdvertisementSpec{},
+		}
+
+		ginkgo.By("Creating the service advertised to all peers")
+		resources := config.Resources{
+			Pools: []metallbv1beta1.IPAddressPool{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "bgp-test",
+					},
+					Spec: metallbv1beta1.IPAddressPoolSpec{
+						Addresses: poolAddresses,
+					},
+				},
+			},
+			Peers:   peers,
+			BGPAdvs: []metallbv1beta1.BGPAdvertisement{bgpAdv},
+		}
+
+		err := ConfigUpdater.Update(resources)
+		Expect(err).NotTo(HaveOccurred())
+
+		svc, _ := testservice.CreateWithBackend(cs, testNamespace, "external-local-lb", func(svc *corev1.Service) {
+			testservice.TrafficPolicyCluster(svc)
+			tweak(svc)
+		})
+		svcDeleted := false
+		defer func() {
+			if !svcDeleted {
+				testservice.Delete(cs, svc)
+			}
+		}()
+
+		for _, i := range svc.Status.LoadBalancer.Ingress {
+			ginkgo.By("validate LoadBalancer IP is in the AddressPool range")
+			ingressIP := jigservice.GetIngressPoint(&i)
+			err = config.ValidateIPInRange(resources.Pools, ingressIP)
+			Expect(err).NotTo(HaveOccurred())
+		}
+
+		nodes, err := cs.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+		Expect(err).ToNot(HaveOccurred())
+		nodesNames := []string{}
+		for _, n := range nodes.Items {
+			nodesNames = append(nodesNames, n.Name)
+		}
+
+		ginkgo.By("Verifying all nodes create a status for the service")
+		Eventually(func() error {
+			for _, n := range nodesNames {
+				s, err := status.BGPForServiceAndNode(ConfigUpdater.Client(), svc, n)
+				if err != nil {
+					return err
+				}
+
+				if !cmp.Equal(peersNames, s.Status.Peers) {
+					return fmt.Errorf("expected status peers to be %v, got %v for node %s\n diff: %s", peersNames, s.Status.Peers, n, cmp.Diff(peersNames, s.Status.Peers))
+				}
+			}
+
+			return nil
+		}, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred())
+
+		ginkgo.By("Adding a dummy peer to the adv")
+		err = ConfigUpdater.Client().Get(context.TODO(), types.NamespacedName{Namespace: bgpAdv.Namespace, Name: bgpAdv.Name}, &bgpAdv)
+		Expect(err).ToNot(HaveOccurred())
+		bgpAdv.Spec.Peers = append(peersNames, "dummy")
+		err = ConfigUpdater.Client().Update(context.TODO(), &bgpAdv)
+		Expect(err).ToNot(HaveOccurred())
+
+		Consistently(func() error {
+			for _, n := range nodesNames {
+				s, err := status.BGPForServiceAndNode(ConfigUpdater.Client(), svc, n)
+				if err != nil {
+					return err
+				}
+
+				if !cmp.Equal(peersNames, s.Status.Peers) {
+					return fmt.Errorf("expected status peers to be %v, got %v for node %s\n diff: %s", peersNames, s.Status.Peers, n, cmp.Diff(peersNames, s.Status.Peers))
+				}
+			}
+
+			return nil
+		}, 5*time.Second, 1*time.Second).ShouldNot(HaveOccurred(), "expected status peers to be the same as before after adding a dummy peer")
+
+		ginkgo.By("Removing the first peer")
+		peer0 := peers[0]
+		peer0.Namespace = ConfigUpdater.Namespace()
+		err = ConfigUpdater.Client().Delete(context.TODO(), &peer0)
+		Expect(err).ToNot(HaveOccurred())
+
+		expectedPeers := peersNames[1:]
+		expectDeletion := len(peers) == 1
+		Eventually(func() error {
+			for _, n := range nodesNames {
+				s, err := status.BGPForServiceAndNode(ConfigUpdater.Client(), svc, n)
+				if expectDeletion && !k8serrors.IsNotFound(err) {
+					return fmt.Errorf("expected status for node %s to be deleted, got %v with err %w", n, s, err)
+				}
+				if expectDeletion && k8serrors.IsNotFound(err) {
+					continue
+				}
+				if err != nil {
+					return err
+				}
+
+				if !cmp.Equal(expectedPeers, s.Status.Peers) {
+					return fmt.Errorf("expected status peers to be %v, got %v for node %s\n diff: %s", expectedPeers, s.Status.Peers, n, cmp.Diff(expectedPeers, s.Status.Peers))
+				}
+			}
+
+			return nil
+		}, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred())
+
+		ginkgo.By("Updating the node selector of the adv to not include the first node")
+		err = ConfigUpdater.Client().Get(context.TODO(), types.NamespacedName{Namespace: bgpAdv.Namespace, Name: bgpAdv.Name}, &bgpAdv)
+		Expect(err).ToNot(HaveOccurred())
+		bgpAdv.Spec.NodeSelectors = []metav1.LabelSelector{
+			{
+				MatchExpressions: []metav1.LabelSelectorRequirement{
+					{
+						Operator: "In",
+						Key:      "kubernetes.io/hostname",
+						Values:   nodesNames[1:],
+					},
+				}},
+		}
+		err = ConfigUpdater.Client().Update(context.TODO(), &bgpAdv)
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() error {
+			s, err := status.BGPForServiceAndNode(ConfigUpdater.Client(), svc, nodesNames[0])
+			if !k8serrors.IsNotFound(err) {
+				return fmt.Errorf("expected status for node %s to be deleted, got %v with err %w", nodesNames[0], s, err)
+			}
+
+			return nil
+		}, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred())
+
+		Eventually(func() error {
+			for _, n := range nodesNames[1:] {
+				s, err := status.BGPForServiceAndNode(ConfigUpdater.Client(), svc, n)
+				if err != nil {
+					return err
+				}
+
+				if !cmp.Equal(expectedPeers, s.Status.Peers) {
+					return fmt.Errorf("expected status peers to be %v, got %v for node %s\n diff: %s", expectedPeers, s.Status.Peers, n, cmp.Diff(expectedPeers, s.Status.Peers))
+				}
+			}
+
+			return nil
+		}, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred())
+
+		ginkgo.By("Validating the the statuses are deleted after deleting the service")
+		testservice.Delete(cs, svc)
+		svcDeleted = true
+		Eventually(func() error {
+			for _, n := range nodesNames {
+				s, err := status.BGPForServiceAndNode(ConfigUpdater.Client(), svc, n)
+				if !k8serrors.IsNotFound(err) {
+					return fmt.Errorf("expected status for node %s to be deleted, got %v with err %w", n, s, err)
+				}
+			}
+
+			return nil
+		}, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred())
+	},
+		ginkgo.Entry("IPV4", ipfamily.IPv4, []string{v4PoolAddresses}, func(_ *corev1.Service) {}),
+		ginkgo.Entry("IPV6", ipfamily.IPv6, []string{v6PoolAddresses}, func(_ *corev1.Service) {}),
+		ginkgo.Entry("DUALSTACK", ipfamily.DualStack, []string{v4PoolAddresses, v6PoolAddresses}, testservice.DualStack))
+
 	ginkgo.Context("BFD", func() {
 		ginkgo.DescribeTable("should work with the given bfd profile", func(bfd metallbv1beta1.BFDProfile, pairingFamily ipfamily.Family, poolAddresses []string, tweak testservice.Tweak) {
 			resources := config.Resources{

diff --git a/e2etest/pkg/k8s/reporter.go b/e2etest/pkg/k8s/reporter.go
@@ -63,6 +63,7 @@ func InitReporter(kubeconfig, path string, namespaces ...string) *k8sreporter.Ku
 		{Cr: &corev1.ServiceList{}},
 		{Cr: &frrk8sv1beta1.FRRConfigurationList{}},
 		{Cr: &frrk8sv1beta1.FRRNodeStateList{}},
+		{Cr: &metallbv1beta1.ServiceBGPStatusList{}},
 	}
 
 	reporter, err := k8sreporter.New(kubeconfig, addToScheme, dumpNamespace, path, crds...)

diff --git a/e2etest/pkg/status/bgpstatus.go b/e2etest/pkg/status/bgpstatus.go
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier:Apache-2.0
+
+package status
+
+import (
+	"context"
+	"fmt"
+
+	"go.universe.tf/e2etest/pkg/metallb"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"go.universe.tf/metallb/api/v1beta1"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Returns the ServiceBGPStatus resource for the given service and node.
+func BGPForServiceAndNode(cs client.Client, svc *v1.Service, node string) (*v1beta1.ServiceBGPStatus, error) {
+	statusList := v1beta1.ServiceBGPStatusList{}
+	err := cs.List(context.TODO(), &statusList,
+		client.InNamespace(metallb.Namespace),
+		client.MatchingLabels{
+			LabelServiceName:      svc.Name,
+			LabelServiceNamespace: svc.Namespace,
+			LabelAnnounceNode:     node,
+		},
+	)
+	svcKey := fmt.Sprintf("%s/%s", svc.Name, svc.Namespace)
+	if err != nil {
+		return nil, fmt.Errorf("could not get status for service %s on node %s, err: %w", svcKey, node, err)
+	}
+
+	if len(statusList.Items) == 0 {
+		return nil, errors.NewNotFound(schema.ParseGroupResource("ServiceBGPStatus.metallb.io"), svcKey)
+	}
+	if len(statusList.Items) > 1 {
+		return nil, fmt.Errorf("got more than 1 ServiceBGPStatus object for service %s node %s: %v", svcKey, node, statusList.Items)
+	}
+
+	return &statusList.Items[0], nil
+}
diff --git a/e2etest/pkg/status/l2status.go b/e2etest/pkg/status/l2status.go
@@ -5,6 +5,7 @@ package status
 import (
 	"context"
 	"fmt"
+
 	"go.universe.tf/e2etest/pkg/metallb"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -14,11 +15,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-const (
-	LabelServiceName      = "metallb.io/service-name"
-	LabelServiceNamespace = "metallb.io/service-namespace"
-)
-
 func L2ForService(cs client.Client, svc *v1.Service) (*v1beta1.ServiceL2Status, error) {
 	statusList := v1beta1.ServiceL2StatusList{}
 	err := cs.List(context.TODO(), &statusList,

diff --git a/e2etest/pkg/status/labels.go b/e2etest/pkg/status/labels.go
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier:Apache-2.0
+
+package status
+
+const (
+	LabelAnnounceNode     = "metallb.io/node"
+	LabelServiceName      = "metallb.io/service-name"
+	LabelServiceNamespace = "metallb.io/service-namespace"
+)