From bb50086b8ffb4289e085fce637a5cd01f8ef4296 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Wed, 6 Sep 2017 11:36:27 -0700 Subject: [PATCH] e2e: network tiers should retry on 404 errors The feature is still Alpha and at times, the IP address previously used by the load balancer in the test will not completely freed even after the load balancer is long gone. In this case, the test URL with the IP would return a 404 response. Tolerate this error and retry until the new load balancer is fully established. --- test/e2e/framework/networking_utils.go | 14 ++++++++++++++ test/e2e/framework/service_util.go | 8 +++++++- test/e2e/network/network_tiers.go | 6 +++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/test/e2e/framework/networking_utils.go b/test/e2e/framework/networking_utils.go index 0914c0641b03d..7bbda2961bf3e 100644 --- a/test/e2e/framework/networking_utils.go +++ b/test/e2e/framework/networking_utils.go @@ -650,11 +650,20 @@ func TestReachableHTTP(ip string, port int, request string, expect string) (bool return TestReachableHTTPWithContent(ip, port, request, expect, nil) } +func TestReachableHTTPWithRetriableErrorCodes(ip string, port int, request string, expect string, retriableErrCodes []int) (bool, error) { + return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, nil, retriableErrCodes, time.Second*5) +} + func TestReachableHTTPWithContent(ip string, port int, request string, expect string, content *bytes.Buffer) (bool, error) { return TestReachableHTTPWithContentTimeout(ip, port, request, expect, content, 5*time.Second) } func TestReachableHTTPWithContentTimeout(ip string, port int, request string, expect string, content *bytes.Buffer, timeout time.Duration) (bool, error) { + return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, content, []int{}, timeout) +} + +func TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip string, port int, request string, expect string, content *bytes.Buffer, retriableErrCodes []int, timeout time.Duration) (bool, error) { + url := fmt.Sprintf("http://%s:%d%s", ip, port, request) if ip == "" { Failf("Got empty IP for reachability check (%s)", url) @@ -679,6 +688,11 @@ func TestReachableHTTPWithContentTimeout(ip string, port int, request string, ex return false, nil } if resp.StatusCode != 200 { + for _, code := range retriableErrCodes { + if resp.StatusCode == code { + return false, nil + } + } return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s", resp.Status, url, string(body)) } diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index d8ea2891f5c06..8a614ca9deaab 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -832,7 +832,13 @@ func (j *ServiceTestJig) LaunchEchoserverPodOnNode(f *Framework, nodeName, podNa } func (j *ServiceTestJig) TestReachableHTTP(host string, port int, timeout time.Duration) { - if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestReachableHTTP(host, port, "/echo?msg=hello", "hello") }); err != nil { + j.TestReachableHTTPWithRetriableErrorCodes(host, port, []int{}, timeout) +} + +func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, port int, retriableErrCodes []int, timeout time.Duration) { + if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { + return TestReachableHTTPWithRetriableErrorCodes(host, port, "/echo?msg=hello", "hello", retriableErrCodes) + }); err != nil { Failf("Could not reach HTTP service through %v:%v after %v: %v", host, port, timeout, err) } } diff --git a/test/e2e/network/network_tiers.go b/test/e2e/network/network_tiers.go index a955358fa0adb..d3a14aab221da 100644 --- a/test/e2e/network/network_tiers.go +++ b/test/e2e/network/network_tiers.go @@ -18,6 +18,7 @@ package network import ( "fmt" + "net/http" "time" computealpha "google.golang.org/api/compute/v0.alpha" @@ -153,7 +154,10 @@ func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existin Expect(ingressIP).To(Equal(svc.Spec.LoadBalancerIP)) } jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer) - jig.TestReachableHTTP(ingressIP, svcPort, checkTimeout) + // If the IP has been used by previous test, sometimes we get the lingering + // 404 errors even after the LB is long gone. Tolerate and retry until the + // the new LB is fully established since this feature is still Alpha in GCP. + jig.TestReachableHTTPWithRetriableErrorCodes(ingressIP, svcPort, []int{http.StatusNotFound}, checkTimeout) // Verify the network tier matches the desired. svcNetTier, err := gcecloud.GetServiceNetworkTier(svc)