net: deflake lookup tests

The build dashboard is dotted with net test failures. We cannot declare all builders to have flaky networks, although all fundamentally do. Instead, add a simple retry/backoff loop to the ones that show up most commonly on the dashboard at this moment. If this approach works well in practice, we can incrementally apply it to other flaky net tests. Change-Id: I69c1ca6ce5b347ad549c7eb18d0438373f6e2489 Reviewed-on: https://go-review.googlesource.com/102397 Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
golang · Mar 25, 2018 · 5ce92d0 · 5ce92d0
1 parent 6f08b9f
commit 5ce92d0
Showing 1 changed file with 37 additions and 3 deletions.
diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go
@@ -60,6 +60,8 @@ var lookupGoogleSRVTests = []struct {
 	},
 }
 
+var backoffDuration = [...]time.Duration{time.Second, 5 * time.Second, 30 * time.Second}
+
 func TestLookupGoogleSRV(t *testing.T) {
 	if testenv.Builder() == "" {
 		testenv.MustHaveExternalNetwork(t)
@@ -69,10 +71,20 @@ func TestLookupGoogleSRV(t *testing.T) {
 		t.Skip("IPv4 is required")
 	}
 
-	for _, tt := range lookupGoogleSRVTests {
+	attempts := 0
+	for i := 0; i < len(lookupGoogleSRVTests); i++ {
+		tt := lookupGoogleSRVTests[i]
 		cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name)
 		if err != nil {
 			testenv.SkipFlakyNet(t)
+			if attempts < len(backoffDuration) {
+				dur := backoffDuration[attempts]
+				t.Logf("backoff %v after failure %v\n", dur, err)
+				time.Sleep(dur)
+				attempts++
+				i--
+				continue
+			}
 			t.Fatal(err)
 		}
 		if len(srvs) == 0 {
@@ -107,9 +119,20 @@ func TestLookupGmailMX(t *testing.T) {
 
 	defer dnsWaitGroup.Wait()
 
-	for _, tt := range lookupGmailMXTests {
+	attempts := 0
+	for i := 0; i < len(lookupGmailMXTests); i++ {
+		tt := lookupGmailMXTests[i]
 		mxs, err := LookupMX(tt.name)
 		if err != nil {
+			testenv.SkipFlakyNet(t)
+			if attempts < len(backoffDuration) {
+				dur := backoffDuration[attempts]
+				t.Logf("backoff %v after failure %v\n", dur, err)
+				time.Sleep(dur)
+				attempts++
+				i--
+				continue
+			}
 			t.Fatal(err)
 		}
 		if len(mxs) == 0 {
@@ -176,9 +199,20 @@ func TestLookupGmailTXT(t *testing.T) {
 
 	defer dnsWaitGroup.Wait()
 
-	for _, tt := range lookupGmailTXTTests {
+	attempts := 0
+	for i := 0; i < len(lookupGmailTXTTests); i++ {
+		tt := lookupGmailTXTTests[i]
 		txts, err := LookupTXT(tt.name)
 		if err != nil {
+			testenv.SkipFlakyNet(t)
+			if attempts < len(backoffDuration) {
+				dur := backoffDuration[attempts]
+				t.Logf("backoff %v after failure %v\n", dur, err)
+				time.Sleep(dur)
+				attempts++
+				i--
+				continue
+			}
 			t.Fatal(err)
 		}
 		if len(txts) == 0 {