Increase the timeout on apiserver requests to fetch node stats.

satnam6502 · Feb 23, 2015 · e114718 · e114718
1 parent e186582
commit e114718
Showing 1 changed file with 15 additions and 3 deletions.
diff --git a/test/e2e/cadvisor.go b/test/e2e/cadvisor.go
@@ -25,6 +25,12 @@ import (
 	. "github.com/onsi/ginkgo"
 )
 
+const (
+	timeout       = 1 * time.Minute
+	maxRetries    = 10
+	sleepDuration = time.Minute
+)
+
 var _ = Describe("Cadvisor", func() {
 	var c *client.Client
 
@@ -44,21 +50,27 @@ func CheckCadvisorHealthOnAllNodes(c *client.Client, timeout time.Duration) {
 	nodeList, err := c.Nodes().List()
 	expectNoError(err)
 	var errors []error
-	for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
+	retries := maxRetries
+	for {
 		errors = []error{}
 		for _, node := range nodeList.Items {
 			// cadvisor is not accessible directly unless its port (4194 by default) is exposed.
 			// Here, we access '/stats/' REST endpoint on the kubelet which polls cadvisor internally.
 			statsResource := fmt.Sprintf("api/v1beta1/proxy/minions/%s/stats/", node.Name)
 			By(fmt.Sprintf("Querying stats from node %s using url %s", node.Name, statsResource))
-			_, err = c.Get().AbsPath(statsResource).Timeout(1 * time.Second).Do().Raw()
+			_, err = c.Get().AbsPath(statsResource).Timeout(timeout).Do().Raw()
 			if err != nil {
 				errors = append(errors, err)
 			}
 		}
 		if len(errors) == 0 {
 			return
 		}
+		if retries--; retries <= 0 {
+			break
+		}
+		Logf("failed to retrieve kubelet stats -\n %v", errors)
+		time.Sleep(sleepDuration)
 	}
-	Failf("Timed out after %v waiting for cadvisor to be healthy on all nodes. Errors:\n%v", timeout, errors)
+	Failf("Failed after retrying %d times for cadvisor to be healthy on all nodes. Errors:\n%v", maxRetries, errors)
 }