Skip to content

Commit

Permalink
Merge pull request kubernetes#15104 from mikedanese/ds-e2e
Browse files Browse the repository at this point in the history 
break up daemonset test into two small tests to isolate flakiness
  • Loading branch information
@mikedanese
mikedanese committed Oct 6, 2015
2 parents 239790a + fe820fc commit 05ef8ed
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 120 deletions.
2 changes: 1 addition & 1 deletion hack/jenkins/e2e.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ AWS_REQUIRED_SKIP_TESTS=(
# -flaky- build variants.
GCE_FLAKY_TESTS=(
"DaemonRestart\sController\sManager"
"Daemon\sset\sshould\slaunch\sa\sdaemon\spod\son\severy\snode\sof\sthe\scluster"
"Daemon\sset\sshould"
"Resource\susage\sof\ssystem\scontainers"
"should\sbe\sable\sto\schange\sthe\stype\sand\snodeport\ssettings\sof\sa\sservice" # file: service.go, issue: #13032
"allows\sscheduling\sof\spods\son\sa\sminion\safter\sit\srejoins\sthe\scluster" # file: resize_nodes.go, issue: #13258
Expand Down
241 changes: 122 additions & 119 deletions test/e2e/daemon_set.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ import (
const (
// this should not be a multiple of 5, because node status updates
// every 5 seconds. See https://github.com/kubernetes/kubernetes/pull/14915.
updateRetryPeriod = 2 * time.Second
updateRetryTimeout = 30 * time.Second
dsRetryPeriod = 2 * time.Second
dsRetryTimeout = 1 * time.Minute

daemonsetLabelPrefix = "daemonset-"
daemonsetNameLabel = daemonsetLabelPrefix + "name"
daemonsetColorLabel = daemonsetLabelPrefix + "color"
Expand All @@ -49,9 +50,17 @@ const (
var _ = Describe("Daemon set", func() {
f := &Framework{BaseName: "daemonsets"}

image := "gcr.io/google_containers/serve_hostname:1.1"
dsName := "daemon-set"

var ns string
var c *client.Client

BeforeEach(func() {
f.beforeEach()
err := clearDaemonSetNodeLabels(f.Client)
ns = f.Namespace.Name
c = f.Client
err := clearDaemonSetNodeLabels(c)
Expect(err).NotTo(HaveOccurred())
})

Expand All @@ -61,8 +70,114 @@ var _ = Describe("Daemon set", func() {
f.afterEach()
})

It("should launch a daemon pod on every node of the cluster", func() {
testDaemonSets(f)
It("should run and stop simple daemon", func() {
label := map[string]string{daemonsetNameLabel: dsName}

Logf("Creating simple daemon set %s", dsName)
_, err := c.DaemonSets(ns).Create(&experimental.DaemonSet{
ObjectMeta: api.ObjectMeta{
Name: dsName,
},
Spec: experimental.DaemonSetSpec{
Template: &api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Labels: label,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: dsName,
Image: image,
Ports: []api.ContainerPort{{ContainerPort: 9376}},
},
},
},
},
},
})
Expect(err).NotTo(HaveOccurred())
defer func() {
Logf("Check that reaper kills all daemon pods for %s", dsName)
dsReaper, err := kubectl.ReaperFor("DaemonSet", c)
Expect(err).NotTo(HaveOccurred())
_, err = dsReaper.Stop(ns, dsName, 0, nil)
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, label))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to be reaped")
}()

By("Check that daemon pods launch on every node of the cluster.")
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, label))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to start")

By("Stop a daemon pod, check that the daemon pod is revived.")
podClient := c.Pods(ns)

podList, err := podClient.List(labels.Set(label).AsSelector(), fields.Everything())
Expect(err).NotTo(HaveOccurred())
Expect(len(podList.Items)).To(BeNumerically(">", 0))
pod := podList.Items[0]
err = podClient.Delete(pod.Name, nil)
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnAllNodes(f, label))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to revive")

})

It("should run and stop complex daemon", func() {
complexLabel := map[string]string{daemonsetNameLabel: dsName}
nodeSelector := map[string]string{daemonsetColorLabel: "blue"}
Logf("Creating daemon with a node selector %s", dsName)
_, err := c.DaemonSets(ns).Create(&experimental.DaemonSet{
ObjectMeta: api.ObjectMeta{
Name: dsName,
},
Spec: experimental.DaemonSetSpec{
Selector: complexLabel,
Template: &api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Labels: complexLabel,
},
Spec: api.PodSpec{
NodeSelector: nodeSelector,
Containers: []api.Container{
{
Name: dsName,
Image: image,
Ports: []api.ContainerPort{{ContainerPort: 9376}},
},
},
},
},
},
})
Expect(err).NotTo(HaveOccurred())

By("Initially, daemon pods should not be running on any nodes.")
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, complexLabel))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on no nodes")

By("Change label of node, check that daemon pod is launched.")
nodeClient := c.Nodes()
nodeList, err := nodeClient.List(labels.Everything(), fields.Everything())
Expect(len(nodeList.Items)).To(BeNumerically(">", 0))
newNode, err := setDaemonSetNodeLabels(c, nodeList.Items[0].Name, nodeSelector)
Expect(err).NotTo(HaveOccurred(), "error setting labels on node")
daemonSetLabels, _ := separateDaemonSetNodeLabels(newNode.Labels)
Expect(len(daemonSetLabels)).To(Equal(1))
err = wait.Poll(dsRetryPeriod, dsRetryTimeout, checkDaemonPodOnNodes(f, complexLabel, []string{newNode.Name}))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on new nodes")

By("remove the node selector and wait for daemons to be unscheduled")
_, err = setDaemonSetNodeLabels(c, nodeList.Items[0].Name, map[string]string{})
Expect(err).NotTo(HaveOccurred(), "error removing labels on node")
Expect(wait.Poll(dsRetryPeriod, dsRetryTimeout, checkRunningOnNoNodes(f, complexLabel))).
NotTo(HaveOccurred(), "error waiting for daemon pod to not be running on nodes")

By("We should now be able to delete the daemon set.")
Expect(c.DaemonSets(ns).Delete(dsName)).NotTo(HaveOccurred())

})
})

Expand Down Expand Up @@ -98,7 +213,7 @@ func setDaemonSetNodeLabels(c *client.Client, nodeName string, labels map[string
nodeClient := c.Nodes()
var newNode *api.Node
var newLabels map[string]string
err := wait.Poll(updateRetryPeriod, updateRetryTimeout, func() (bool, error) {
err := wait.Poll(dsRetryPeriod, dsRetryTimeout, func() (bool, error) {
node, err := nodeClient.Get(nodeName)
if err != nil {
return false, err
Expand Down Expand Up @@ -146,6 +261,7 @@ func checkDaemonPodOnNodes(f *Framework, selector map[string]string, nodeNames [
for _, pod := range pods {
nodesToPodCount[pod.Spec.NodeName] += 1
}
Logf("nodesToPodCount: %#v", nodesToPodCount)

// Ensure that exactly 1 pod is running on all nodes in nodeNames.
for _, nodeName := range nodeNames {
Expand Down Expand Up @@ -178,116 +294,3 @@ func checkRunningOnAllNodes(f *Framework, selector map[string]string) func() (bo
func checkRunningOnNoNodes(f *Framework, selector map[string]string) func() (bool, error) {
return checkDaemonPodOnNodes(f, selector, make([]string, 0))
}

func testDaemonSets(f *Framework) {
ns := f.Namespace.Name
c := f.Client
simpleDSName := "simple-daemon-set"
image := "gcr.io/google_containers/serve_hostname:1.1"
label := map[string]string{daemonsetNameLabel: simpleDSName}
retryTimeout := 1 * time.Minute
retryInterval := 5 * time.Second

Logf("Creating simple daemon set %s", simpleDSName)
_, err := c.DaemonSets(ns).Create(&experimental.DaemonSet{
ObjectMeta: api.ObjectMeta{
Name: simpleDSName,
},
Spec: experimental.DaemonSetSpec{
Template: &api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Labels: label,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: simpleDSName,
Image: image,
Ports: []api.ContainerPort{{ContainerPort: 9376}},
},
},
},
},
},
})
Expect(err).NotTo(HaveOccurred())
defer func() {
Logf("Check that reaper kills all daemon pods for %s", simpleDSName)
dsReaper, err := kubectl.ReaperFor("DaemonSet", c)
Expect(err).NotTo(HaveOccurred())
_, err = dsReaper.Stop(ns, simpleDSName, 0, nil)
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(retryInterval, retryTimeout, checkRunningOnNoNodes(f, label))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to be reaped")
}()

By("Check that daemon pods launch on every node of the cluster.")
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(retryInterval, retryTimeout, checkRunningOnAllNodes(f, label))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to start")

By("Stop a daemon pod, check that the daemon pod is revived.")
podClient := c.Pods(ns)

podList, err := podClient.List(labels.Set(label).AsSelector(), fields.Everything())
Expect(err).NotTo(HaveOccurred())
Expect(len(podList.Items)).To(BeNumerically(">", 0))
pod := podList.Items[0]
err = podClient.Delete(pod.Name, nil)
Expect(err).NotTo(HaveOccurred())
err = wait.Poll(retryInterval, retryTimeout, checkRunningOnAllNodes(f, label))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pod to revive")

complexDSName := "complex-daemon-set"
complexLabel := map[string]string{daemonsetNameLabel: complexDSName}
nodeSelector := map[string]string{daemonsetColorLabel: "blue"}
Logf("Creating daemon with a node selector %s", complexDSName)
_, err = c.DaemonSets(ns).Create(&experimental.DaemonSet{
ObjectMeta: api.ObjectMeta{
Name: complexDSName,
},
Spec: experimental.DaemonSetSpec{
Selector: complexLabel,
Template: &api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Labels: complexLabel,
},
Spec: api.PodSpec{
NodeSelector: nodeSelector,
Containers: []api.Container{
{
Name: complexDSName,
Image: image,
Ports: []api.ContainerPort{{ContainerPort: 9376}},
},
},
},
},
},
})
Expect(err).NotTo(HaveOccurred())

By("Initially, daemon pods should not be running on any nodes.")
err = wait.Poll(retryInterval, retryTimeout, checkRunningOnNoNodes(f, complexLabel))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on no nodes")

By("Change label of node, check that daemon pod is launched.")
nodeClient := c.Nodes()
nodeList, err := nodeClient.List(labels.Everything(), fields.Everything())
Expect(len(nodeList.Items)).To(BeNumerically(">", 0))
newNode, err := setDaemonSetNodeLabels(c, nodeList.Items[0].Name, nodeSelector)
Expect(err).NotTo(HaveOccurred(), "error setting labels on node")
daemonSetLabels, _ := separateDaemonSetNodeLabels(newNode.Labels)
Expect(len(daemonSetLabels)).To(Equal(1))
err = wait.Poll(retryInterval, retryTimeout, checkDaemonPodOnNodes(f, complexLabel, []string{newNode.Name}))
Expect(err).NotTo(HaveOccurred(), "error waiting for daemon pods to be running on new nodes")

By("remove the node selector and wait for daemons to be unscheduled")
_, err = setDaemonSetNodeLabels(c, nodeList.Items[0].Name, map[string]string{})
Expect(err).NotTo(HaveOccurred(), "error removing labels on node")
Expect(wait.Poll(retryInterval, retryTimeout, checkRunningOnNoNodes(f, complexLabel))).
NotTo(HaveOccurred(), "error waiting for daemon pod to not be running on nodes")

By("We should now be able to delete the daemon set.")
Expect(c.DaemonSets(ns).Delete(complexDSName)).NotTo(HaveOccurred())
}

0 comments on commit 05ef8ed

Please sign in to comment.