Merge pull request #36009 from rkouj/GCE-PD-test

Automatic merge from submit-queue Add test to detach a pd whose node was deleted **What this PR does / why we need it**: A test for the following issue : If a node with a GCE PD attached is deleted (before the volume is detached), subsequent attempts by the attach/detach controller to detach it should not fail. **Bonus** :Added additional code to ensure that the pd can still be attached to a different node. Edit : Removed it as it was making the test much slower. #29358
kubernetes · Dec 20, 2016 · b3e5725 · b3e5725
2 parents aa86fca + f67b495
commit b3e5725
Show file tree

Hide file tree

Showing 3 changed files with 79 additions and 4 deletions.
diff --git a/test/e2e/pd.go b/test/e2e/pd.go
@@ -19,6 +19,7 @@ package e2e
 import (
 	"fmt"
 	mathrand "math/rand"
+	"os/exec"
 	"strings"
 	"time"
 
@@ -420,6 +421,61 @@ var _ = framework.KubeDescribe("Pod Disks", func() {
 		waitForPDDetach(disk1Name, host0Name)
 		waitForPDDetach(disk2Name, host0Name)
 	})
+
+	It("should be able to detach from a node which was deleted [Slow] [Disruptive]", func() {
+		framework.SkipUnlessProviderIs("gce")
+
+		initialGroupSize, err := GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup)
+		framework.ExpectNoError(err, "Error getting group size")
+
+		By("Creating a pd")
+		diskName, err := createPDWithRetry()
+		framework.ExpectNoError(err, "Error creating a pd")
+
+		host0Pod := testPDPod([]string{diskName}, host0Name, false, 1)
+
+		containerName := "mycontainer"
+
+		defer func() {
+			By("Cleaning up PD-RW test env")
+			podClient.Delete(host0Pod.Name, v1.NewDeleteOptions(0))
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name})
+		}()
+
+		By("submitting host0Pod to kubernetes")
+		_, err = podClient.Create(host0Pod)
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0pod: %v", err))
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name))
+
+		testFile := "/testpd1/tracker"
+		testFileContents := fmt.Sprintf("%v", mathrand.Int())
+
+		framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFile, testFileContents))
+		framework.Logf("Wrote value: %v", testFileContents)
+
+		// Verify that disk shows up in node 0's volumeInUse list
+		framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, true /* should exist*/))
+
+		output, err := exec.Command("gcloud", "compute", "instances", "list").CombinedOutput()
+		framework.ExpectNoError(err, fmt.Sprintf("Unable to get list of node instances %v", err))
+		Expect(true, strings.Contains(string(output), string(host0Name)))
+
+		By("deleting host0")
+
+		output, err = exec.Command("gcloud", "compute", "instances", "delete", string(host0Name), "--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput()
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to delete host0pod: %v", err))
+
+		output, err = exec.Command("gcloud", "compute", "instances", "list").CombinedOutput()
+		framework.ExpectNoError(err, fmt.Sprintf("Unable to get list of node instances %v", err))
+		Expect(false, strings.Contains(string(output), string(host0Name)))
+
+		// The disk should be detached from host0 on it's deletion
+		By("Waiting for pd to detach from host0")
+		waitForPDDetach(diskName, host0Name)
+		framework.ExpectNoError(WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get back the cluster to inital size")
+		return
+	})
 })
 
 func createPDWithRetry() (string, error) {

diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go
@@ -120,8 +120,8 @@ func GroupSize(group string) (int, error) {
 }
 
 func WaitForGroupSize(group string, size int32) error {
-	timeout := 10 * time.Minute
-	for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
+	timeout := 30 * time.Minute
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(20 * time.Second) {
 		currentSize, err := GroupSize(group)
 		if err != nil {
 			framework.Logf("Failed to get node instance group size: %v", err)

diff --git a/test/test_owners.csv b/test/test_owners.csv
@@ -46,7 +46,7 @@ CronJob should not schedule jobs when suspended,soltysh,1
 CronJob should not schedule new jobs when ForbidConcurrent,soltysh,1
 CronJob should replace jobs when ReplaceConcurrent,soltysh,1
 CronJob should schedule multiple jobs concurrently,soltysh,1
-DNS config map should be able to change configuration,derekwaynecarr,0
+DNS config map should be able to change configuration,rkouj,0
 DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios,MrHohn,0
 DNS horizontal autoscaling kube-dns-autoscaler should scale kube-dns pods when cluster size changed,MrHohn,0
 DNS should provide DNS for ExternalName services,rmmh,1
@@ -264,6 +264,8 @@ Kubelet Container Manager Validate OOM score adjustments once the node is setup
 Kubelet Container Manager Validate OOM score adjustments once the node is setup pod infra containers oom-score-adj should be -998 and best effort container's should be 1000,timothysc,1
 Kubelet Eviction Manager hard eviction test pod using the most disk space gets evicted when the node disk usage is above the eviction hard threshold should evict the pod using the most disk space,rrati,0
 Kubelet Volume Manager Volume Manager On terminatation of pod with memory backed volume should remove the volume from the node,derekwaynecarr,0
+Kubelet Eviction Manager hard eviction test pod using the most disk space gets evicted when the node disk usage is above the eviction hard threshold should evict the pod using the most disk space,karlkfi,1
+Kubelet Volume Manager Volume Manager On terminatation of pod with memory backed volume should remove the volume from the node,rkouj,0
 Kubelet experimental resource usage tracking resource tracking for * pods per node,yujuhong,0
 Kubelet regular resource usage tracking resource tracking for * pods per node,yujuhong,0
 Kubelet when scheduling a busybox command in a pod it should print the output to logs,ixdy,1
@@ -342,6 +344,9 @@ PersistentVolumes PersistentVolumes:NFS with multiple PVs and PVCs all in same n
 Pet Store should scale to persist a nominal number ( * ) of transactions in * seconds,xiang90,1
 "Pod Disks Should schedule a pod w/ a RW PD, gracefully remove it, then schedule it on another host",alex-mohr,1
 "Pod Disks Should schedule a pod w/ a readonly PD on two hosts, then remove both gracefully.",rrati,0
+"Pod Disks Should schedule a pod w/ a RW PD, gracefully remove it, then schedule it on another host",saad-ali,0
+"Pod Disks Should schedule a pod w/ a readonly PD on two hosts, then remove both gracefully.",saad-ali,0
+Pod Disks should be able to detach from a node which was deleted,rkouj,0
 "Pod Disks should schedule a pod w/ a RW PD shared between multiple containers, write to PD, delete pod, verify contents, and repeat in rapid succession",saad-ali,0
 "Pod Disks should schedule a pod w/ a RW PD, ungracefully remove it, then schedule it on another host",mml,1
 "Pod Disks should schedule a pod w/ a readonly PD on two hosts, then remove both ungracefully.",saad-ali,1
@@ -468,8 +473,11 @@ StatefulSet Basic StatefulSet functionality Scaling should happen in predictable
 StatefulSet Basic StatefulSet functionality Should recreate evicted statefulset,rrati,0
 StatefulSet Basic StatefulSet functionality should allow template updates,derekwaynecarr,0
 StatefulSet Basic StatefulSet functionality should handle healthy stateful pod restarts during scale,kevin-wangzefeng,1
+StatefulSet Basic StatefulSet functionality Scaling should happen in predictable order and halt if any pet is unhealthy,rkouj,0
+StatefulSet Basic StatefulSet functionality should allow template updates,rkouj,0
+StatefulSet Basic StatefulSet functionality should handle healthy pet restarts during scale,girishkalele,1
 StatefulSet Basic StatefulSet functionality should provide basic identity,bprashanth,1
-StatefulSet Deploy clustered applications should creating a working CockroachDB cluster,derekwaynecarr,0
+StatefulSet Deploy clustered applications should creating a working CockroachDB cluster,rkouj,0
 StatefulSet Deploy clustered applications should creating a working mysql cluster,yujuhong,1
 StatefulSet Deploy clustered applications should creating a working redis cluster,yifan-gu,1
 StatefulSet Deploy clustered applications should creating a working zookeeper cluster,pmorie,1
@@ -555,6 +563,13 @@ k8s.io/kubernetes/pkg/api/v1,rrati,0
 k8s.io/kubernetes/pkg/api/v1/endpoints,madhusudancs,0
 k8s.io/kubernetes/pkg/api/v1/pod,madhusudancs,0
 k8s.io/kubernetes/pkg/api/v1/service,madhusudancs,0
+k8s.io/kubernetes/pkg/api/unversioned,kevin-wangzefeng,1
+k8s.io/kubernetes/pkg/api/unversioned/validation,brendandburns,1
+k8s.io/kubernetes/pkg/api/util,ghodss,1
+k8s.io/kubernetes/pkg/api/v1,vulpecula,1
+k8s.io/kubernetes/pkg/api/v1/endpoints,rkouj,0
+k8s.io/kubernetes/pkg/api/v1/pod,rkouj,0
+k8s.io/kubernetes/pkg/api/v1/service,rkouj,0
 k8s.io/kubernetes/pkg/api/validation,smarterclayton,1
 k8s.io/kubernetes/pkg/api/validation/path,luxas,1
 k8s.io/kubernetes/pkg/apimachinery,gmarek,1
@@ -649,6 +664,9 @@ k8s.io/kubernetes/pkg/credentialprovider/gcp,mml,1
 k8s.io/kubernetes/pkg/dns,rrati,0
 k8s.io/kubernetes/pkg/dns/config,derekwaynecarr,0
 k8s.io/kubernetes/pkg/dns/federation,derekwaynecarr,0
+k8s.io/kubernetes/pkg/dns,jdef,1
+k8s.io/kubernetes/pkg/dns/config,rkouj,0
+k8s.io/kubernetes/pkg/dns/federation,rkouj,0
 k8s.io/kubernetes/pkg/dns/treecache,bowei,0
 k8s.io/kubernetes/pkg/fieldpath,childsb,1
 k8s.io/kubernetes/pkg/fields,jsafrane,1
@@ -791,6 +809,7 @@ k8s.io/kubernetes/pkg/registry/storage/storageclass,brendandburns,1
 k8s.io/kubernetes/pkg/registry/storage/storageclass/etcd,eparis,1
 k8s.io/kubernetes/pkg/runtime,wojtek-t,0
 k8s.io/kubernetes/pkg/runtime/schema,madhusudancs,0
+k8s.io/kubernetes/pkg/runtime/schema,rkouj,0
 k8s.io/kubernetes/pkg/runtime/serializer,wojtek-t,0
 k8s.io/kubernetes/pkg/runtime/serializer/json,wojtek-t,0
 k8s.io/kubernetes/pkg/runtime/serializer/protobuf,wojtek-t,0