Skip to content

Commit

Permalink
Merge pull request kubernetes#41213 from wongma7/external-e2e-role
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue

Add e2e test for external provisioners

this is a retry of kubernetes#39545

This time around:
* take advantage of the system:persistent-volume-provisioner bootstrap cluster role to grant the external provisioner pod serviceaccount permissions
* add storageclass suffix so that the first and third test don't conflict when one creates the storageclass with the same name before the other

also tested more thoroughly myself on gce :)

@jsafrane if you would like to re-review
  • Loading branch information
Kubernetes Submit Queue authored Feb 10, 2017
2 parents 03bde62 + 97d2c70 commit 8fcbd56
Showing 1 changed file with 147 additions and 28 deletions.
175 changes: 147 additions & 28 deletions test/e2e/volume_provisioning.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ import (

"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/authentication/serviceaccount"
"k8s.io/kubernetes/pkg/api/v1"
rbacv1beta1 "k8s.io/kubernetes/pkg/apis/rbac/v1beta1"
storage "k8s.io/kubernetes/pkg/apis/storage/v1beta1"
storageutil "k8s.io/kubernetes/pkg/apis/storage/v1beta1/util"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
Expand All @@ -34,13 +37,11 @@ import (
const (
// Requested size of the volume
requestedSize = "1500Mi"
// Expected size of the volume is 2GiB, for "openstack", "gce", "aws", "gke", as they allocate volumes in 1GiB chunks
expectedSize = "2Gi"
// vsphere provider does not allocate volumes in 1GiB chunks, so setting expected size equal to requestedSize
vsphereExpectedSize = "1500Mi"
// Plugin name of the external provisioner
externalPluginName = "example.com/nfs"
)

func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim) {
func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVolumeClaim, expectedSize string) {
err := framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -55,9 +56,6 @@ func testDynamicProvisioning(client clientset.Interface, claim *v1.PersistentVol

// Check sizes
expectedCapacity := resource.MustParse(expectedSize)
if framework.ProviderIs("vsphere") {
expectedCapacity = resource.MustParse(vsphereExpectedSize)
}
pvCapacity := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)]
Expect(pvCapacity.Value()).To(Equal(expectedCapacity.Value()), "pvCapacity is not equal to expectedCapacity")

Expand Down Expand Up @@ -110,20 +108,28 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")

By("creating a StorageClass")
class := newStorageClass()
class := newStorageClass("", "internal")
_, err := c.Storage().StorageClasses().Create(class)
defer c.Storage().StorageClasses().Delete(class.Name, nil)
Expect(err).NotTo(HaveOccurred())

By("creating a claim with a dynamic provisioning annotation")
claim := newClaim(ns, false)
claim := newClaim(ns, "internal", false)
defer func() {
c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
}()
claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())

testDynamicProvisioning(c, claim)
if framework.ProviderIs("vsphere") {
// vsphere provider does not allocate volumes in 1GiB chunks, so setting expected size
// equal to requestedSize
testDynamicProvisioning(c, claim, requestedSize)
} else {
// Expected size of the volume is 2GiB, because the other three supported cloud
// providers allocate volumes in 1GiB chunks.
testDynamicProvisioning(c, claim, "2Gi")
}
})
})

Expand All @@ -132,19 +138,59 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")

By("creating a claim with an alpha dynamic provisioning annotation")
claim := newClaim(ns, true)
claim := newClaim(ns, "", true)
defer func() {
c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
}()
claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())

testDynamicProvisioning(c, claim)
if framework.ProviderIs("vsphere") {
testDynamicProvisioning(c, claim, requestedSize)
} else {
testDynamicProvisioning(c, claim, "2Gi")
}
})
})

framework.KubeDescribe("DynamicProvisioner External", func() {
It("should let an external dynamic provisioner create and delete persistent volumes [Slow]", func() {
// external dynamic provisioner pods need additional permissions provided by the
// persistent-volume-provisioner role
framework.BindClusterRole(c.Rbac(), "system:persistent-volume-provisioner", ns,
rbacv1beta1.Subject{Kind: rbacv1beta1.ServiceAccountKind, Namespace: ns, Name: "default"})

err := framework.WaitForAuthorizationUpdate(c.AuthorizationV1beta1(),
serviceaccount.MakeUsername(ns, "default"),
"", "get", schema.GroupResource{Group: "storage.k8s.io", Resource: "storageclasses"}, true)
framework.ExpectNoError(err, "Failed to update authorization: %v", err)

By("creating an external dynamic provisioner pod")
pod := startExternalProvisioner(c, ns)
defer c.Core().Pods(ns).Delete(pod.Name, nil)

By("creating a StorageClass")
class := newStorageClass(externalPluginName, "external")
_, err = c.Storage().StorageClasses().Create(class)
defer c.Storage().StorageClasses().Delete(class.Name, nil)
Expect(err).NotTo(HaveOccurred())

By("creating a claim with a dynamic provisioning annotation")
claim := newClaim(ns, "external", false)
defer func() {
c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
}()
claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
Expect(err).NotTo(HaveOccurred())

// Expected size of the externally provisioned volume depends on the external
// provisioner: for nfs-provisioner used here, it's equal to requested
testDynamicProvisioning(c, claim, requestedSize)
})
})
})

func newClaim(ns string, alpha bool) *v1.PersistentVolumeClaim {
func newClaim(ns, suffix string, alpha bool) *v1.PersistentVolumeClaim {
claim := v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "pvc-",
Expand All @@ -168,7 +214,7 @@ func newClaim(ns string, alpha bool) *v1.PersistentVolumeClaim {
}
} else {
claim.Annotations = map[string]string{
storageutil.StorageClassAnnotation: "fast",
storageutil.StorageClassAnnotation: "myclass-" + suffix,
}

}
Expand Down Expand Up @@ -223,27 +269,100 @@ func runInPodWithVolume(c clientset.Interface, ns, claimName, command string) {
framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
}

func newStorageClass() *storage.StorageClass {
var pluginName string

switch {
case framework.ProviderIs("gke"), framework.ProviderIs("gce"):
pluginName = "kubernetes.io/gce-pd"
case framework.ProviderIs("aws"):
pluginName = "kubernetes.io/aws-ebs"
case framework.ProviderIs("openstack"):
pluginName = "kubernetes.io/cinder"
case framework.ProviderIs("vsphere"):
pluginName = "kubernetes.io/vsphere-volume"
func newStorageClass(pluginName, suffix string) *storage.StorageClass {
if pluginName == "" {
switch {
case framework.ProviderIs("gke"), framework.ProviderIs("gce"):
pluginName = "kubernetes.io/gce-pd"
case framework.ProviderIs("aws"):
pluginName = "kubernetes.io/aws-ebs"
case framework.ProviderIs("openstack"):
pluginName = "kubernetes.io/cinder"
case framework.ProviderIs("vsphere"):
pluginName = "kubernetes.io/vsphere-volume"
}
}

return &storage.StorageClass{
TypeMeta: metav1.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: metav1.ObjectMeta{
Name: "fast",
Name: "myclass-" + suffix,
},
Provisioner: pluginName,
}
}

func startExternalProvisioner(c clientset.Interface, ns string) *v1.Pod {
podClient := c.Core().Pods(ns)

provisionerPod := &v1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
GenerateName: "external-provisioner-",
},

Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "nfs-provisioner",
Image: "quay.io/kubernetes_incubator/nfs-provisioner:v1.0.3",
SecurityContext: &v1.SecurityContext{
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"DAC_READ_SEARCH"},
},
},
Args: []string{
"-provisioner=" + externalPluginName,
"-grace-period=0",
},
Ports: []v1.ContainerPort{
{Name: "nfs", ContainerPort: 2049},
{Name: "mountd", ContainerPort: 20048},
{Name: "rpcbind", ContainerPort: 111},
{Name: "rpcbind-udp", ContainerPort: 111, Protocol: v1.ProtocolUDP},
},
Env: []v1.EnvVar{
{
Name: "POD_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
},
},
ImagePullPolicy: v1.PullIfNotPresent,
VolumeMounts: []v1.VolumeMount{
{
Name: "export-volume",
MountPath: "/export",
},
},
},
},
Volumes: []v1.Volume{
{
Name: "export-volume",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
},
},
}
provisionerPod, err := podClient.Create(provisionerPod)
framework.ExpectNoError(err, "Failed to create %s pod: %v", provisionerPod.Name, err)

framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, provisionerPod))

By("locating the provisioner pod")
pod, err := podClient.Get(provisionerPod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "Cannot locate the provisioner pod %v: %v", provisionerPod.Name, err)

return pod
}

0 comments on commit 8fcbd56

Please sign in to comment.