Merge pull request kubernetes#15228 from mesosphere/sttts-conformance…

…-tags Use [Conformance] to tag conformance tests
mikedanese · Oct 10, 2015 · b635fc5 · b635fc5
2 parents bbe4266 + cb00df9
commit b635fc5
Show file tree

Hide file tree

Showing 24 changed files with 86 additions and 110 deletions.
diff --git a/hack/conformance-test.sh b/hack/conformance-test.sh
@@ -74,38 +74,14 @@ git show-ref | grep $HEAD_SHA | grep refs/tags
 echo
 echo "Conformance test checking conformance with Kubernetes version 1.0"
 
-# It runs a whitelist of tests.  This whitelist was assembled at commit
-# b70b7084c93d4ce80b7463f48c23d5ac04edb2b1 starting from this list of tests:
-#   grep -h 'It(\|Describe(' -R test
+# It runs a whitelist of tests: all tests which are flagged with [Conformance]
+# somewhere in the description (i.e. either in the Describe part or the It part).
+# The list of tagged conformance tests can be retrieved by:
 #
-# List of test name patterns not included and why not included:
-#  Cadvisor: impl detail how stats gotten from containers.
-#  MasterCerts: GKE/GCE specific
-#  Density: performance
-#  Cluster level logging...: optional feature
-#  Etcd failure: reliability
-#  Load Capacity: performance
-#  Monitoring: optional feature.
-#  Namespaces.*seconds: performance.
-#  Pod disks: uses GCE specific feature.
-#  Reboot: node management
-#  Nodes: node management.
-#  Restart: node management.
-#  Scale: performance
-#  Services.*load balancer: not all cloud providers have a load balancer.
-#  Services.*NodePort: requires you to open the firewall yourself, so not covered.
-#  Services.*nodeport: requires you to open the firewall yourself, so not covered.
-#  Shell: replies on optional ssh access to nodes.
-#  SSH: optional feature.
-#  Addon\supdate: requires SSH
-#  Volumes: contained only skipped tests.
-#  Clean\sup\spods\son\snode: performance
-#  MaxPods\slimit\snumber\sof\spods: not sure why this wasn't working on GCE but it wasn't.
-#  Kubectl\sclient\sSimple\spod: not sure why this wasn't working on GCE but it wasn't
-#  DNS: not sure why this wasn't working on GCE but it wasn't
-export CONFORMANCE_TEST_SKIP_REGEX="Cadvisor|MasterCerts|Density|Cluster\slevel\slogging|Etcd\sfailure|Load\sCapacity|Monitoring|Namespaces.*seconds|Pod\sdisks|Reboot|Restart|Nodes|Scale|Services.*load\sbalancer|Services.*NodePort|Services.*nodeport|Shell|SSH|Addon\supdate|Volumes|Clean\sup\spods\son\snode|Skipped|skipped|MaxPods\slimit\snumber\sof\spods|Kubectl\sclient\sSimple\spod|DNS|Resource\susage\sof\ssystem\scontainers"
+# NUM_MINIONS=4 KUBERNETES_CONFORMANCE_TEST="y" \
+# hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]' -ginkgo.dryRun=true
 
 declare -x KUBERNETES_CONFORMANCE_TEST="y"
 declare -x NUM_MINIONS=4
-hack/ginkgo-e2e.sh
+hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]'
 exit $?
diff --git a/test/e2e/container_probe.go b/test/e2e/container_probe.go
@@ -40,7 +40,7 @@ var _ = Describe("Probing container", func() {
 
 	AfterEach(framework.afterEach)
 
-	It("with readiness probe should not be ready before initial delay and never restart", func() {
+	It("with readiness probe should not be ready before initial delay and never restart [Conformance]", func() {
 		p, err := podClient.Create(makePodSpec(probe.withInitialDelay().build(), nil))
 		expectNoError(err)
 		startTime := time.Now()
@@ -73,7 +73,7 @@ var _ = Describe("Probing container", func() {
 		Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount)
 	})
 
-	It("with readiness probe that fails should never be ready and never restart", func() {
+	It("with readiness probe that fails should never be ready and never restart [Conformance]", func() {
 		p, err := podClient.Create(makePodSpec(probe.withFailing().build(), nil))
 		expectNoError(err)
 

diff --git a/test/e2e/docker_containers.go b/test/e2e/docker_containers.go
@@ -46,13 +46,13 @@ var _ = Describe("Docker Containers", func() {
 		}
 	})
 
-	It("should use the image defaults if command and args are blank", func() {
+	It("should use the image defaults if command and args are blank [Conformance]", func() {
 		testContainerOutputInNamespace("use defaults", c, entrypointTestPod(), 0, []string{
 			"[/ep default arguments]",
 		}, ns)
 	})
 
-	It("should be able to override the image's default arguments (docker cmd)", func() {
+	It("should be able to override the image's default arguments (docker cmd) [Conformance]", func() {
 		pod := entrypointTestPod()
 		pod.Spec.Containers[0].Args = []string{"override", "arguments"}
 
@@ -63,7 +63,7 @@ var _ = Describe("Docker Containers", func() {
 
 	// Note: when you override the entrypoint, the image's arguments (docker cmd)
 	// are ignored.
-	It("should be able to override the image's default commmand (docker entrypoint)", func() {
+	It("should be able to override the image's default commmand (docker entrypoint) [Conformance]", func() {
 		pod := entrypointTestPod()
 		pod.Spec.Containers[0].Command = []string{"/ep-2"}
 
@@ -72,7 +72,7 @@ var _ = Describe("Docker Containers", func() {
 		}, ns)
 	})
 
-	It("should be able to override the image's default command and arguments", func() {
+	It("should be able to override the image's default command and arguments [Conformance]", func() {
 		pod := entrypointTestPod()
 		pod.Spec.Containers[0].Command = []string{"/ep-2"}
 		pod.Spec.Containers[0].Args = []string{"override", "arguments"}

diff --git a/test/e2e/downward_api.go b/test/e2e/downward_api.go
@@ -28,7 +28,7 @@ import (
 var _ = Describe("Downward API", func() {
 	framework := NewFramework("downward-api")
 
-	It("should provide pod name and namespace as env vars", func() {
+	It("should provide pod name and namespace as env vars [Conformance]", func() {
 		podName := "downward-api-" + string(util.NewUUID())
 		pod := &api.Pod{
 			ObjectMeta: api.ObjectMeta{

diff --git a/test/e2e/downwardapi_volume.go b/test/e2e/downwardapi_volume.go
@@ -28,7 +28,7 @@ import (
 var _ = Describe("Downward API volume", func() {
 	f := NewFramework("downward-api")
 
-	It("should provide labels and annotations files", func() {
+	It("should provide labels and annotations files [Conformance]", func() {
 		podName := "metadata-volume-" + string(util.NewUUID())
 		pod := &api.Pod{
 			ObjectMeta: api.ObjectMeta{

diff --git a/test/e2e/empty_dir.go b/test/e2e/empty_dir.go
@@ -37,59 +37,59 @@ var _ = Describe("EmptyDir volumes", func() {
 
 	f := NewFramework("emptydir")
 
-	It("volume on tmpfs should have the correct mode", func() {
+	It("volume on tmpfs should have the correct mode [Conformance]", func() {
 		doTestVolumeMode(f, testImageRootUid, api.StorageMediumMemory)
 	})
 
-	It("should support (root,0644,tmpfs)", func() {
+	It("should support (root,0644,tmpfs) [Conformance]", func() {
 		doTest0644(f, testImageRootUid, api.StorageMediumMemory)
 	})
 
-	It("should support (root,0666,tmpfs)", func() {
+	It("should support (root,0666,tmpfs) [Conformance]", func() {
 		doTest0666(f, testImageRootUid, api.StorageMediumMemory)
 	})
 
-	It("should support (root,0777,tmpfs)", func() {
+	It("should support (root,0777,tmpfs) [Conformance]", func() {
 		doTest0777(f, testImageRootUid, api.StorageMediumMemory)
 	})
 
-	It("should support (non-root,0644,tmpfs)", func() {
+	It("should support (non-root,0644,tmpfs) [Conformance]", func() {
 		doTest0644(f, testImageNonRootUid, api.StorageMediumMemory)
 	})
 
-	It("should support (non-root,0666,tmpfs)", func() {
+	It("should support (non-root,0666,tmpfs) [Conformance]", func() {
 		doTest0666(f, testImageNonRootUid, api.StorageMediumMemory)
 	})
 
-	It("should support (non-root,0777,tmpfs)", func() {
+	It("should support (non-root,0777,tmpfs) [Conformance]", func() {
 		doTest0777(f, testImageNonRootUid, api.StorageMediumMemory)
 	})
 
-	It("volume on default medium should have the correct mode", func() {
+	It("volume on default medium should have the correct mode [Conformance]", func() {
 		doTestVolumeMode(f, testImageRootUid, api.StorageMediumDefault)
 	})
 
-	It("should support (root,0644,default)", func() {
+	It("should support (root,0644,default) [Conformance]", func() {
 		doTest0644(f, testImageRootUid, api.StorageMediumDefault)
 	})
 
-	It("should support (root,0666,default)", func() {
+	It("should support (root,0666,default) [Conformance]", func() {
 		doTest0666(f, testImageRootUid, api.StorageMediumDefault)
 	})
 
-	It("should support (root,0777,default)", func() {
+	It("should support (root,0777,default) [Conformance]", func() {
 		doTest0777(f, testImageRootUid, api.StorageMediumDefault)
 	})
 
-	It("should support (non-root,0644,default)", func() {
+	It("should support (non-root,0644,default) [Conformance]", func() {
 		doTest0644(f, testImageNonRootUid, api.StorageMediumDefault)
 	})
 
-	It("should support (non-root,0666,default)", func() {
+	It("should support (non-root,0666,default) [Conformance]", func() {
 		doTest0666(f, testImageNonRootUid, api.StorageMediumDefault)
 	})
 
-	It("should support (non-root,0777,default)", func() {
+	It("should support (non-root,0777,default) [Conformance]", func() {
 		doTest0777(f, testImageNonRootUid, api.StorageMediumDefault)
 	})
 })

diff --git a/test/e2e/events.go b/test/e2e/events.go
@@ -34,7 +34,7 @@ import (
 var _ = Describe("Events", func() {
 	framework := NewFramework("events")
 
-	It("should be sent by kubelets and the scheduler about pods scheduling and running", func() {
+	It("should be sent by kubelets and the scheduler about pods scheduling and running [Conformance]", func() {
 
 		podClient := framework.Client.Pods(framework.Namespace.Name)
 

diff --git a/test/e2e/examples.go b/test/e2e/examples.go
@@ -449,7 +449,7 @@ var _ = Describe("Examples e2e", func() {
 	})
 
 	Describe("[Example]ClusterDns", func() {
-		It("should create pod that uses dns", func() {
+		It("should create pod that uses dns [Conformance]", func() {
 			mkpath := func(file string) string {
 				return filepath.Join(testContext.RepoRoot, "examples/cluster-dns", file)
 			}

diff --git a/test/e2e/expansion.go b/test/e2e/expansion.go
@@ -26,7 +26,7 @@ import (
 var _ = Describe("Variable Expansion", func() {
 	framework := NewFramework("var-expansion")
 
-	It("should allow composing env vars into new env vars", func() {
+	It("should allow composing env vars into new env vars [Conformance]", func() {
 		podName := "var-expansion-" + string(util.NewUUID())
 		pod := &api.Pod{
 			ObjectMeta: api.ObjectMeta{
@@ -66,7 +66,7 @@ var _ = Describe("Variable Expansion", func() {
 		})
 	})
 
-	It("should allow substituting values in a container's command", func() {
+	It("should allow substituting values in a container's command [Conformance]", func() {
 		podName := "var-expansion-" + string(util.NewUUID())
 		pod := &api.Pod{
 			ObjectMeta: api.ObjectMeta{
@@ -96,7 +96,7 @@ var _ = Describe("Variable Expansion", func() {
 		})
 	})
 
-	It("should allow substituting values in a container's args", func() {
+	It("should allow substituting values in a container's args [Conformance]", func() {
 		podName := "var-expansion-" + string(util.NewUUID())
 		pod := &api.Pod{
 			ObjectMeta: api.ObjectMeta{

diff --git a/test/e2e/host_path.go b/test/e2e/host_path.go
@@ -59,7 +59,7 @@ var _ = Describe("hostPath", func() {
 		}
 	})
 
-	It("should give a volume the correct mode", func() {
+	It("should give a volume the correct mode [Conformance]", func() {
 		volumePath := "/test-volume"
 		source := &api.HostPathVolumeSource{
 			Path: "/tmp",
@@ -76,7 +76,7 @@ var _ = Describe("hostPath", func() {
 			namespace.Name)
 	})
 
-	It("should support r/w", func() {
+	It("should support r/w [Conformance]", func() {
 		volumePath := "/test-volume"
 		filePath := path.Join(volumePath, "test-file")
 		retryDuration := 180

diff --git a/test/e2e/kube-ui.go b/test/e2e/kube-ui.go
@@ -38,7 +38,7 @@ var _ = Describe("kube-ui", func() {
 
 	f := NewFramework("kube-ui")
 
-	It("should check that the kube-ui instance is alive", func() {
+	It("should check that the kube-ui instance is alive [Conformance]", func() {
 		By("Checking the kube-ui service exists.")
 		err := waitForService(f.Client, uiNamespace, uiServiceName, true, poll, serviceStartTimeout)
 		Expect(err).NotTo(HaveOccurred())