be more conservative in adding new test

Signed-off-by: Yannis Zarkadas <yanniszark@arrikto.com>
kubeflow · Oct 11, 2019 · d687f30 · d687f30
1 parent 1e89194
commit d687f30
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 6 deletions.
diff --git a/prow_config.yaml b/prow_config.yaml
@@ -90,7 +90,8 @@ workflows:
     component: kfctl_go_test
     name: kfctl-go-existing
     job_types:
-      - presubmit
+      # Enable once we have confirmed the stability of the test
+      # - presubmit
       - postsubmit
     include_dirs:
       - bootstrap/*

diff --git a/testing/kfctl/kf_is_ready_test.py b/testing/kfctl/kf_is_ready_test.py
@@ -43,11 +43,6 @@ def test_kf_is_ready(namespace, use_basic_auth, use_istio, app_path):
       "katib-manager",
       "katib-manager-rest",
       "katib-ui",
-      "metadata-db",
-      "metadata-deployment",
-      "metadata-envoy-deployment",
-      "metadata-grpc-deployment",
-      "metadata-ui",
       "minio",
       "ml-pipeline",
       "ml-pipeline-persistenceagent",
@@ -63,6 +58,19 @@ def test_kf_is_ready(namespace, use_basic_auth, use_istio, app_path):
       "workflow-controller",
   ]
 
+  # TODO(yanniszark): these deployments seems to cause flakiness in the test.
+  # More specifically, they take a long time to get ready.
+  # Look into creating test clusters with bigger machines or longer timeout limits.
+  postsubmit_deployments = [
+      "metadata-db",
+      "metadata-deployment",
+      "metadata-envoy-deployment",
+      "metadata-grpc-deployment",
+      "metadata-ui",
+  ]
+  if os.getenv("JOB_TYPE") == "postsubmit":
+    deployment_names.extend(postsubmit_deployments)
+
   stateful_set_names = [
     "admission-webhook-bootstrap-stateful-set",
     "application-controller-stateful-set",