parameterize kf_is_ready check

Signed-off-by: Yannis Zarkadas <yanniszark@arrikto.com>
kubeflow · Sep 23, 2019 · 5b69308 · 5b69308
1 parent 3bb729c
commit 5b69308
Show file tree

Hide file tree

Showing 6 changed files with 187 additions and 4 deletions.
diff --git a/prow_config.yaml b/prow_config.yaml
@@ -83,7 +83,7 @@ workflows:
       useBasicAuth: true
       useIstio: true
       configPath: https://raw.githubusercontent.com/kubeflow/manifests/master/kfdef/kfctl_gcp_basic_auth.yaml
-  # E2E tests for kfctl_existing_arrikto
+  # E2E tests for kfctl_existing_arrikto 
   - app_dir: kubeflow/kubeflow/testing/workflows
     component: kfctl_go_test
     name: kfctl-go-existing
@@ -104,6 +104,7 @@ workflows:
       testEndpoint: false
       configPath: https://raw.githubusercontent.com/kubeflow/manifests/master/kfdef/kfctl_existing_arrikto.yaml
       cluster_creation_script: create_existing_cluster.sh
+      checklist: kfctl_existing_arrikto_checklist.yaml
   # Only run  kfctl presubmit test with basic auth if
   # files related to basic auth are modified.
   - app_dir: kubeflow/kubeflow/testing/workflows

diff --git a/testing/kfctl/conftest.py b/testing/kfctl/conftest.py
@@ -36,6 +36,11 @@ def pytest_addoption(parser):
   parser.addoption(
       "--cluster_creation_script", action="store", default="",
       help="The script to use to create a K8s cluster before running kfctl.")
+
+  parser.addoption(
+      "--checklist", action="store", default="",
+      help="Checklist of Deployments/StatefulSets"
+  )
 
 @pytest.fixture
 def app_path(request):
@@ -65,6 +70,10 @@ def config_path(request):
 def cluster_creation_script(request):
   return request.config.getoption("--cluster_creation_script")
 
+@pytest.fixture
+def checklist(request):
+  return request.config.getoption("--checklist")
+
 @pytest.fixture
 def use_basic_auth(request):
   value = request.config.getoption("--use_basic_auth").lower()

diff --git a/testing/kfctl/kf_is_ready_checklists/kfctl_existing_arrikto_checklist.yaml b/testing/kfctl/kf_is_ready_checklists/kfctl_existing_arrikto_checklist.yaml
@@ -0,0 +1,153 @@
+# Deployments in namespace kubeflow
+- name: admission-webhook-deployment
+  namespace: kubeflow
+  kind: Deployment
+- name: argo-ui
+  namespace: kubeflow
+  kind: Deployment
+- name: centraldashboard
+  namespace: kubeflow
+  kind: Deployment
+- name: dex
+  namespace: kubeflow
+  kind: Deployment
+- name: jupyter-web-app-deployment
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-controller
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-db
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-manager
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-manager-rest
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-suggestion-bayesianoptimization
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-suggestion-grid
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-suggestion-hyperband
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-suggestion-nasrl
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-suggestion-random
+  namespace: kubeflow
+  kind: Deployment
+- name: katib-ui
+  namespace: kubeflow
+  kind: Deployment
+- name: metadata-db
+  namespace: kubeflow
+  kind: Deployment
+- name: metadata-deployment
+  namespace: kubeflow
+  kind: Deployment
+- name: metadata-ui
+  namespace: kubeflow
+  kind: Deployment
+- name: minio
+  namespace: kubeflow
+  kind: Deployment
+- name: ml-pipeline
+  namespace: kubeflow
+  kind: Deployment
+- name: ml-pipeline-persistenceagent
+  namespace: kubeflow
+  kind: Deployment
+- name: ml-pipeline-scheduledworkflow
+  namespace: kubeflow
+  kind: Deployment
+- name: ml-pipeline-ui
+  namespace: kubeflow
+  kind: Deployment
+- name: ml-pipeline-viewer-controller-deployment
+  namespace: kubeflow
+  kind: Deployment
+- name: mysql
+  namespace: kubeflow
+  kind: Deployment
+- name: notebook-controller-deployment
+  namespace: kubeflow
+  kind: Deployment
+- name: profiles-deployment
+  namespace: kubeflow
+  kind: Deployment
+- name: pytorch-operator
+  namespace: kubeflow
+  kind: Deployment
+- name: tensorboard
+  namespace: kubeflow
+  kind: Deployment
+- name: tf-job-dashboard
+  namespace: kubeflow
+  kind: Deployment
+- name: tf-job-operator
+  namespace: kubeflow
+  kind: Deployment
+- name: workflow-controller
+  namespace: kubeflow
+  kind: Deployment
+
+# StatefulSets in namespace kubeflow
+
+- name: admission-webhook-bootstrap-stateful-set
+  namespace: kubeflow
+  kind: StatefulSet
+- name: application-controller-stateful-set
+  namespace: kubeflow
+  kind: StatefulSet
+- name: metacontroller
+  namespace: kubeflow
+  kind: StatefulSet
+- name: seldon-operator-controller-manager
+  namespace: kubeflow
+  kind: StatefulSet
+
+# Deployments in namespace istio-system
+- name: authservice
+  namespace: istio-system
+  kind: Deployment
+- name: grafana
+  namespace: istio-system
+  kind: Deployment
+- name: istio-citadel
+  namespace: istio-system
+  kind: Deployment
+- name: istio-egressgateway
+  namespace: istio-system
+  kind: Deployment
+- name: istio-galley
+  namespace: istio-system
+  kind: Deployment
+- name: istio-ingressgateway
+  namespace: istio-system
+  kind: Deployment
+- name: istio-pilot
+  namespace: istio-system
+  kind: Deployment
+- name: istio-policy
+  namespace: istio-system
+  kind: Deployment
+- name: istio-sidecar-injector
+  namespace: istio-system
+  kind: Deployment
+- name: istio-telemetry
+  namespace: istio-system
+  kind: Deployment
+- name: istio-tracing
+  namespace: istio-system
+  kind: Deployment
+- name: kiali
+  namespace: istio-system
+  kind: Deployment
+- name: prometheus
+  namespace: istio-system
+  kind: Deployment
diff --git a/testing/kfctl/kf_is_ready_test.py b/testing/kfctl/kf_is_ready_test.py
@@ -4,14 +4,15 @@
 import subprocess
 import tempfile
 import uuid
+import yaml
 from retrying import retry
 
 import pytest
 
 from kubeflow.testing import util
 from testing import deploy_utils
 
-def test_kf_is_ready(namespace, use_basic_auth, use_istio):
+def test_kf_is_ready(namespace, use_basic_auth, use_istio, checklist):
   """Test that Kubeflow was successfully deployed.
 
   Args:
@@ -29,6 +30,20 @@ def test_kf_is_ready(namespace, use_basic_auth, use_istio):
 
   util.load_kube_config()
 
+  # If a checklist is provided, use that instead.
+  if checklist:
+    with open(checklist) as f:
+      items = yaml.safe_load(f)
+    for item in items:
+      if item["kind"] == "Deployment":
+        util.wait_for_deployment(api_client, item["namespace"], item["name"], 10)
+      elif item["kind"] == "StatefulSet":
+        util.wait_for_statefulset(api_client, item["namespace"], item["name"], 10)
+      else:
+        logging.fatal("Unrecognised kind '%s' in checklist.", item["kind"])
+        raise ValueError()
+    return
+
   # Verify that components are actually deployed.
   # TODO(jlewi): We need to parameterize this list based on whether
   # we are using IAP or basic auth.

diff --git a/testing/workflows/components/kfctl_go_test.jsonnet b/testing/workflows/components/kfctl_go_test.jsonnet
@@ -36,10 +36,13 @@ local kubeConfig = testDir + "/kfctl_test/.kube/kubeconfig";
 // cluster_creation_script specifies the script to run in order to create 
 // a cluster before running kfctl.
 // Only applicable to configs that don't create their own clusters.
-local cluster_creation_script = srcDir + "/testing/kfctl/" + params.cluster_creation_script;
+local cluster_creation_script = if (params.cluster_creation_script=="") then "" else srcDir + "/testing/kfctl/" + params.cluster_creation_script;
 local delete_cluster = if (params.cluster_creation_script=="") then false else true;
 
-// Name for the Kubeflow app.
+// checklist is the path to a list of deployments/statefulsets to check
+// for ready status
+local checklist = if (params.checklist=="") then "" else srcDir + "/testing/kfctl/kf_is_ready_checklists/" + params.checklist;
+
 // This needs to be unique for each test run because it is
 // used to name GCP resources
 // We take the suffix of the name because it should provide some random salt.
@@ -262,6 +265,7 @@ local dagTemplates = [
         // Test suite name needs to be unique based on parameters
         "-o", "junit_suite_name=test_kf_is_ready_" + nameSuffix,         
         "--app_path=" + appDir,
+        "--checklist=" + checklist,
       ],
       working_dir=srcDir+ "/testing/kfctl",
     ),

diff --git a/testing/workflows/components/params.libsonnet b/testing/workflows/components/params.libsonnet
@@ -44,6 +44,7 @@
       testEndpoint: "false",
       configPath: "bootstrap/config/kfctl_gcp_iap_master.yaml",
       cluster_creation_script: "",
+      checklist: "",
     },
     click_deploy_test: {
       bucket: "kubeflow-ci_temp",