kfctl: existing_arrikto: initial e2e test

Signed-off-by: Yannis Zarkadas <yanniszark@arrikto.com>
kubeflow · Sep 20, 2019 · 4f92110 · 4f92110
1 parent 2f15d8d
commit 4f92110
Show file tree

Hide file tree

Showing 8 changed files with 141 additions and 5 deletions.
diff --git a/bootstrap/config/kfctl_existing_arrikto.yaml b/bootstrap/config/kfctl_existing_arrikto.yaml
@@ -1,5 +1,5 @@
 # This is the config to install Kubeflow on an existing K8s cluster, with support
-# for multi-user and LDAP auth using Dex.
+# for multi-user and LDAP auth using Dex. 
 
 apiVersion: kfdef.apps.kubeflow.org/v1alpha1
 kind: KfDef
@@ -222,4 +222,4 @@ spec:
     uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
   - name: kubeflow
     root: kubeflow-master
-    uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
+    uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
diff --git a/prow_config.yaml b/prow_config.yaml
@@ -83,6 +83,27 @@ workflows:
       useBasicAuth: true
       useIstio: true
       configPath: https://raw.githubusercontent.com/kubeflow/manifests/master/kfdef/kfctl_gcp_basic_auth.yaml
+  # E2E tests for kfctl_existing_arrikto
+  - app_dir: kubeflow/kubeflow/testing/workflows
+    component: kfctl_go_test
+    name: kfctl-go-existing
+    job_types:
+      - presubmit
+    include_dirs:
+      - bootstrap/*
+      - deployment/*
+      - dependencies/*
+      - kubeflow/*
+      - testing/*
+    params:
+      platform: gke
+      gkeApiVersion: v1
+      workflowName: kfctl-go
+      useBasicAuth: false
+      useIstio: true
+      testEndpoint: false
+      configPath: https://raw.githubusercontent.com/kubeflow/manifests/master/kfdef/kfctl_existing_arrikto.yaml
+      cluster_creation_script: create_existing_cluster.sh
   # Only run  kfctl presubmit test with basic auth if
   # files related to basic auth are modified.
   - app_dir: kubeflow/kubeflow/testing/workflows

diff --git a/testing/kfctl/conftest.py b/testing/kfctl/conftest.py
@@ -32,6 +32,10 @@ def pytest_addoption(parser):
   parser.addoption(
       "--use_istio", action="store", default="False",
       help="Use istio.")
+
+  parser.addoption(
+      "--cluster_creation_script", action="store", default="",
+      help="The script to use to create a K8s cluster before running kfctl.")
 
 @pytest.fixture
 def app_path(request):
@@ -57,6 +61,10 @@ def project(request):
 def config_path(request):
   return request.config.getoption("--config_path")
 
+@pytest.fixture
+def cluster_creation_script(request):
+  return request.config.getoption("--cluster_creation_script")
+
 @pytest.fixture
 def use_basic_auth(request):
   value = request.config.getoption("--use_basic_auth").lower()

diff --git a/testing/kfctl/create_existing_cluster.sh b/testing/kfctl/create_existing_cluster.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+export PROJECT="kubeflow-ci"
+export GCP_ZONE="us-central1-a"
+export GCP_USER="$(gcloud config list account --format "value(core.account)" )"
+export GCP_PROJECT="$(gcloud config list project --format "value(core.project)" )"
+export CLUSTER_NAME="kfctl-existing-arrikto-cluster"
+export CLUSTER_VERSION="$(gcloud container get-server-config --zone=${GCP_ZONE} --format="value(validMasterVersions[0])" )"
+
+############################
+# Create and setup cluster #
+############################
+
+gcloud container clusters create "${CLUSTER_NAME}" \
+--project "${GCP_PROJECT}" \
+--zone "${GCP_ZONE}" \
+--username "admin" \
+--cluster-version "${CLUSTER_VERSION}" \
+--machine-type "n1-standard-4" --num-nodes "1" \
+--image-type "UBUNTU" \
+--local-ssd-count=4 \
+--disk-type "pd-ssd" --disk-size "50" \
+--no-enable-cloud-logging --no-enable-cloud-monitoring \
+--no-enable-ip-alias \
+--enable-network-policy \
+--enable-autoupgrade --enable-autorepair
+
+echo "Getting credentials for newly created cluster..."
+gcloud container clusters get-credentials "${CLUSTER_NAME}" --zone="${GCP_ZONE}"
+
+echo "Setting up GKE RBAC..."
+kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user="${GCP_USER}"
diff --git a/testing/kfctl/delete_existing_cluster.py b/testing/kfctl/delete_existing_cluster.py
@@ -0,0 +1,32 @@
+import logging
+import os
+import pytest
+
+from kubeflow.testing import util
+
+
+def test_create_existing_cluster(cluster_creation_script):
+    if not cluster_creation_script:
+        logging.fatal("--script to create cluster was not provided.")
+
+    # Need to activate account for scopes.
+    if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"):
+        util.run([
+            "gcloud", "auth", "activate-service-account",
+            "--key-file=" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
+        ])
+
+    logging.info("Deleting existing cluster...")
+    util.run(["gcloud", "container", "clusters", "delete",
+             "kfctl-existing-arrikto-cluster", "--zone", "us-central1-a"])
+
+
+if __name__ == "__main__":
+  logging.basicConfig(
+      level=logging.INFO,
+      format=("%(levelname)s|%(asctime)s"
+              "|%(pathname)s|%(lineno)d| %(message)s"),
+      datefmt="%Y-%m-%dT%H:%M:%S",
+  )
+  logging.getLogger().setLevel(logging.INFO)
+  pytest.main()
diff --git a/testing/kfctl/kfctl_go_test.py b/testing/kfctl/kfctl_go_test.py
@@ -2,7 +2,6 @@
 import logging
 import os
 import requests
-import subprocess
 import tempfile
 import uuid
 import urllib
@@ -50,7 +49,7 @@ def verify_kubeconfig(project, zone, app_path):
     raise RuntimeError(msg)
 
 
-def test_build_kfctl_go(app_path, project, use_basic_auth, use_istio, config_path):
+def test_build_kfctl_go(app_path, project, use_basic_auth, use_istio, config_path, cluster_creation_script):
   """Test building and deploying Kubeflow.
 
   Args:
@@ -149,6 +148,11 @@ def test_build_kfctl_go(app_path, project, use_basic_auth, use_istio, config_pat
     logging.info(str(config_spec))
   with open(os.path.join(parent_dir, "tmp.yaml"), "w") as f:
     yaml.dump(config_spec, f)
+
+  if cluster_creation_script:
+    logging.info("Cluster creation script specified: %s", cluster_creation_script)
+    util.run(["/bin/bash", "-c", cluster_creation_script], cwd=os.getcwd())
+
   util.run([
       kfctl_path, "init", app_path, "-V",
       "--config=" + os.path.join(parent_dir, "tmp.yaml")], cwd=parent_dir)
@@ -159,13 +163,17 @@ def test_build_kfctl_go(app_path, project, use_basic_auth, use_istio, config_pat
   ],
                    cwd=app_path)
 
+  # Create $HOME/.kubeconfig file for kfctl to consume.
+  # util.run(["gcloud", "container", "clusters", "get-credentials", "kubeflow-testing", "--zone", zone])
+
   # We need to use retries because if we don't we see random failures
   # where kfctl just appears to die.
   #
   # Do not run with retries since it masks errors
   util.run([kfctl_path, "apply", "-V", "all"], cwd=app_path)
 
-  verify_kubeconfig(project, zone, app_path)
+  if not cluster_creation_script:
+    verify_kubeconfig(project, zone, app_path)
 
 def filterSpartakus(spec):
   for i, app in enumerate(spec["applications"]):

diff --git a/testing/workflows/components/kfctl_go_test.jsonnet b/testing/workflows/components/kfctl_go_test.jsonnet
@@ -33,6 +33,12 @@ local runPath = srcDir + "/testing/workflows/run.sh";
 local kfCtlPath = srcDir + "/bootstrap/bin/kfctl";
 local kubeConfig = testDir + "/kfctl_test/.kube/kubeconfig";
 
+// cluster_creation_script specifies the script to run in order to create 
+// a cluster before running kfctl.
+// Only applicable to configs that don't create their own clusters.
+local cluster_creation_script = srcDir + "/testing/kfctl/" + params.cluster_creation_script;
+local delete_cluster = if (params.cluster_creation_script=="") then false else true;
+
 // Name for the Kubeflow app.
 // This needs to be unique for each test run because it is
 // used to name GCP resources
@@ -226,6 +232,7 @@ local dagTemplates = [
         "--use_basic_auth=" + params.useBasicAuth,
         "--use_istio=" + params.useIstio,
         "--config_path=" + params.configPath,
+        "--cluster_creation_script=" + cluster_creation_script,
         // Increase the log level so that info level log statements show up.
         "--log-cli-level=info",        
         "--junitxml=" + artifactsDir + "/junit_kfctl-build-test" + nameSuffix + ".xml",
@@ -319,6 +326,30 @@ local deleteStep = if deleteKubeflow then
   }]
 else [];
 
+local deleteClusterStep = if delete_cluster then
+  [{
+    template: buildTemplate(
+      "kfctl-delete-cluster",
+      [
+        "pytest",
+        "delete_existing_cluster.py",
+        // I think -s mean stdout/stderr will print out to aid in debugging.
+        // Failures still appear to be captured and stored in the junit file.
+        "-s",
+        // Increase the log level so that info level log statements show up.
+        "--log-cli-level=info",
+        // Test timeout in seconds.
+        "--timeout=1000",
+        "--cluster_creation_script=" + cluster_creation_script,
+        "--junitxml=" + artifactsDir + "/junit_kfctl-go-delete-cluster-test.xml",
+      ],
+      working_dir=srcDir+ "/testing/kfctl",
+    ),
+    dependencies: null,
+  }]
+else [];
+
+
 local testDirDeleteStep = {
       template:
         buildTemplate("test-dir-delete", [
@@ -336,6 +367,7 @@ local testDirDeleteStep = {
 
 local exitTemplates =
   deleteStep +
+  deleteClusterStep +
   [
     {
       template: buildTemplate("copy-artifacts", [

diff --git a/testing/workflows/components/params.libsonnet b/testing/workflows/components/params.libsonnet
@@ -43,6 +43,7 @@
       useIstio: "true",
       testEndpoint: "false",
       configPath: "bootstrap/config/kfctl_gcp_iap_master.yaml",
+      cluster_creation_script: "",
     },
     click_deploy_test: {
       bucket: "kubeflow-ci_temp",