feat: add kubeflow (#26)

koor-tech · Apr 11, 2024 · 33d6215 · 33d6215
1 parent b12e899
commit 33d6215
Show file tree

Hide file tree

Showing 14 changed files with 470 additions and 5 deletions.
diff --git a/exp/kubeflow/README.md b/exp/kubeflow/README.md
@@ -0,0 +1,146 @@
+# Kubeflow with Ceph
+
+## Tools
+```bash
+brew install argocd
+brew install jq
+```
+
+## Install rook-ceph
+```bash
+helm repo add rook-release https://charts.rook.io/release
+helm repo update
+helm install --create-namespace --namespace rook-ceph rook-ceph rook-release/rook-ceph 
+helm install --namespace rook-ceph rook-ceph-cluster \
+   --set operatorNamespace=rook-ceph rook-release/rook-ceph-cluster
+
+kubectl rook-ceph ceph status
+```
+
+## [Optional] Install KDCC
+```bash
+helm repo add data-control-center https://koor-tech.github.io/data-control-center
+helm repo update
+helm install --create-namespace --namespace rook-ceph data-control-center data-control-center/data-control-center -f deploy/kdcc-values.yaml
+
+kubectl --namespace rook-ceph port-forward svc/data-control-center 8282:8282
+```
+Access `localhost:8282`
+
+## Create OBC
+```bash
+kubectl apply -f deploy/obc.yaml
+```
+
+![](pics/kdcc.png)
+
+## Install argocd
+```bash
+# clone the deploykf repo
+# NOTE: we use 'main', as the latest plugin version always lives there
+git clone -b main https://github.com/deployKF/deployKF.git ./deploykf
+
+# ensure the script is executable
+chmod +x ./deploykf/argocd-plugin/install_argocd.sh
+
+# run the install script
+# WARNING: this will install into your current kubectl context
+bash ./deploykf/argocd-plugin/install_argocd.sh
+
+kubectl get po -n argocd
+```
+
+To access:
+```bash
+kubectl port-forward svc/argocd-server -n argocd 8081:443
+```
+
+Access `localhost:8081`
+
+The initial password is: 
+```bash
+kubectl get -n argocd secret/argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d
+```
+
+## Install app of apps
+```bash
+kubectl apply -f deploy/deploykf-app-of-apps.yaml
+```
+
+## Sync Argocd
+```bash
+# clone the deploykf repo
+# NOTE: we use 'main', as the latest script always lives there
+git clone -b main https://github.com/deployKF/deployKF.git ./deploykf
+
+# ensure the script is executable
+chmod +x ./deploykf/scripts/sync_argocd_apps.sh
+
+# run the script
+bash ./deploykf/scripts/sync_argocd_apps.sh
+```
+
+![](pics/argocd.png)
+
+## Access the platform
+Modify hosts:
+You will need to add the following lines to the END of your local /etc/hosts file:
+
+```
+127.0.0.1 deploykf.example.com
+127.0.0.1 argo-server.deploykf.example.com
+127.0.0.1 minio-api.deploykf.example.com
+127.0.0.1 minio-console.deploykf.example.com
+```
+
+Port forward:
+```bash
+kubectl port-forward \
+  --namespace "deploykf-istio-gateway" \
+  svc/deploykf-gateway 8080:http 8443:https
+```
+
+Go to deploykf.example.com
+
+Username: user1@example.com
+Password: user1
+
+![](pics/dashboard.png)
+
+## Use in a notebook
+### Create a bucket
+```bash
+kubectl apply -f deploy/team-1-obc.yaml
+```
+
+### Create a PodDefault 
+```bash
+kubectl apply -f deploy/pod-default-add-obc.yaml
+```
+
+When creating a notebook, select the configuration under advanced settings
+
+![](pics/notebook-config.png)
+
+In the notebook, install boto3
+```
+!pip install boto3
+```
+
+Then use it as follows
+```python
+import os
+import boto3
+
+aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
+aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
+endpoint_url = "http://" + os.environ['BUCKET_HOST']
+
+s3_client = boto3.client('s3', 
+    aws_access_key_id=aws_access_key_id,
+    aws_secret_access_key=aws_secret_access_key,
+    endpoint_url=endpoint_url,
+    use_ssl=False,
+)
+s3_client.list_buckets()
+```
diff --git a/exp/kubeflow/deploy/deploykf-app-of-apps.yaml b/exp/kubeflow/deploy/deploykf-app-of-apps.yaml
@@ -0,0 +1,123 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: deploykf-app-of-apps
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: deploykf-app-of-apps
+    app.kubernetes.io/part-of: deploykf
+spec:
+  project: "default"
+  source:
+    ## source git repo configuration
+    ##  - we use the 'deploykf/deploykf' repo so we can read its 'sample-values.yaml'
+    ##    file, but you may use any repo (even one with no files)
+    ##
+    repoURL: "https://github.com/deployKF/deployKF.git"
+    targetRevision: "v0.1.4"
+    path: "."
+
+    ## plugin configuration
+    ##
+    plugin:
+      name: "deploykf"
+      parameters:
+
+        ## the deployKF generator version
+        ##  - available versions: https://github.com/deployKF/deployKF/releases
+        ##
+        - name: "source_version"
+          string: "0.1.4"
+
+        ## paths to values files within the `repoURL` repository
+        ##  - the values in these files are merged, with later files taking precedence
+        ##  - we strongly recommend using 'sample-values.yaml' as the base of your values
+        ##    so you can easily upgrade to newer versions of deployKF
+        ##
+        - name: "values_files"
+          array:
+            - "./sample-values.yaml"
+
+        ## a string containing the contents of a values file
+        ##  - this parameter allows defining values without needing to create a file in the repo
+        ##  - these values are merged with higher precedence than those defined in `values_files`
+        ##
+        - name: "values"
+          string: |
+            deploykf_core:
+              deploykf_auth:
+              # [optional] allows access to rook's obc
+                extraManifests:
+                  - |
+                    apiVersion: rbac.authorization.k8s.io/v1
+                    kind: ClusterRole
+                    metadata:
+                      name: rook-obc-edit
+                      labels:
+                        helm.sh/chart: {{ include "deploykf-auth.labels.chart" . }}
+                        app.kubernetes.io/name: {{ include "deploykf-auth.labels.name" . }}
+                        app.kubernetes.io/instance: {{ .Release.Name }}
+                        app.kubernetes.io/managed-by: {{ .Release.Service }}
+                        app.kubernetes.io/component: cluster-rbac
+                        rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true"
+                    rules:
+                      - apiGroups:
+                          - objectbucket.io
+                        resources:
+                          - objectbucketclaims
+                        verbs:
+                          - create
+                          - delete
+                          - deletecollection
+                          - get
+                          - list
+                          - patch
+                          - update
+                          - watch
+
+              deploykf_dashboard:
+                navigation:
+                  # [Optional] If Koor Data Control Center is used, you can add a link here
+                  externalLinks:
+                    - text: Koor Data Control Center
+                      url: http://localhost:8282
+                      icon: dashboard
+
+              deploykf_profiles_generator:
+                profileDefaults:
+                  tools:
+                    kubeflowPipelines:
+                      objectStoreAuth:
+                        existingSecret: "kubeflow-pipelines"
+                        existingSecretNamespace: "kubeflow"
+                        existingSecretAccessKeyKey: "AWS_ACCESS_KEY_ID"
+                        existingSecretSecretKeyKey: "AWS_SECRET_ACCESS_KEY"
+
+            deploykf_opt:
+              deploykf_minio:
+                enabled: false
+
+            kubeflow_tools:
+              pipelines:
+                bucket:
+                  name: kubeflow-pipelines
+                  region: ""
+
+                objectStore:
+                  useExternal: true
+                  host: rook-ceph-rgw-ceph-objectstore.rook-ceph.svc
+                  port: "80"
+                  useSSL: false
+                  auth:
+                    existingSecret: "kubeflow-pipelines"
+                    existingSecretAccessKeyKey: "AWS_ACCESS_KEY_ID"
+                    existingSecretSecretKeyKey: "AWS_SECRET_ACCESS_KEY"
+
+                ## NOTE: only required if you are using 'sample-values.yaml' as a base
+                ##       as `minioFix` can only be 'true' when using the embedded MinIO
+                kfpV2:
+                  minioFix: false
+
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: "argocd"
diff --git a/exp/kubeflow/deploy/kdcc-values.yaml b/exp/kubeflow/deploy/kdcc-values.yaml
@@ -0,0 +1,4 @@
+config:
+  users:
+    - username: admin
+      password: password
diff --git a/exp/kubeflow/deploy/kubeflow-obc.yaml b/exp/kubeflow/deploy/kubeflow-obc.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kubeflow
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: kubeflow-pipelines
+  namespace: kubeflow
+spec:
+  bucketName: kubeflow-pipelines
+  storageClassName: ceph-bucket
diff --git a/exp/kubeflow/deploy/pod-default-add-obc.yaml b/exp/kubeflow/deploy/pod-default-add-obc.yaml
@@ -0,0 +1,15 @@
+apiVersion: kubeflow.org/v1alpha1
+kind: PodDefault
+metadata:
+  name: add-bucket-config
+  namespace: team-1
+spec:
+  selector:
+    matchLabels:
+      add-bucket-config: "true"
+  desc: "Add rook bucket"
+  envFrom:
+    - configMapRef:
+        name: team-1-bucket
+    - secretRef:
+        name: team-1-bucket
diff --git a/exp/kubeflow/deploy/team-1-obc.yaml b/exp/kubeflow/deploy/team-1-obc.yaml
@@ -0,0 +1,8 @@
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: team-1-bucket
+  namespace: team-1
+spec:
+  generateBucketName: team-1-bucket
+  storageClassName: ceph-bucket
diff --git a/exp/kubeflow/deploy/team-1-rbac.yaml b/exp/kubeflow/deploy/team-1-rbac.yaml
@@ -0,0 +1,20 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rook-obc-edit
+  labels:
+    rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true"
+rules:
+  - apiGroups:
+      - objectbucket.io
+    resources:
+      - objectbucketclaims
+    verbs:
+      - create
+      - delete
+      - deletecollection
+      - get
+      - list
+      - patch
+      - update
+      - watch
diff --git a/exp/kubeflow/pics/argocd.png b/exp/kubeflow/pics/argocd.png
diff --git a/exp/kubeflow/pics/dashboard.png b/exp/kubeflow/pics/dashboard.png
diff --git a/exp/kubeflow/pics/kdcc.png b/exp/kubeflow/pics/kdcc.png
diff --git a/exp/kubeflow/pics/notebook-config.png b/exp/kubeflow/pics/notebook-config.png