Checking the behavior of cstor pool pod when a node is drained (opene…

…bs#1904) Signed-off-by: gprasath <giridhara.prasad@cloudbyte.com>
dhruvikn · Sep 27, 2018 · 16f1739 · 16f1739
1 parent 690394c
commit 16f1739
Show file tree

Hide file tree

Showing 4 changed files with 260 additions and 169 deletions.
diff --git a/e2e/ansible/playbooks/hyperconverged/test-k8s-drain-nodes/cleanup.yml b/e2e/ansible/playbooks/hyperconverged/test-k8s-drain-nodes/cleanup.yml
@@ -12,44 +12,48 @@
     ns: "{{namespace}}"
     lkey: name
     lvalue: percona
+
+- block:
 
-- name: uncordon the node which is drained
-  shell: source ~/.profile; kubectl uncordon {{ node1.stdout }}
-  args:
-    executable: /bin/bash
-  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
-  register: result
-  until: "'uncordoned' in result.stdout"
-  delay: 5
-  retries: 10
-  changed_when: True
-
-- name: Delete storage class 
-  shell: source ~/.profile; kubectl delete -f "{{result_kube_home.stdout}}/{{ create_sc }}"
-  args:
-    executable: /bin/bash
-  register: sc_out
-  until: "'deleted' in sc_out.stdout"
-  delay: 10
-  retries: 5
-  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+    - name: uncordon the node which is drained
+      shell: source ~/.profile; kubectl uncordon {{ node1.stdout }}
+      args:
+       executable: /bin/bash
+      delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+      register: result
+      until: "'uncordoned' in result.stdout"
+      delay: 5
+      retries: 10
+      changed_when: True
 
-- name: Replace the replica count in openebs-storageclasses yaml
-  replace:
-    path: "{{result_kube_home.stdout}}/{{ create_sc }}"
-    regexp: '      - name: ReplicaCount\n        value: "{{ (node_count) |int-1 }}"'
-    replace: '      - name: ReplicaCount\n        value: "3"'
-  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+    - name: Delete storage class 
+      shell: source ~/.profile; kubectl delete -f "{{result_kube_home.stdout}}/{{ create_sc }}"
+      args:
+        executable: /bin/bash
+      register: sc_out
+      until: "'deleted' in sc_out.stdout"
+      delay: 10
+      retries: 5
+      delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
 
-- name:  Apply storage class yaml
-  shell: source ~/.profile; kubectl apply -f "{{ result_kube_home.stdout }}/{{ create_sc }}"
-  args:
-    executable: /bin/bash
-  register: sc_out
-  until: "'created' in sc_out.stdout"
-  delay: 10
-  retries: 5
-  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+    - name: Replace the replica count in openebs-storageclasses yaml
+      replace:
+        path: "{{result_kube_home.stdout}}/{{ create_sc }}"
+        regexp: '      - name: ReplicaCount\n        value: "{{ (node_count) |int-1 }}"'
+        replace: '      - name: ReplicaCount\n        value: "3"'
+      delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+    - name:  Apply storage class yaml
+      shell: source ~/.profile; kubectl apply -f "{{ result_kube_home.stdout }}/{{ create_sc }}"
+      args:
+        executable: /bin/bash
+      register: sc_out
+      until: "'created' in sc_out.stdout"
+      delay: 10
+      retries: 5
+      delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+  when: storage_engine == 'jiva'
 
 - name: Remove test artifacts
   file:
@@ -59,4 +63,5 @@
   with_items:
      - "{{test_artifacts}}"
      - "{{ create_sc }}"
+     - "{{ percona_file }}"
 
diff --git a/e2e/ansible/playbooks/hyperconverged/test-k8s-drain-nodes/drain-node-cstor.yml b/e2e/ansible/playbooks/hyperconverged/test-k8s-drain-nodes/drain-node-cstor.yml
@@ -0,0 +1,80 @@
+---
+
+- name: Replace storage-class to use cstor storage engine
+  replace:
+    path: "{{ result_kube_home.stdout }}/{{ percona_file }}"
+    regexp: 'openebs-standard'
+    replace: '{{ cstor_sc }}'
+  delegate_to: "{{ groups['kubernetes-kubemasters'].0 }}"
+
+- name: Deploy Percona application
+  include_tasks: "{{utils_path}}/deploy_task.yml"
+  vars:
+    app_yml: "{{ percona_file }}"
+    ns: "{{ namespace }}"
+
+- name: Check if the application is running.
+  include_tasks: "{{utils_path}}/deploy_check.yml"
+  vars:
+    ns: "{{ namespace }}"
+    lkey: name
+    lvalue: percona
+
+- name: Obtain the number of pools
+  shell: source ~/.profile; kubectl get pods -n {{ operator_ns }} | grep cstor | grep -i running | wc -l
+  args:
+    executable: /bin/bash
+  register: pool_count
+
+- name: Obtaining the cstor pool pods
+  shell: source ~/.profile; kubectl get pods -n {{ operator_ns }} | grep cstor | awk {'print $1'}
+  args:
+    executable: /bin/bash
+  register: cstor_pods
+  delegate_to: "{{ groups['kubernetes-kubemasters'].0}}"
+
+- name: Getting the node name
+  shell: source ~/.profile; kubectl get pods -n {{ operator_ns }} {{ cstor_pods.stdout_lines[0] }} --no-headers -o custom-columns=:spec.nodeName
+  args:
+    executable: /bin/bash
+  register: node
+  delegate_to: "{{ groups['kubernetes-kubemasters'].0}}"
+
+- name: Drain one of the nodes
+  shell: source ~/.profile; kubectl drain {{ node.stdout }} --ignore-daemonsets --force
+  args:
+    executable: /bin/bash
+  delegate_to: "{{ groups['kubernetes-kubemasters'].0}}"
+
+- name: Check if one of the pools is pending.
+  shell: source ~/.profile; kubectl get pods -n {{operator_ns}} | grep cstor | grep -i pending | wc -l
+  args:
+    executable: /bin/bash
+  register: pending_pod
+  until: "'1' in pending_pod.stdout"
+  delay: 30
+  retries: 20
+  delegate_to: "{{ groups['kubernetes-kubemasters'].0}}"
+
+- name: Uncordon the node
+  shell: source ~/.profile; kubectl uncordon {{ node.stdout }}
+  args:
+    executable: /bin/bash
+  register: result
+  delegate_to: "{{ groups['kubernetes-kubemasters'].0}}"
+
+- name: Obtain the number of pools
+  shell: source ~/.profile; kubectl get pods -n {{ operator_ns }} | grep cstor | grep -i running | wc -l
+  args:
+    executable: /bin/bash
+  register: pool_count
+  until: "'3' in pool_count.stdout"
+  delay: 60
+  retries: 20
+
+- name: Check if the application is running.
+  include_tasks: "{{utils_path}}/deploy_check.yml"
+  vars:
+    ns: "{{ namespace }}"
+    lkey: name
+    lvalue: percona
diff --git a/e2e/ansible/playbooks/hyperconverged/test-k8s-drain-nodes/drain-node-jiva.yml b/e2e/ansible/playbooks/hyperconverged/test-k8s-drain-nodes/drain-node-jiva.yml
@@ -0,0 +1,132 @@
+---
+- name: Obtaining storage classes yaml
+  shell: source ~/.profile; kubectl get sc openebs-standard -o yaml > "{{ result_kube_home.stdout }}/{{ create_sc }}"
+  args:
+    executable: /bin/bash
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Replace the replica count in openebs-storageclasses yaml
+  replace:
+    path: "{{ result_kube_home.stdout }}/{{ create_sc }}"
+    regexp: '      - name: ReplicaCount\n        value: "3"'
+    replace: '      - name: ReplicaCount\n        value: "{{ (node_count) |int-1 }}"'
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Delete the existing storage class and create new one
+  shell: source ~/.profile; kubectl delete sc openebs-standard; kubectl apply -f "{{ result_kube_home.stdout }}/{{ create_sc }}"
+  args:
+    executable: /bin/bash
+  register: sc_out
+  until: "'created' in sc_out.stdout"
+  delay: 10
+  retries: 5
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Deploy percona application
+  include_tasks: "{{utils_path}}/deploy_task.yml"
+  vars:
+    app_yml: "{{ percona_file }}"
+    ns: "{{ namespace }}"
+
+- name: Check percona pod is running
+  include_tasks: "{{utils_path}}/deploy_check.yml"
+  vars:
+    ns: "{{namespace}}"
+    lkey: name
+    lvalue: percona
+
+- name: Check if the replica pods are created and running
+  shell: source ~/.profile; kubectl get pods -n {{ namespace }} | grep rep | grep -i running |wc -l
+  args:
+    executable: /bin/bash
+  register: rep_count
+  until: "'2' in rep_count.stdout"
+  delay: 60
+  retries: 5
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Obtaining the PV name
+  shell: source ~/.profile; kubectl get pvc -n {{ namespace }} --no-headers | awk {'print  $3'}
+  args:
+    executable: /bin/bash
+  register: pv_name
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Copy replica_patch.yaml to master
+  copy:
+    src: "{{ patch_file }}"
+    dest: "{{ result_kube_home.stdout }}"
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Get the node name where replica1 is scheduled
+  shell: source ~/.profile; kubectl get po -n {{ namespace }} -o wide | grep "{{ pv_name.stdout }}" | grep rep | awk '{print $7}' | awk 'FNR == 1 {print}'
+  args:
+    executable: /bin/bash 
+  register: node1
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Get the node name where replica2 is scheduled
+  shell: source ~/.profile; kubectl get po -n {{ namespace }} -o wide | grep "{{ pv_name.stdout }}" | grep rep | awk '{print $7}' | awk 'FNR == 2 {print}'
+  args:
+    executable: /bin/bash
+  register: node2
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Update replica1 node name in replica_patch file
+  replace:
+    path: "{{ result_kube_home.stdout }}/{{ patch_file }}"
+    regexp: 'nodename_where_replica_pod_1_got_scheduled'
+    replace: '{{ node1.stdout }}'
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Update replica2 node name in replica_patch file
+  replace:
+    path: "{{ result_kube_home.stdout }}/{{ patch_file }}"
+    regexp: 'nodename_where_replica_pod_2_got_scheduled'
+    replace: '{{ node2.stdout }}'
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Get the deployment name
+  shell: source ~/.profile; kubectl get deploy -n {{ namespace }} |grep "{{ pv_name.stdout }}" | grep rep |awk '{print $1}'
+  args:
+    executable: /bin/bash
+  register: deploy_name
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Apply the node affinity property
+  shell: source ~/.profile; kubectl patch deployment "{{ deploy_name.stdout }}" -n {{ namespace }} -p "$(cat replica_patch.yaml)"
+  args:
+    executable: /bin/bash
+  register: patch_out
+  until: "'patched' in patch_out.stdout"
+  delay: 20
+  retries: 5
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Drain the node where replica is scheduled
+  shell: source ~/.profile; kubectl drain {{ node1.stdout }} --ignore-daemonsets --force
+  args:
+    executable: /bin/bash
+  register: result
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Check the available replicas in deployment
+  shell: source ~/.profile; kubectl get deploy -n {{ namespace }} | grep "{{ pv_name.stdout }}" | grep rep | awk '{print $2}'
+  args:
+    executable: /bin/bash
+  register: available_pods
+  until: "'2' in available_pods.stdout"
+  delay: 30
+  retries: 5
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+
+- name: Check if the replica is in pending state after node drain
+  shell: source ~/.profile; kubectl get pods -n {{ namespace }} -o wide | grep "{{ pv_name.stdout }}" | grep rep | grep Pending
+  args:
+    executable: /bin/bash
+  register: result
+  until: "'Pending' in result.stdout"
+  delay: 30
+  retries: 15
+  delegate_to: "{{groups['kubernetes-kubemasters'].0}}"
+